diff --git a/CMakeLists.txt b/CMakeLists.txt index 566d4ad29d7dff9c43ab61cac03da9a4dfdef63a..fb2b306f65467361dd5612f5bf1e6629a7a0c306 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,8 @@ if (NOT DEFINED TD_SOURCE_DIR) set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} ) endif() +SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) + set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib") diff --git a/CODE_OF_CONDUCT-CN.md b/CODE_OF_CONDUCT-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..cd14ea9c1c5c51aeaa93f7e6eccd4108a0ff9ec4 --- /dev/null +++ b/CODE_OF_CONDUCT-CN.md @@ -0,0 +1,47 @@ +# 贡献者契约行为准则 + +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v1.4%20adopted-ff69b4.svg)](code_of_conduct.md) + +## 我们的承诺 + +为了营造一个开放和热情的环境，作为贡献者和维护者，我们承诺让每个人参与我们的项目和社区成为一种无骚扰的体验，无论年龄、体型、残疾、种族、性别特征、性别认同和表达、经验水平、教育、社会经济地位、国籍、个人外表、种族、宗教或性认同和取向如何。 + +## 我们的标准 + +有助于创造积极环境的行为示例包括： + +- 使用热情和包容的语言 +- 尊重不同的观点和经历 +- 优雅地接受建设性的批评 +- 专注于对社区最有利的事情 +- 对其他社区成员表示同情 + +参与者不可接受的行为示例包括： + +- 使用性感的语言或图像以及不受欢迎的性关注或进步 +- 拖钓、侮辱/贬损评论以及人身或政治攻击 +- 公共或私人骚扰 +- 未经明确许可发布他人的私人信息，例如物理地址或电子地址 +- 在专业环境中可能被合理认为不适当的其他行为 + +## 我们的责任 + +项目维护人员负责阐明可接受行为的标准，并期望针对任何不可接受行为的情况采取适当和公平的纠正措施。 + +项目维护者有权利和责任删除、编辑或拒绝评论、提交、代码、wiki 编辑、问题和其他不符合本行为准则的贡献，或暂时或永久禁止任何贡献者从事他们认为不适当、威胁、冒犯或有害的其他行为。 + +## 范围 + +本行为准则适用于所有项目空间，也适用于个人在公共场所代表项目或其社区时。 代表项目或社区的示例包括使用官方项目电子邮件地址、通过官方社交媒体帐户发布信息或在在线或离线活动中担任指定代表。 项目的表示可以由项目维护者进一步定义和澄清。 + +## 执法 + +可以通过 support@taosdata.com 联系项目团队来报告辱骂、骚扰或其他不可接受的行为。 所有投诉都将被审查和调查，并将产生被认为必要且适合具体情况的回应。 项目团队有义务对事件的报告者保密。 具体执行政策的更多细节可能会单独发布。 + +不善意遵守或执行行为准则的项目维护者可能会面临由项目领导的其他成员确定的临时或永久影响。 + +## 来源 + +本行为准则改编自贡献者公约 1.4 版，可在 https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 获取 + +有关此行为准则的常见问题的答案，请参阅 https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/Jenkinsfile2 b/Jenkinsfile2 index bb7fa70c57fb7957967b59c04ce908a417ad4b56..55bd5466ed4abe40cfaedc634b75a59b43668759 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -460,7 +460,7 @@ pipeline { cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 600 ''' + extra_param + ''' ''' } } diff --git a/cmake/cmake.define b/cmake/cmake.define index 10f217254152db82518099560ef902b2ebabf499..5b65738c700bd7fa862da6284881ada17f245302 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -121,7 +121,7 @@ ELSE () MESSAGE(STATUS "Compile with Address Sanitizer!") ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () # disable all assert diff --git a/cmake/cmake.version b/cmake/cmake.version index d38ac40b90c932b41717d6d16dadc176a842a2b2..232e86d891afebc524c1a6b2df78fac6a9137211 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.3.0") + SET(TD_VER_NUMBER "3.0.4.0") ENDIF () IF (DEFINED VERCOMPATIBLE) @@ -16,7 +16,7 @@ find_program(HAVE_GIT NAMES git) IF (DEFINED GITINFO) SET(TD_VER_GIT ${GITINFO}) ELSEIF (HAVE_GIT) - execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID) #message(STATUS "git log result:${GIT_COMMITID}") IF (GIT_COMMITID) string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) @@ -30,6 +30,23 @@ ELSE () SET(TD_VER_GIT "no git commit id") ENDIF () +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () +ELSE () + message(STATUS "no git cmd") + SET(TD_VER_GIT_INTERNAL "no git commit id") +ENDIF () + IF (DEFINED VERDATE) SET(TD_VER_DATE ${VERDATE}) ELSE () diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 1c401ae80e0a7a1ec11701c39842f5a0eb1052ce..ba937b40c14305dbf45ccc81db9bb79ed9d89cb5 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG d8059ff + GIT_TAG e02ddb2 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 736ab0b09514295557e59f645fb1db2d03d7eada..4838e97dd79c55d58c0442a065a3c1e3171895b2 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG d9ec91d + GIT_TAG ffc2e6f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/compile_flags.txt b/compile_flags.txt new file mode 100644 index 0000000000000000000000000000000000000000..c61f9701abb7750b5e36c10eac480f61886fd5ca --- /dev/null +++ b/compile_flags.txt @@ -0,0 +1,9 @@ +-DLINUX +-DWEBSOCKET +-I/usr/include +-Iinclude +-Iinclude/os +-Iinclude/common +-Iinclude/util +-Iinclude/libs/transport +-Itools/shell/inc diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index c70c2589c2ce93c92c8114fbf47a830c506280e7..3282f600ac81fd6a4cc45fb8c09e3c036570cc18 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -188,7 +188,7 @@ You can use the TDengine CLI to monitor your TDengine deployment and execute ad -After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengine Server. +After the installation is complete, please run `sc start taosd` or run `C:\TDengine\taosd.exe` with administrator privilege to start TDengine Server. ## Command Line Interface (CLI) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 71abe5905559b2df70977b773b22c2d6cbafcbb0..7465cc0a12ecc9c92a48a2308fe101d057f80492 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -23,7 +23,7 @@ By subscribing to a topic, a consumer can obtain the latest data in that topic i To implement these features, TDengine indexes its write-ahead log (WAL) file for fast random access and provides configurable methods for replacing and retaining this file. You can define a retention period and size for this file. For information, see the CREATE DATABASE statement. In this way, the WAL file is transformed into a persistent storage engine that remembers the order in which events occur. However, note that configuring an overly long retention period for your WAL files makes database compression inefficient. TDengine then uses the WAL file instead of the time-series database as its storage engine for queries in the form of topics. TDengine reads the data from the WAL file; uses a unified query engine instance to perform filtering, transformations, and other operations; and finally pushes the data to consumers. - +Tips:The default data subscription is to consume data from the wal. If the wal is deleted, the consumed data will be incomplete. At this time, you can set the parameter experimental.snapshot.enable to true to obtain all data from the tsdb, but in this way, the consumption order of the data cannot be guaranteed. Therefore, it is recommended to set a reasonable retention policy for WAL based on your consumption situation to ensure that you can subscribe all data from WAL. ## Data Schema and API @@ -285,18 +285,17 @@ You configure the following parameters when creating a consumer: | Parameter | Type | Description | Remarks | | :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- | -| `td.connect.ip` | string | Used in establishing a connection; same as `taos_connect` | | -| `td.connect.user` | string | Used in establishing a connection; same as `taos_connect` | | -| `td.connect.pass` | string | Used in establishing a connection; same as `taos_connect` | | -| `td.connect.port` | string | Used in establishing a connection; same as `taos_connect` | | +| `td.connect.ip` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | +| `td.connect.user` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | +| `td.connect.pass` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | +| `td.connect.port` | string | Used in establishing a connection; same as `taos_connect` | Only valid for establishing native connection | | `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. | | `client.id` | string | Client ID | Maximum length: 192. | | `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | -| `enable.auto.commit` | boolean | Commit automatically | Specify `true` or `false`. | +| `enable.auto.commit` | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself | Default value is true | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | -| `enable.heartbeat.background` | boolean | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | | -| `experimental.snapshot.enable` | boolean | Specify whether to consume messages from TSDB | | -| `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | +| `experimental.snapshot.enable` | boolean | Specify whether to consume data in TSDB; true: both data in WAL and in TSDB can be consumed; false: only data in WAL can be consumed | default value: false | +| `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | default value: false The method of specifying these parameters depends on the language used: @@ -369,7 +368,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index a1c46875aa59b7ce6eb099d4c13a20a1a68e560b..ec007d68302f719ec09b1b13f58a1a6bddb2748c 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -35,8 +35,8 @@ database_option: { | TABLE_SUFFIX value | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value - | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value + | WAL_ROLL_PERIOD value | WAL_SEGMENT_SIZE value } ``` @@ -75,11 +75,10 @@ database_option: { - TABLE_PREFIX：The prefix length in the table name that is ignored when distributing table to vnode based on table name. - TABLE_SUFFIX：The suffix length in the table name that is ignored when distributing table to vnode based on table name. - TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. -- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the time of WAL files to keep has no upper limit. -- WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept after consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that WAL files are not required to keep after consumption. -1: the total size of WAL files to keep has no upper limit. +- WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep for consumption. Alter it with a proper value at first to create topics. +- WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that the total size of WAL files to keep for consumption has no upper limit. - WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. - WAL_SEGMENT_SIZE: specifies the maximum size of a WAL file. After the current WAL file reaches this size, a new WAL file is created. The default value is 0. A value of 0 indicates that a new WAL file is created only after TSDB data in memory are flushed to disk. - ### Example Statement ```sql @@ -123,6 +122,8 @@ alter_database_option: { | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value + | WAL_RETENTION_PERIOD value + | WAL_RETENTION_SIZE value } ``` @@ -179,6 +180,14 @@ TRIM DATABASE db_name; The preceding SQL statement deletes data that has expired and orders the remaining data in accordance with the storage configuration. +## Flush Data + +```sql +FLUSH DATABASE db_name; +``` + +Flush data from memory onto disk. Before shutting down a node, executing this command can avoid data restore after restarting and speed up the startup process. + ## Redistribute Vgroup ```sql diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index 37257286372b403721032e48d56cc1f1c3559538..a1d103eaf04a30214a8c1ef88e43a949710bcdbc 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -13,12 +13,11 @@ create_definition: col_name column_definition column_definition: - type_name [COMMENT 'string_value'] + type_name ``` **More explanations** - Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. -- When you create a supertable, you can add comments to columns and tags. - The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: - A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column. - The name of a tag column cannot be the same as the name of any other column. diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 75462c3d96c7f5e7368a80a2d21441449cf67fc5..de7294f7a9dbc78b854dffa25ffd0e7edd7e7b66 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -248,13 +248,13 @@ You can also use the NULLS keyword to specify the position of null values. Ascen The LIMIT keyword controls the number of results that are displayed. You can also use the OFFSET keyword to specify the result to display first. `LIMIT` and `OFFSET` are executed after `ORDER BY` in the query execution. You can include an offset in a LIMIT clause. For example, LIMIT 5 OFFSET 2 can also be written LIMIT 2, 5. Both of these clauses display the third through the seventh results. -In a statement that includes a PARTITION BY clause, the LIMIT keyword is performed on each partition, not on the entire set of results. +In a statement that includes a PARTITION BY/GROUP BY clause, the LIMIT keyword is performed on each partition/group, not on the entire set of results. ## SLIMIT -The SLIMIT keyword is used with a PARTITION BY clause to control the number of partitions that are displayed. You can include an offset in a SLIMIT clause. For example, SLIMIT 5 OFFSET 2 can also be written LIMIT 2, 5. Both of these clauses display the third through the seventh partitions. +The SLIMIT keyword is used with a PARTITION BY/GROUP BY clause to control the number of partitions/groups that are displayed. You can include an offset in a SLIMIT clause. For example, SLIMIT 5 OFFSET 2 can also be written LIMIT 2, 5. Both of these clauses display the third through the seventh partitions/groups. -Note: If you include an ORDER BY clause, only one partition can be displayed. +Note: If you include an ORDER BY clause, only one partition/group can be displayed. ## Special Query diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 8dfa9c2851c16ac16d43559555e492991b44dd15..5c1a833e05c927e91363a7405339750204761ce0 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **Description**: UNIX timestamp converted from a string of date/time format -**Return value type**: BIGINT +**Return value type**: BIGINT, TIMESTAMP **Applicable column types**: VARCHAR and NCHAR @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - The input string must be compatible with ISO8601/RFC3339 standard, NULL will be returned if the string can't be converted - The precision of the returned timestamp is same as the precision set for the current data base in use +- return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT. ### Time and Date Functions diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 1edae160268d39b48b8df7b371ab3762d87cb005..01d7065905359dd312346f54293c5cf8d8f04afb 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -75,6 +75,16 @@ These pseudocolumns occur after the aggregation clause. 5. LINEAR：Fill with the closest non-NULL value, `FILL(LINEAR)` 6. NEXT：Fill with the next non-NULL value, `FILL(NEXT)` +In the above filling modes, except for `NONE` mode, the `fill` clause will be ignored if there is no data in the defined time range, i.e. no data would be filled and the query result would be empty. This behavior is reasonable when the filling mode is `PREV`, `NEXT`, `LINEAR`, because filling can't be performed if there is not any data. For filling modes `NULL` and `VALUE`, however, filling can be performed even though there is not any data, filling or not depends on the choice of user's application. To accomplish the need of this force filling behavior and not break the behavior of existing filling modes, TDengine added two new filling modes since version 3.0.3.0. + +1. NULL_F: Fill `NULL` by force +2. VALUE_F: Fill `VALUE` by force + +The detailed beaviors of `NULL`, `NULL_F`, `VALUE`, and VALUE_F are described below： +- When used with `INTERVAL`: `NULL_F` and `VALUE_F` are filling by force；`NULL` and `VALUE` don't fill by force. The behavior of each filling mode is exactly same as what the name suggests. +- When used with `INTERVAL` in stream processing: `NULL_F` and `NULL` are same, i.e. don't fill by force; `VALUE_F` and `VALUE` and same, i.e. don't fill by force. It's suggested that there is no filling by force in stream processing. +- When used with `INTERP`: `NULL` and `NULL_F` and same, i.e. filling by force; `VALUE` and `VALUE_F` are same, i.e. filling by force. It's suggested that there is always filling by force when used with `INTERP`. + :::info 1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000. @@ -161,7 +171,7 @@ If you want to perform event window based query on the result set of a sub-query For example, the diagram below illustrates the event windows generated by the query below: ```sql -select _wstart, _wend, count(*) from t start with c1 > 0 end with c2 < 10 +select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10 ``` ![Event Window Illustration](./event_window.webp) diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md index fc369ec663cf9130bd7d1af650f1f47a1054fad6..43ff8e3091309e4f55ca1e7b59d1edca09424e3d 100644 --- a/docs/en/12-taos-sql/23-perf.md +++ b/docs/en/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE | 1 | consumer_id | BIGINT | Consumer ID | | 2 | consumer_group | BINARY(192) | Consumer group | | 3 | client_id | BINARY(192) | Client ID (user-defined) | -| 4 | status | BINARY(20) | Consumer status | +| 4 | status | BINARY(20) | Consumer status. All possible status include: ready(consumer is in normal state), lost(the connection between consumer and mnode is broken), rebalance(the redistribution of vgroups that belongs to current consumer is now in progress), unknown(consumer is in invalid state) | 5 | topics | BINARY(204) | Subscribed topic. Returns one row for each topic. | | 6 | up_time | TIMESTAMP | Time of first connection to TDengine Server | | 7 | subscribe_time | TIMESTAMP | Time of first subscription | diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 65aab4c47b3487fc75518a97b7bef6299a5a83d4..e2aff7a878893eda03c998d963840ffa046e6405 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -308,9 +308,11 @@ Query OK, 24 row(s) in set (0.002444s) - The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: - - `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. - - `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. +The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: +- `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. +- `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. + +Note that only the information about the data blocks in the data file will be displayed here, and the information about the data in the stt file will not be displayed. ## SHOW TAGS diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 341791d6755aa66fddd98561823ee0e5ac74ba3f..f4606f263fbda42e8e221d067912b884a3be4a40 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ The following data types can be used in the schema for standard tables. | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." | 2 | ALTER ALL DNODES | Added | Modifies the configuration of all dnodes. -| 3 | ALTER DATABASE | Modified | Deprecated +| 3 | ALTER DATABASE | Modified | Deprecated | 4 | ALTER STABLE | Modified | Deprecated | 5 | ALTER TABLE | Modified | Deprecated | 6 | ALTER USER | Modified | Deprecated diff --git a/docs/en/13-operation/01-pkg-install.md b/docs/en/13-operation/01-pkg-install.md index 6e6c4aaebf7da8ed633cbb89233f9028f56d8302..561013947100a0c938154c37fa9e11b17fd428b6 100644 --- a/docs/en/13-operation/01-pkg-install.md +++ b/docs/en/13-operation/01-pkg-install.md @@ -15,14 +15,14 @@ About details of installing TDenine, please refer to [Installation Guide](../../ ## Uninstall - + -Apt-get package of TDengine can be uninstalled as below: +Uninstall package of TDengine by apt-get can be uninstalled as below: ```bash $ sudo apt-get remove tdengine Reading package lists... Done -Building dependency tree +Building dependency tree Reading state information... Done The following packages will be REMOVED: tdengine @@ -35,7 +35,7 @@ TDengine is removed successfully! ``` -Apt-get package of taosTools can be uninstalled as below: +If you have installed taos-tools, please uninstall it first before uninstall TDengine. The command of uninstall is following: ``` $ sudo apt remove taostools @@ -111,8 +111,20 @@ taos tools is uninstalled successfully! ``` + Run C:\TDengine\unins000.exe to uninstall TDengine on a Windows system. + + + + +TDengine can be uninstalled as below: + +``` +$ rmtaos +TDengine is removed successfully! +``` + @@ -150,13 +162,13 @@ There are two aspects in upgrade operation: upgrade installation package and upg To upgrade a package, follow the steps mentioned previously to first uninstall the old version then install the new version. -Upgrading a running server is much more complex. First please check the version number of the old version and the new version. The version number of TDengine consists of 4 sections, only if the first 3 sections match can the old version be upgraded to the new version. The steps of upgrading a running server are as below: +Upgrading a running server is much more complex. First please check the version number of the old version and the new version. The version number of TDengine consists of 4 sections, only if the first 2 sections match can the old version be upgraded to the new version. The steps of upgrading a running server are as below: - Stop inserting data -- Make sure all data is persisted to disk +- Make sure all data is persisted to disk, please use command `flush database` - Stop the cluster of TDengine - Uninstall old version and install new version - Start the cluster of TDengine -- Execute simple queries, such as the ones executed prior to installing the new package, to make sure there is no data loss +- Execute simple queries, such as the ones executed prior to installing the new package, to make sure there is no data loss - Run some simple data insertion statements to make sure the cluster works well - Restore business services diff --git a/docs/en/13-operation/03-tolerance.md b/docs/en/13-operation/03-tolerance.md index 4f33748e68a0b51be619fe7aa8cc3b2152c71bf2..c8d2c3f3f62f3eeba950de335ef636d08d19f180 100644 --- a/docs/en/13-operation/03-tolerance.md +++ b/docs/en/13-operation/03-tolerance.md @@ -18,14 +18,8 @@ To achieve absolutely no data loss, set wal_level to 2 and wal_fsync_period to 0 ## Disaster Recovery -TDengine uses replication to provide high availability. +TDengine provides disaster recovery by using taosX to replicate data between two TDengine clusters which are deployed in two distant data centers. Assume there are two TDengine clusters, A and B, A is the source and B is the target, and A takes the workload of writing and querying. You can deploy `taosX` in the data center where cluster A resides in, `taosX` consumes the data written into cluster A and writes into cluster B. If the data center of cluster A is disrupted because of disaster, you can switch to cluster B to take the workload of data writing and querying, and deploy a `taosX` in the data center of cluster B to replicate data from cluster B to cluster A if cluster A has been recovered, or another cluster C if cluster A has not been recovered. -A TDengine cluster is managed by mnodes. You can configure up to three mnodes to ensure high availability. The data replication between mnode replicas is performed in a synchronous way to guarantee metadata consistency. +You can use the data replication feature of `taosX` to build more complicated disaster recovery solution. -The number of replicas for time series data in TDengine is associated with each database. There can be many databases in a cluster and each database can be configured with a different number of replicas. When creating a database, the parameter `replica` is used to specify the number of replicas. To achieve high availability, set `replica` to 3. - -The number of dnodes in a TDengine cluster must NOT be lower than the number of replicas for any database, otherwise it would fail when trying to create a table. - -As long as the dnodes of a TDengine cluster are deployed on different physical machines and the replica number is higher than 1, high availability can be achieved without any other assistance. For disaster recovery, dnodes of a TDengine cluster should be deployed in geographically different data centers. - -Alternatively, you can use taosX to synchronize the data from one TDengine cluster to another cluster in a remote location. However, taosX is only available in TDengine enterprise version, for more information please contact tdengine.com. +taosX is only provided in TDengine enterprise edition, for more details please contact business@tdengine.com. diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 346b874059a11c5608027b4ea6e2550765d64186..19107240bff5942ebb28e9a8353bc227ccb2b00e 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -42,3 +42,304 @@ An existing Grafana Notification Channel can be specified with parameter `-E`, t Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`. For more use cases and restrictions please refer to [TDinsight](/reference/tdinsight/). + +## log database + +The data of tdinsight dashboard is stored in `log` database (default. You can change it in taoskeeper's config file. For more infrmation, please reference to [taoskeeper document](/reference/taosKeeper)). The taoskeeper will create log database on taoskeeper startup. + +### cluster\_info table + +`cluster_info` table contains cluster information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||first ep of cluster| +|first\_ep\_dnode\_id|INT||dnode id or first\_ep| +|version|VARCHAR||tdengine version. such as: 3.0.4.0| +|master\_uptime|FLOAT||days of master's uptime| +|monitor\_interval|INT||monitor interval in second| +|dbs\_total|INT||total number of databases in cluster| +|tbs\_total|BIGINT||total number of tables in cluster| +|stbs\_total|INT||total number of stables in cluster| +|dnodes\_total|INT||total number of dnodes in cluster| +|dnodes\_alive|INT||total number of dnodes in ready state| +|mnodes\_total|INT||total number of mnodes in cluster| +|mnodes\_alive|INT||total number of mnodes in ready state| +|vgroups\_total|INT||total number of vgroups in cluster| +|vgroups\_alive|INT||total number of vgroups in ready state| +|vnodes\_total|INT||total number of vnode in cluster| +|vnodes\_alive|INT||total number of vnode in ready state| +|connections\_total|INT||total number of connections to cluster| +|topics\_total|INT||total number of topics in cluster| +|streams\_total|INT||total number of streams in cluster| +|protocol|INT||protocol version| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info table + +`d_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode status| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info table + +`m_info` table contains mnode information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||the role of mnode. leader or follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info table + +`dnodes_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||cpu usage of tdengine. read from `/proc//stat`| +|cpu\_system|FLOAT||cpu usage of server. read from `/proc/stat`| +|cpu\_cores|FLOAT||cpu cores of server| +|mem\_engine|INT||memory usage of tdengine. read from `/proc//status`| +|mem\_system|INT||available memory on the server| +|mem\_total|INT||total memory of server in `KB`| +|disk\_engine|INT||| +|disk\_used|BIGINT||usage of data dir in `bytes`| +|disk\_total|BIGINT||the capacity of data dir in `bytes`| +|net\_in|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|net\_out|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|io\_read|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_write|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_read\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|io\_write\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|req\_select|INT||number of select queries received per dnode| +|req\_select\_rate|FLOAT||number of select queries received per dnode divided by monitor interval.| +|req\_insert|INT||number of insert queries received per dnode| +|req\_insert\_success|INT||number of successfully insert queries received per dnode| +|req\_insert\_rate|FLOAT||number of insert queries received per dnode divided by monitor interval| +|req\_insert\_batch|INT||number of batch insertions| +|req\_insert\_batch\_success|INT||number of successful batch insertions| +|req\_insert\_batch\_rate|FLOAT||number of batch insertions divided by monitor interval| +|errors|INT||dnode errors| +|vnodes\_num|INT||number of vnodes per dnode| +|masters|INT||number of master vnodes| +|has\_mnode|INT||if the dnode has mnode| +|has\_qnode|INT||if the dnode has qnode| +|has\_snode|INT||if the dnode has snode| +|has\_bnode|INT||if the dnode has bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir table + +`data_dir` table contains data directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data directory. default is `/var/lib/taos`| +|level|INT||level for multi-level storage| +|avail|BIGINT||available space for data directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir table + +`log_dir` table contains log directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log directory. default is `/var/log/taos/`| +|avail|BIGINT||available space for log directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir table + +`temp_dir` table contains temp dir information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp directory. default is `/tmp/`| +|avail|BIGINT||available space for temp directory| +|used|BIGINT||used space for temp directory| +|total|BIGINT||total space for temp directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info table + +`vgroups_info` table contains vgroups information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||database for the vgroup| +|tables\_num|BIGINT||number of tables per vgroup| +|status|VARCHAR||status| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role table + +`vnodes_role` table contains vnode role information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||role. leader or follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs table + +`logs` table contains login information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary table + +`log_summary` table contains log summary information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error count| +|info|INT||info count| +|debug|INT||debug count| +|trace|INT||trace count| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info table + +`grants_info` table contains grants information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||time until grants expire in seconds| +|timeseries\_used|BIGINT||timeseries used| +|timeseries\_total|BIGINT||total timeseries| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor table + +`keeper_monitor` table contains keeper monitor information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu usage| +|mem|FLOAT||memory usage| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total table + +`taosadapter_restful_http_request_total` table contains taosadapter rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail table + +`taosadapter_restful_http_request_fail` table contains taosadapter failed rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight table + +`taosadapter_restful_http_request_in_flight` table contains taosadapter rest request information record in real time. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds table + +`taosadapter_restful_http_request_summary_milliseconds` table contains the summary or rest information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent table + +`taosadapter_system_mem_percent` table contains taosadapter memory usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent table + +`taosadapter_system_cpu_percent` table contains taosadapter cup usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||mertic value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 1691b8be8b53480199898ab4ec6b5917eba7979f..409e079b92d5c8375ce6b3da60c0b2aa053ef662 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -382,6 +382,130 @@ Response body: } ``` +## REST API between TDengine 2.x and 3.0 + +### URI + +| URI | TDengine 2.x | TDengine 3.0 | +| :--------------------| :------------------: | :--------------------------------------------------: | +| /rest/sql | Supported | Supported (with different response code and body) | +| /rest/sqlt | Supported | No more supported | +| /rest/sqlutc | Supported | No more supported | + +### HTTP code + +| HTTP code | TDengine 2.x | TDengine 3.0 | note | +| :--------------------| :------------------: | :----------: | :-----------------------------------: | +| 200 | Supported | Supported | Success or taosc return error | +| 400 | Not supported | Supported | Parameter error | +| 401 | Not supported | Supported | Authentication failure | +| 404 | Supported | Supported | URI not exist | +| 500 | Not supported | Supported | Internal error | +| 503 | Supported | Supported | Insufficient system resources | + +### Response body + +#### REST response body return from TDengine 2.x + +```JSON +{ + "status": "succ", + "head": [ + "name", + "created_time", + "ntables", + "vgroups", + "replica", + "quorum", + "days", + "keep1,keep2,keep(D)", + "cache(MB)", + "blocks", + "minrows", + "maxrows", + "wallevel", + "fsync", + "comp", + "precision", + "status" + ], + "data": [ + [ + "log", + "2020-09-02 17:23:00.039", + 4, + 1, + 1, + 1, + 10, + "30,30,30", + 1, + 3, + 100, + 4096, + 1, + 3000, + 2, + "us", + "ready" + ] + ], + "rows": 1 +} +``` +``` + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], +``` + +#### REST response body return from TDengine 3.0 + +```JSON +{ + "code": 0, + "column_meta": [ + [ + "name", + "VARCHAR", + 64 + ], + [ + "ntables", + "BIGINT", + 8 + ], + [ + "status", + "VARCHAR", + 10 + ] + ], + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], + "rows": 2 +} +``` + ## Reference [taosAdapter](/reference/taosadapter/) diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index e7a1303aafc641981fe5166df37d9f2f888b33e7..fd4b4641d75e2116267b3f7f13a995079913f663 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -725,7 +725,7 @@ consumer.close() For more information, see [Data Subscription](../../../develop/tmq). -### Usage examples +#### Full Sample Code diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index da2f54708f550bc4ba9f35da81ad59a575bda169..0088f230069b15f12e455b5b6d2d6fa261fc5a14 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -120,7 +120,7 @@ _taosSql_ implements Go's `database/sql/driver` interface via cgo. You can use t Use `taosSql` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName`, DSN supports the following parameters. -* configPath specifies the `taos.cfg` directory +* cfg specifies the `taos.cfg` directory For example: diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index c885d416c8303ecb95a55b42125de31aa5dcd65a..cc5c8f4e69e208c98c773e94c6aa2ca1ace334e7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -10,10 +10,11 @@ import TabItem from "@theme/TabItem"; `taospy` is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. `taospy` wraps both the [native interface](/reference/connector/cpp) and [REST interface](/reference/rest-api) of TDengine, which correspond to the `taos` and `taosrest` modules of the `taospy` package, respectively. In addition to wrapping the native and REST interfaces, `taospy` also provides a set of programming interfaces that conforms to the [Python Data Access Specification (PEP 249)](https://peps.python.org/pep-0249/). It is easy to integrate `taospy` with many third-party tools, such as [SQLAlchemy](https://www.sqlalchemy.org/) and [pandas](https://pandas.pydata.org/). -The direct connection to the server using the native interface provided by the client driver is referred to hereinafter as a "native connection"; the connection to the server using the REST interface provided by taosAdapter is referred to hereinafter as a "REST connection". +`taos-ws-py` is an optional package to enable using WebSocket to connect TDengine. -The source code for the Python connector is hosted on [GitHub](https://github.com/taosdata/taos-connector-python). +The direct connection to the server using the native interface provided by the client driver is referred to hereinafter as a "native connection"; the connection to the server using the REST or WebSocket interface provided by taosAdapter is referred to hereinafter as a "REST connection" or "WebSocket connection". +The source code for the Python connector is hosted on [GitHub](https://github.com/taosdata/taos-connector-python). ## Supported platforms - The [supported platforms](/reference/connector/#supported-platforms) for the native connection are the same as the ones supported by the TDengine client. @@ -114,6 +115,15 @@ For REST connections, verifying that the `taosrest` module can be imported succe import taosrest ``` + + + +For WebSocket connection, verifying that the `taosws` module can be imported successfully can be done in the Python Interactive Shell by typing. + +```python +import taosws +``` + @@ -182,6 +192,28 @@ If the test is successful, it will output the server version information, e.g. } ``` + + + +For WebSocket connection, make sure the cluster and taosAdapter component, are running. This can be testetd using the following `curl` command. + +``` +curl -i -N -d "show databases" -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -H "Connection: Upgrade" -H "Upgrade: websocket" -H "Host: :" -H "Origin: http://:" http://:/rest/sql +``` + +The FQDN above is the FQDN of the machine running taosAdapter, PORT is the port taosAdapter listening, default is `6041`. + +If the test is successful, it will output the server version information, e.g. + +```json +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Tue, 21 Mar 2023 09:29:17 GMT +Transfer-Encoding: chunked + +{"status":"succ","head":["server_version()"],"column_meta":[["server_version()",8,8]],"data":[["2.6.0.27"]],"rows":1} +``` + @@ -321,6 +353,86 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +### Usage with req_id + +By using the optional req_id parameter, you can specify a request ID that can be used for tracing. + + + + +##### TaosConnection class + +The `TaosConnection` class contains both an implementation of the PEP249 Connection interface (e.g., the `cursor()` method and the `close()` method) and many extensions (e.g., the `execute()`, `query()`, `schemaless_insert()`, and `subscribe()` methods). + +```python title="execute method" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} +``` + +```python title="query method" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} +``` + +:::tip +The queried results can only be fetched once. For example, only one of `fetch_all()` and `fetch_all_into_dict()` can be used in the example above. Repeated fetches will result in an empty list. +::: + +##### Use of TaosResult class + +In the above example of using the `TaosConnection` class, we have shown two ways to get the result of a query: `fetch_all()` and `fetch_all_into_dict()`. In addition, `TaosResult` also provides methods to iterate through the result set by rows (`rows_iter`) or by data blocks (`blocks_iter`). Using these two methods will be more efficient in scenarios where the query has a large amount of data. + +```python title="blocks_iter method" +{{#include docs/examples/python/result_set_with_req_id_examples.py}} +``` +##### Use of the TaosCursor class + +The `TaosConnection` class and the `TaosResult` class already implement all the functionality of the native interface. If you are familiar with the interfaces in the PEP249 specification, you can also use the methods provided by the `TaosCursor` class. + +```python title="Use of TaosCursor" +{{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} +``` + +:::note +The TaosCursor class uses native connections for write and query operations. In a client-side multi-threaded scenario, this cursor instance must remain thread exclusive and cannot be shared across threads for use, otherwise, it will result in errors in the returned results. + +::: + + + + +##### Use of TaosRestCursor class + +The `TaosRestCursor` class is an implementation of the PEP249 Cursor interface. + +```python title="Use of TaosRestCursor" +{{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} +``` +- `cursor.execute`: Used to execute arbitrary SQL statements. +- `cursor.rowcount` : For write operations, returns the number of successful rows written. For query operations, returns the number of rows in the result set. +- `cursor.description` : Returns the description of the field. Please refer to [TaosRestCursor](https://docs.taosdata.com/api/taospy/taosrest/cursor.html) for the specific format of the description information. + +##### Use of the RestClient class + +The `RestClient` class is a direct wrapper for the [REST API](/reference/rest-api). It contains only a `sql()` method for executing arbitrary SQL statements and returning the result. + +```python title="Use of RestClient" +{{#include docs/examples/python/rest_client_with_req_id_example.py}} +``` + +For a more detailed description of the `sql()` method, please refer to [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html). + + + + +```python +{{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} +``` + +- `conn.execute`: can use to execute arbitrary SQL statements, and return the number of rows affected. +- `conn.query`: can use to execute query SQL statements, and return the query results. + + + + ### Used with pandas @@ -347,6 +459,56 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +### Schemaless Insert + +Connector support schemaless insert. + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### Other sample programs | Example program links | Example program content | diff --git a/docs/en/14-reference/03-connector/_category_.yml b/docs/en/14-reference/03-connector/_category_.yml index e470f64aa013b137f05f03db112641faf2956297..6a766e96574844db87a70ff7eb5f0005cb6acbfb 100644 --- a/docs/en/14-reference/03-connector/_category_.yml +++ b/docs/en/14-reference/03-connector/_category_.yml @@ -1 +1 @@ -label: "connector" \ No newline at end of file +label: "Connector" diff --git a/docs/en/14-reference/03-connector/_linux_install.mdx b/docs/en/14-reference/03-connector/_linux_install.mdx index 97f6cd333961d24bd015a2ebcf7223f89fecca2c..398593cfe6b23cbb3269182a68fb7563e2964484 100644 --- a/docs/en/14-reference/03-connector/_linux_install.mdx +++ b/docs/en/14-reference/03-connector/_linux_install.mdx @@ -14,7 +14,7 @@ import PkgListV3 from "/components/PkgListV3"; Once the package is unzipped, you will see the following files in the directory: - _ install_client.sh_: install script - - _ taos.tar.gz_: client driver package + - _ package.tar.gz_: client driver package - _ driver_: TDengine client driver - _examples_: some example programs of different programming languages (C/C#/go/JDBC/MATLAB/python/R) You can run `install_client.sh` to install it. diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 7ab894a1c72651e3568f3effa351b044c397b2dd..6bc49768c692a47ea2796fb006d9ee942ef818ba 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ Command-line arguments take precedence over environment variables over configura ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` Note: @@ -332,6 +329,10 @@ This parameter controls the number of results returned by the following interfac taosAdapter uses the parameter `httpCodeServerError` to set whether to return a non-200 http status code http status code other than when the C interface returns an error. When set to true, different http status codes will be returned according to the error code returned by C. For details, see [RESTful API](https://docs.tdengine.com/reference/rest-api/) HTTP Response Code chapter. +## Configure whether schemaless writes automatically create DBs + +Starting from version 3.0.4.0, the taosAdapter provides the parameter "smlAutoCreateDB" to control whether to automatically create DBs when writing with the schemaless protocol. The default value is false, which means that the DB will not be automatically created and the user needs to manually create the DB before performing schemaless writing. + ## Troubleshooting You can check the taosAdapter running status with the `systemctl status taosadapter` command. diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index bfb311d72e180990870eb40726aeb6c1f996200f..f49424d3ba817be1d338ab78fb5738f859a5a235 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -403,11 +403,11 @@ See [General Configuration Parameters](#General Configuration Parameters) for de #### Configuration parameters for executing the specified query statement -The configuration parameters for querying the sub-tables or the normal tables are set in `specified_table_query`. +The configuration parameters for querying the specified table (it can be a super table, a sub-table or a normal table) are set in `specified_table_query`. - **query_interval** : The query interval in seconds, the default value is 0. -- **threads**: The number of threads to execute the query SQL, the default value is 1. +- **threads/concurrent**: The number of threads to execute the query SQL, the default value is 1. - **sqls**. - **sql**: the SQL command to be executed. @@ -434,9 +434,9 @@ The configuration parameters of the super table query are set in `super_table_qu #### Configuration parameters for executing the specified subscription statement -The configuration parameters for subscribing to a sub-table or a generic table are set in `specified_table_query`. +The configuration parameters for subscribing to a specified table (it can be a super table, a sub-table or a generic table) are set in `specified_table_query`. -- **threads**: The number of threads to execute SQL, default is 1. +- **threads/concurrent**: The number of threads to execute SQL, default is 1. - **interval**: The time interval to execute the subscription, in seconds, default is 0. diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 7833ac861fea5fb2dfd66ccc9ec2db8f0ddeff3f..e66da7ec1768227d5ec50ce039bd8f794cf90c48 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -61,12 +61,14 @@ And many more parameters. - -c CONFIGDIR: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` - -C: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c - -d DATABASE: Specify the database to use when connecting to the server +- -E dsn: connect to the TDengine Cloud or a server which provides WebSocket connection - -f FILE: Execute the SQL script file in non-interactive mode Note that each SQL statement in the script file must be only one line. - -k: Test the operational status of the server. 0: unavailable; 1: network ok; 2: service ok; 3: service degraded; 4: exiting - -l PKTLEN: Test package size to be used for network testing - -n NETROLE: test scope for network connection test, default is `client`. The value can be `client` or `server`. - -N PKTNUM: Number of packets used for network testing - -r: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) +- -R: Use RESTful mode when connecting - -s COMMAND: execute SQL commands in non-interactive mode - -t: Test the boot status of the server. The statuses of -k apply. - -w DISPLAYWIDTH: Specify the number of columns of the server display. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index a76074f5078f1273cea06d1404230b8ad389455e..2e6f5ec1e27b7cbba035ad745e4d1f6ec3633f22 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -99,6 +99,9 @@ The parameters described in this document by the effect that they have on the sy ## Monitoring Parameters +:::note +Please note the `taoskeeper` needs to be installed and running to create the `log` database and receiving metrics sent by `taosd` as the full monitoring solution. + ### monitor | Attribute | Description | @@ -625,6 +628,16 @@ The charset that takes effect is UTF-8. | Default Value | 1 | | Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +### enableScience + +| Attribute | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Applicable | Only taos-CLI client | +| Meaning | Whether to show float and double with the scientific notation | +| Value Range | 0: false, 1: true | +| Default Value | 0 | + + ### udf | Attribute | Description | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 3f75364081d7ec242d96a30f3adf0861637a06eb..aad0e63a4228ca303302d4a3970182355f750d53 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -3,13 +3,11 @@ title: Schemaless Writing description: This document describes how to use the schemaless write component of TDengine. --- -In IoT applications, data is collected for many purposes such as intelligent control, business analysis, device monitoring and so on. Due to changes in business or functional requirements or changes in device hardware, the application logic and even the data collected may change. Schemaless writing automatically creates storage structures for your data as it is being written to TDengine, so that you do not need to create supertables in advance. When necessary, schemaless writing -will automatically add the required columns to ensure that the data written by the user is stored correctly. +In IoT applications, data is collected for many purposes such as intelligent control, business analysis, device monitoring and so on. Due to changes in business or functional requirements or changes in device hardware, the application logic and even the data collected may change. Schemaless writing automatically creates storage structures for your data as it is being written to TDengine, so that you do not need to create supertables in advance. When necessary, schemaless writing will automatically add the required columns to ensure that the data written by the user is stored correctly. The schemaless writing method creates super tables and their corresponding subtables. These are completely indistinguishable from the super tables and subtables created directly via SQL. You can write data directly to them via SQL statements. Note that the names of tables created by schemaless writing are based on fixed mapping rules for tag values, so they are not explicitly ideographic and they lack readability. -Tips: -The schemaless write will automatically create a table. You do not need to create a table manually, or an unknown error may occur. +Note: Schemaless writing creates tables automatically. Creating tables manually is not supported with schemaless writing. ## Schemaless Writing Line Protocol @@ -50,8 +48,7 @@ In the schemaless writing data line protocol, each data item in the field_set ne - `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types. -For example, the following data rows write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column -as "passit" (BINARY), c4 column as 4 (DOUBLE), and the primary key timestamp as 1626006833639000000 to child table with the t1 label as "3" (NCHAR), the t2 label as "4" (NCHAR), and the t3 label as "t3" (NCHAR) and the super table named `st`. +For example, the following string indicates that the one row of data is written to the st supertable with the t1 tag as "3" (NCHAR), the t2 tag as "4" (NCHAR), and the t3 tag as "t3" (NCHAR); the c1 column is 3 (BIGINT), the c2 column is false (BOOL), the c3 column is "passit" (BINARY), the c4 column is 4 (DOUBLE), and the primary key timestamp is 1626006833639000000. ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -69,23 +66,31 @@ Schemaless writes process row data according to the following principles. "measurement,tag_key1=tag_value1,tag_key2=tag_value2" ``` +:::tip Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. -The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has. +The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. +::: + You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. + **Important:** Manually creating supertables for schemaless writing is not supported. Schemaless writing creates appropriate supertables automatically. + 3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2. + 4. If the specified tag or regular column in the data row does not exist, the corresponding tag or regular column is added to the super table (only incremental). -5. If there are some tag columns or regular columns in the super table that are not specified to take values in a data row, then the values of these columns are set to - NULL. + +5. If there are some tag columns or regular columns in the super table that are not specified to take values in a data row, then the values of these columns are set to NULL. + 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. + 7. Errors encountered throughout the processing will interrupt the writing process and return an error code. -8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, discarded since 3.0.3.0) -:::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed -48KB, and the total length of tag value cannot exceed 16KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur. + Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used. +:::tip +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition @@ -114,8 +119,7 @@ In OpenTSDB file and JSON protocol modes, the precision of the timestamp is dete ## Data Model Mapping -This section describes how data in line protocol is mapped to a schema. The data measurement in each line is mapped to a -supertable name. The tag name in tag_set is the tag name in the schema, and the name in field_set is the column name in the schema. The following example shows how data is mapped: +This section describes how data in InfluxDB line protocol is mapped to a schema. The data measurement in each line is mapped to a supertable name. The tag name in tag_set is the tag name in the schema, and the name in field_set is the column name in the schema. The following example shows how data is mapped: ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -160,7 +164,7 @@ The preceding data includes a new entry, c6, with type binary(6). When this occu TDengine guarantees the idempotency of data writes. This means that you can repeatedly call the API to perform write operations with bad data. However, TDengine does not guarantee the atomicity of multi-row writes. In a multi-row write, some data may be written successfully and other data unsuccessfully. -##: Error Codes +## Error Codes The TSDB_CODE_TSC_LINE_SYNTAX_ERROR indicates an error in the schemaless writing component. This error occurs when writing text. For other errors, schemaless writing uses the standard TDengine error codes diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 16c591154705521fc194a13ddcf6d2bbb5552470..cfc25543873f56f5511abcf3b6c20d556b3da4fc 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -4,23 +4,24 @@ title: taosKeeper description: This document describes how to use taosKeeper, a tool for exporting TDengine monitoring metrics. --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + ## Introduction taosKeeper is a tool for TDengine that exports monitoring metrics. With taosKeeper, you can easily monitor the operational status of your TDengine deployment. taosKeeper uses the TDengine REST API. It is not necessary to install TDengine Client to use taosKeeper. ## Installation - +There are two ways to install taosKeeper: Methods of installing taosKeeper: - - -- You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. --> -You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. +- Installing the official TDengine installer will automatically install taosKeeper. Please refer to [TDengine installation](/operation/pkg-install) for details. -## Run +- You can compile taosKeeper separately and install it. Please refer to the [taosKeeper](https://github.com/taosdata/taoskeeper) repository for details. +## Configuration and Launch -### Configuration and running methods +### Configuration taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: [Command-line arguments](#command-line-arguments-in-detail), [environment variable](#environment-variable-in-detail) and [configuration file](#configuration-file-parameters-in-detail). The precedence of those is Command-line, environment variable and configuration file. @@ -33,28 +34,81 @@ monitorFqdn localhost # taoskeeper's FQDN For more information, see [TDengine Monitoring Configuration](../config/#monitoring). -### Command-Line Parameters +### Quick Launch -You can use command-line parameters to run taosKeeper and control its behavior: + + -```shell -$ taosKeeper +After the installation is complete, run the following command to start the taoskeeper service: + +```bash +systemctl start taoskeeper ``` -### Environment variable -You can use Environment variable to run taosKeeper and control its behavior: +Run the following command to confirm that taoskeeper is running normally: -```shell -$ export TAOS_KEEPER_TDENGINE_HOST=192.168.64.3 - -$ taoskeeper +```bash +systemctl status taoskeeper +``` + +Output similar to the following indicates that taoskeeper is running normally: + +``` +Active: active (running) +``` + +Output similar to the following indicates that taoskeeper has not started successfully: + +``` +Active: inactive (dead) ``` -you can run `taoskeeper -h` for more detail. +The following `systemctl` commands can help you manage taoskeeper service: + +- Start taoskeeper Server: `systemctl start taoskeeper` + +- Stop taoskeeper Server: `systemctl stop taoskeeper` + +- Restart taoskeeper Server: `systemctl restart taoskeeper` + +- Check taoskeeper Server status: `systemctl status taoskeeper` + +:::info + +- The `systemctl` command requires _root_ privileges. If you are not logged in as the _root_ user, use the `sudo` command. +- The `systemctl stop taoskeeper` command will instantly stop taoskeeper Server. +- If your system does not include `systemd`, you can run `/usr/local/taos/bin/taoskeeper` to start taoskeeper manually. + +::: + + + + +After the installation is complete, run `launchctl start com.tdengine.taoskeeper` to start taoskeeper Server. + +The following `launchctl` commands can help you manage taoskeeper service: + +- Start taoskeeper Server: `sudo launchctl start com.tdengine.taoskeeper` + +- Stop taoskeeper Server: `sudo launchctl stop com.tdengine.taoskeeper` + +- Check taoskeeper Server status: `sudo launchctl list | grep taoskeeper` -### Configuration File +:::info +- Please use `sudo` to run `launchctl` to manage _com.tdengine.taoskeeper_ with administrator privileges. +- The administrator privilege is required for service management to enhance security. +- Troubleshooting: +- The first column returned by the command `launchctl list | grep taoskeeper` is the PID of the program. If it's `-`, that means the taoskeeper service is not running. +- If the service is abnormal, please check the `launchd.log` file from the system log. -You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. +::: + + + + +#### Launch With Configuration File + +You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/taoskeeper.toml` is used by default. If this file does not specify configurations, the default values are used. ```shell $ taoskeeper -c @@ -99,6 +153,10 @@ database = "log" # standard tables to monitor tables = ["normal_table"] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### Obtain Monitoring Metrics @@ -132,19 +190,62 @@ $ curl http://127.0.0.1:6043/metrics Sample result set (excerpt): ```shell -# HELP taos_cluster_info_connections_total +# HELP taos_cluster_info_connections_total # TYPE taos_cluster_info_connections_total counter taos_cluster_info_connections_total{cluster_id="5981392874047724755"} 16 -# HELP taos_cluster_info_dbs_total +# HELP taos_cluster_info_dbs_total # TYPE taos_cluster_info_dbs_total counter taos_cluster_info_dbs_total{cluster_id="5981392874047724755"} 2 -# HELP taos_cluster_info_dnodes_alive +# HELP taos_cluster_info_dnodes_alive # TYPE taos_cluster_info_dnodes_alive counter taos_cluster_info_dnodes_alive{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_dnodes_total +# HELP taos_cluster_info_dnodes_total # TYPE taos_cluster_info_dnodes_total counter taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_first_ep +# HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 -``` \ No newline at end of file +``` + +### check\_health + +``` +$ curl -i http://127.0.0.1:6043/check_health +``` + +Response: + +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Mon, 03 Apr 2023 07:20:38 GMT +Content-Length: 19 + +{"version":"1.0.0"} +``` + +### taoskeeper with Prometheus + +There is `/metrics` api in taoskeeper provide TDengine metric data for Prometheus. + +#### scrape config + +Scrape config in Prometheus specifies a set of targets and parameters describing how to scrape metric data from endpoint. For more information, please reference to [Prometheus documents](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +There is a dashboard named `TaosKeeper Prometheus Dashboard for 3.x`, which provides a monitoring dashboard similar to TInsight. + +In Grafana, click the Dashboard menu and click `import`, enter the dashboard ID `18587` and click the `Load` button. Then finished importing `TaosKeeper Prometheus Dashboard for 3.x` dashboard. + diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index 5a2942b14497401ef1a2cb8f31ad2773f6b828e3..b0e9dea707fb2fb326329895921850cd6d721b6d 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource You can also download zip files from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and install manually. The commands are as follows: ```bash -GF_VERSION=3.2.7 +GF_VERSION=3.3.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 088c4ba2dcd5f2e6cd4fce789d9d24db8ab006e8..acfbf6a0baa63d830aed57e7f08c370ea04722c0 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,22 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + +## 3.0.3.2 + + + +## 3.0.3.1 + + + +## 3.0.3.1 + + + ## 3.0.3.0 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 7d45110bb06fcc504f6393b6d6132d5541f98b2c..e8f7a54566128260937cce3a31a5b82f22f1d1b6 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,18 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.12 + + + +## 2.4.11 + + + +## 2.4.10 + + + ## 2.4.9 diff --git a/docs/examples/c/tmq_example.c b/docs/examples/c/tmq_example.c index a3bade308a4ce41f556878d90861e299893315c4..d958428b8ffec9064166ddfd502cc0e2dbd9a200 100644 --- a/docs/examples/c/tmq_example.c +++ b/docs/examples/c/tmq_example.c @@ -70,7 +70,7 @@ static int32_t init_env() { taos_free_result(pRes); // create database - pRes = taos_query(pConn, "create database tmqdb"); + pRes = taos_query(pConn, "create database tmqdb wal_retention_period 3600"); if (taos_errno(pRes) != 0) { printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/docs/examples/csharp/influxdbLine/Program.cs b/docs/examples/csharp/influxdbLine/Program.cs index fa3cb21fe04977b5081c922d623dee5514056770..a620c016090c9177966f8361eac0e65d23192bd8 100644 --- a/docs/examples/csharp/influxdbLine/Program.cs +++ b/docs/examples/csharp/influxdbLine/Program.cs @@ -48,7 +48,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/optsJSON/Program.cs b/docs/examples/csharp/optsJSON/Program.cs index b67b5af62bf0a1fd9028125da0b665f723f2e4ec..8dcc1dce92be2f134ce68910c916896c07fc4003 100644 --- a/docs/examples/csharp/optsJSON/Program.cs +++ b/docs/examples/csharp/optsJSON/Program.cs @@ -54,7 +54,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/optsTelnet/Program.cs b/docs/examples/csharp/optsTelnet/Program.cs index e73ceb041accf88222176342d46fe1a669584211..ccd29d0cfc62ac1a59bea8465630595c2e65b5c0 100644 --- a/docs/examples/csharp/optsTelnet/Program.cs +++ b/docs/examples/csharp/optsTelnet/Program.cs @@ -58,7 +58,7 @@ namespace TDengineExample static void PrepareDatabase(IntPtr conn) { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE test"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600"); if (TDengine.ErrorNo(res) != 0) { throw new Exception("failed to create database, reason: " + TDengine.Error(res)); diff --git a/docs/examples/csharp/sqlInsert/Program.cs b/docs/examples/csharp/sqlInsert/Program.cs index f23a6e1663023d1d2fafb3e92e0b605f8ac55e52..25a945a45952d596153707bb4ea8d6c6e327b098 100644 --- a/docs/examples/csharp/sqlInsert/Program.cs +++ b/docs/examples/csharp/sqlInsert/Program.cs @@ -11,7 +11,7 @@ namespace TDengineExample IntPtr conn = GetConnection(); try { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); CheckRes(conn, res, "failed to create database"); res = TDengine.Query(conn, "USE power"); CheckRes(conn, res, "failed to change database"); diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 80cadb2ff8b596a0484d05ff15aeaa50f22ff859..2e856a49bbbbae5424a517363e3ae1360ba6a0c7 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -76,7 +76,7 @@ namespace TDengineExample static void PrepareSTable() { - IntPtr res = TDengine.Query(conn, "CREATE DATABASE power"); + IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600"); CheckResPtr(res, "failed to create database"); res = TDengine.Query(conn, "USE power"); CheckResPtr(res, "failed to change database"); diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 1f7218936fbe457615562ded1b938daca95225cb..cb24e351ab8cc5265cefbed9565d9a404aadf08d 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -15,7 +15,7 @@ func main() { panic(err) } defer db.Close() - _, err = db.Exec("create database if not exists example_tmq") + _, err = db.Exec("create database if not exists example_tmq wal_retention_period 3600") if err != nil { panic(err) } @@ -35,7 +35,6 @@ func main() { "td.connect.port": "6030", "client.id": "test_tmq_client", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", }) diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index e9af5e9ce0c0473f4513cbb949dcbd9f433c0c92..8da6f77bae432dee987d4567229c2fbebd657be8 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -35,7 +35,7 @@ public class SubscribeDemo { try (Statement statement = connection.createStatement()) { statement.executeUpdate("drop topic if exists " + TOPIC); statement.executeUpdate("drop database if exists " + DB_NAME); - statement.executeUpdate("create database " + DB_NAME); + statement.executeUpdate("create database " + DB_NAME + " wal_retention_period 3600"); statement.executeUpdate("use " + DB_NAME); statement.executeUpdate( "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java index d953a7364160a686d67b4390f3370999b02ce5d4..83cb04f55244aacf860573a42ddc952e1e522803 100644 --- a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -35,7 +35,7 @@ public class WebsocketSubscribeDemo { Statement statement = connection.createStatement()) { statement.executeUpdate("drop topic if exists " + TOPIC); statement.executeUpdate("drop database if exists " + DB_NAME); - statement.executeUpdate("create database " + DB_NAME); + statement.executeUpdate("create database " + DB_NAME + " wal_retention_period 3600"); statement.executeUpdate("use " + DB_NAME); statement.executeUpdate( "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); diff --git a/docs/examples/python/conn_websocket_pandas.py b/docs/examples/python/conn_websocket_pandas.py index eac386732c6ec6e512c6f6723f47f1f09751d9f9..5cad5384b251b7016f895d4d0e8792e42bfc395f 100644 --- a/docs/examples/python/conn_websocket_pandas.py +++ b/docs/examples/python/conn_websocket_pandas.py @@ -4,7 +4,7 @@ import taos taos_conn = taos.connect() taos_conn.execute('drop database if exists power') -taos_conn.execute('create database if not exists power') +taos_conn.execute('create database if not exists power wal_retention_period 3600') taos_conn.execute("use power") taos_conn.execute( "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") diff --git a/docs/examples/python/connect_rest_with_req_id_examples.py b/docs/examples/python/connect_rest_with_req_id_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..3feb574fa60d5701aa040b606846cb6cf7a28aaa --- /dev/null +++ b/docs/examples/python/connect_rest_with_req_id_examples.py @@ -0,0 +1,44 @@ +# ANCHOR: connect +from taosrest import connect, TaosRestConnection, TaosRestCursor + +conn = connect(url="http://localhost:6041", + user="root", + password="taosdata", + timeout=30) + +# ANCHOR_END: connect +# ANCHOR: basic +# create STable +cursor = conn.cursor() +cursor.execute("DROP DATABASE IF EXISTS power", req_id=1) +cursor.execute("CREATE DATABASE power", req_id=2) +cursor.execute( + "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3) + +# insert data +cursor.execute("""INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)""", req_id=4) +print("inserted row count:", cursor.rowcount) + +# query data +cursor.execute("SELECT * FROM power.meters LIMIT 3", req_id=5) +# get total rows +print("queried row count:", cursor.rowcount) +# get column names from cursor +column_names = [meta[0] for meta in cursor.description] +# get rows +data = cursor.fetchall() +print(column_names) +for row in data: + print(row) + +# output: +# inserted row count: 8 +# queried row count: 3 +# ['ts', 'current', 'voltage', 'phase', 'location', 'groupid'] +# [datetime.datetime(2018, 10, 3, 14, 38, 5, 500000, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 11.8, 221, 0.28, 'california.losangeles', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 16, 600000, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 13.4, 223, 0.29, 'california.losangeles', 2] +# [datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.8, 223, 0.29, 'california.losangeles', 3] +# ANCHOR_END: basic diff --git a/docs/examples/python/connect_websocket_examples.py b/docs/examples/python/connect_websocket_examples.py index c50976efbf595e63b91f16faff0af7eb37e90756..29452bbf9d32b2e630575ef4f51fd2d10b59495a 100644 --- a/docs/examples/python/connect_websocket_examples.py +++ b/docs/examples/python/connect_websocket_examples.py @@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041") # ANCHOR: basic conn.execute("drop database if exists connwspy") -conn.execute("create database if not exists connwspy") +conn.execute("create database if not exists connwspy wal_retention_period 3600") conn.execute("use connwspy") conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)") conn.execute("create table if not exists tb1 using stb tags (1)") diff --git a/docs/examples/python/connect_websocket_with_req_id_examples.py b/docs/examples/python/connect_websocket_with_req_id_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..f5f76c8446649e8b9b77b754d45eca0cb5d5a210 --- /dev/null +++ b/docs/examples/python/connect_websocket_with_req_id_examples.py @@ -0,0 +1,29 @@ +# ANCHOR: connect +import taosws + +conn = taosws.connect("taosws://root:taosdata@localhost:6041") +# ANCHOR_END: connect + +# ANCHOR: basic +conn.execute("drop database if exists connwspy", req_id=1) +conn.execute("create database if not exists connwspy", req_id=2) +conn.execute("use connwspy", req_id=3) +conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4) +conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5) +conn.execute("insert into tb1 values (now, 1)", req_id=6) +conn.execute("insert into tb1 values (now, 2)", req_id=7) +conn.execute("insert into tb1 values (now, 3)", req_id=8) + +r = conn.execute("select * from stb", req_id=9) +result = conn.query("select * from stb", req_id=10) +num_of_fields = result.field_count +print(num_of_fields) + +for row in result: + print(row) + +# output: +# 3 +# ('2023-02-28 15:56:13.329 +08:00', 1, 1) +# ('2023-02-28 15:56:13.333 +08:00', 2, 1) +# ('2023-02-28 15:56:13.337 +08:00', 3, 1) diff --git a/docs/examples/python/connection_usage_native_reference_with_req_id.py b/docs/examples/python/connection_usage_native_reference_with_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..24d0914ad54eb9545ab56468230201f3e2bd3484 --- /dev/null +++ b/docs/examples/python/connection_usage_native_reference_with_req_id.py @@ -0,0 +1,45 @@ +import taos + +# ANCHOR: insert +conn = taos.connect() +# Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement. +conn.execute("DROP DATABASE IF EXISTS test", req_id=1) +conn.execute("CREATE DATABASE test", req_id=2) +# change database. same as execute "USE db" +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) +affected_row = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)", req_id=4) +print("affected_row", affected_row) +# output: +# affected_row 3 +# ANCHOR_END: insert + +# ANCHOR: query +# Execute a sql and get its result set. It's useful for SELECT statement +result = conn.query("SELECT * from weather", req_id=5) + +# Get fields from result +fields = result.fields +for field in fields: + print(field) # {name: ts, type: 9, bytes: 8} + +# output: +# {name: ts, type: 9, bytes: 8} +# {name: temperature, type: 6, bytes: 4} +# {name: location, type: 4, bytes: 4} + +# Get data from result as list of tuple +data = result.fetch_all() +print(data) +# output: +# [(datetime.datetime(2022, 4, 27, 9, 4, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 5, 25, 367000), 23.5, 1), (datetime.datetime(2022, 4, 27, 9, 6, 25, 367000), 24.399999618530273, 1)] + +# Or get data from result as a list of dict +# map_data = result.fetch_all_into_dict() +# print(map_data) +# output: +# [{'ts': datetime.datetime(2022, 4, 27, 9, 1, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 2, 15, 343000), 'temperature': 23.5, 'location': 1}, {'ts': datetime.datetime(2022, 4, 27, 9, 3, 15, 343000), 'temperature': 24.399999618530273, 'location': 1}] +# ANCHOR_END: query + + +conn.close() \ No newline at end of file diff --git a/docs/examples/python/cursor_usage_native_reference_with_req_id.py b/docs/examples/python/cursor_usage_native_reference_with_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..15207ee6bc97fdb00918b98439fff785770f1334 --- /dev/null +++ b/docs/examples/python/cursor_usage_native_reference_with_req_id.py @@ -0,0 +1,32 @@ +import taos + +conn = taos.connect() +cursor = conn.cursor() + +cursor.execute("DROP DATABASE IF EXISTS test", req_id=1) +cursor.execute("CREATE DATABASE test", req_id=2) +cursor.execute("USE test", req_id=3) +cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4) + +for i in range(1000): + location = str(i % 10) + tb = "t" + location + cursor.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)", req_id=5+i) + +cursor.execute("SELECT count(*) FROM weather", req_id=1005) +data = cursor.fetchall() +print("count:", data[0][0]) +cursor.execute("SELECT tbname, * FROM weather LIMIT 2", req_id=1006) +col_names = [meta[0] for meta in cursor.description] +print(col_names) +rows = cursor.fetchall() +print(rows) + +cursor.close() +conn.close() + +# output: +# count: 2000 +# ['tbname', 'ts', 'temperature', 'location'] +# row_count: -1 +# [('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 392000), 23.5, 0), ('t0', datetime.datetime(2022, 4, 27, 14, 54, 24, 393000), 23.5, 0)] diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py index 566748c94e2542aabe8265ed55c85e4b725d69bb..1c735abfc080f63f5bc4126a7f44474dbf5d157c 100644 --- a/docs/examples/python/kafka_example_common.py +++ b/docs/examples/python/kafka_example_common.py @@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', 'California.SantaClara', 'California.Cupertino'] -CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1' +CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600' USE_DATABASE_SQL = 'use {}' DROP_TABLE_SQL = 'drop table if exists meters' DROP_DATABASE_SQL = 'drop database if exists {}' diff --git a/docs/examples/python/rest_client_with_req_id_example.py b/docs/examples/python/rest_client_with_req_id_example.py new file mode 100644 index 0000000000000000000000000000000000000000..918398e51eb8c5e33c6fed3b7e05afdbd1c66878 --- /dev/null +++ b/docs/examples/python/rest_client_with_req_id_example.py @@ -0,0 +1,9 @@ +from taosrest import RestClient + +client = RestClient("http://localhost:6041", user="root", password="taosdata") +res: dict = client.sql("SELECT ts, current FROM power.meters LIMIT 1", req_id=1) +print(res) + +# output: +# {'status': 'succ', 'head': ['ts', 'current'], 'column_meta': [['ts', 9, 8], ['current', 6, 4]], 'data': [[datetime.datetime(2018, 10, 3, 14, 38, 5, tzinfo=datetime.timezone(datetime.timedelta(seconds=28800), '+08:00')), 10.3]], 'rows': 1} + diff --git a/docs/examples/python/result_set_with_req_id_examples.py b/docs/examples/python/result_set_with_req_id_examples.py new file mode 100644 index 0000000000000000000000000000000000000000..90ae2f4f2671bf00669adc69270dc943a02c16aa --- /dev/null +++ b/docs/examples/python/result_set_with_req_id_examples.py @@ -0,0 +1,33 @@ +import taos + +conn = taos.connect() +conn.execute("DROP DATABASE IF EXISTS test", req_id=1) +conn.execute("CREATE DATABASE test", req_id=2) +conn.select_db("test") +conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3) +# prepare data +for i in range(2000): + location = str(i % 10) + tb = "t" + location + conn.execute(f"INSERT INTO {tb} USING weather TAGS({location}) VALUES (now+{i}a, 23.5) (now+{i + 1}a, 23.5)", req_id=4+i) + +result: taos.TaosResult = conn.query("SELECT * FROM weather", req_id=2004) + +block_index = 0 +blocks: taos.TaosBlocks = result.blocks_iter() +for rows, length in blocks: + print("block ", block_index, " length", length) + print("first row in this block:", rows[0]) + block_index += 1 + +conn.close() + +# possible output: +# block 0 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 46000), 23.5, 0) +# block 1 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 76000), 23.5, 3) +# block 2 length 1200 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 99000), 23.5, 6) +# block 3 length 400 +# first row in this block: (datetime.datetime(2022, 4, 27, 15, 14, 52, 122000), 23.5, 9) diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py new file mode 100644 index 0000000000000000000000000000000000000000..334a4b728f5fb3914bdd346083ba76607ea78ce2 --- /dev/null +++ b/docs/examples/python/schemaless_insert.py @@ -0,0 +1,21 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py new file mode 100644 index 0000000000000000000000000000000000000000..0fda7dc505e8f1ff95912282e85938eef8d8963c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw.py @@ -0,0 +1,74 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + res = conn.schemaless_insert_raw(lines, 1, 0) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except SchemalessError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..606e51098665a611d54379368fb3c099f9b6e33b --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -0,0 +1,76 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..cf57792534cce73317dbdac3a9915adfa73a389c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -0,0 +1,73 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..ee1472db6939474ccf34c591fe4dcf4891f0d2ab --- /dev/null +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=1) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=2) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..85050439f2a451b9d12f7fe20ebbae5297adb5a4 --- /dev/null +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 6f7fb87c89ce4cb96793d09a837f60ad54ae69bc..5b462fa1536d34a11824eed09310f1068735a7dc 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -6,7 +6,7 @@ def init_tmq_env(db, topic): conn = taos.connect() conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {}".format(db)) + conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 1fab5185296d9d2e4addc5cd6cb51bdff969919b..bdf6a730d0b4b456ef1e689966935af173871394 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -178,7 +178,7 @@ Active: inactive (dead) ::: -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 @@ -186,9 +186,9 @@ Active: inactive (dead) -安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 +安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 @@ -215,7 +215,7 @@ Active: inactive (dead) ::: -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index e144c563b97304f6257d3a1989d7caf85d3789aa..16172277b5accef4e0e5c707d1bb7f09ace23446 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -4,7 +4,7 @@ description: '快速设置 TDengine 环境并体验其高效写入和查询' --- import xiaot from './xiaot.webp' -import xiaot_new from './xiaot-new.webp' +import xiaot_new from './xiaot-03.webp' import channel from './channel.webp' import official_account from './official-account.webp' @@ -19,17 +19,6 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; ``` -## 学习 TDengine 知识地图 - -TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概念实体之间的调用关系和数据流向。学习和了解 TDengine 知识地图有助于你快速掌握 TDengine 的知识体系。 - -
-
- -
图 1. TDengine 知识地图
-
-
- ## 加入 TDengine 官方社区 微信扫描以下二维码，学习了解 TDengine 的最新技术，与大家共同交流物联网大数据技术应用、TDengine 使用问题和技巧等话题。 @@ -41,7 +30,7 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点，揭示了各概 TDengine 微信公众号 -加入“物联网大数据技术群”
与大家进行技术交流 +加入 TDengine 微信群
了解学习最新物联网技术 关注 TDengine 视频号
收看技术直播与教学视频 关注 TDengine 公众号
阅读技术文章与行业案例 diff --git a/docs/zh/05-get-started/xiaot-03.webp b/docs/zh/05-get-started/xiaot-03.webp new file mode 100644 index 0000000000000000000000000000000000000000..c115346e9e84ffb990aec5947b2107b5daf6e19b Binary files /dev/null and b/docs/zh/05-get-started/xiaot-03.webp differ diff --git a/docs/zh/07-develop/06-stream.md b/docs/zh/07-develop/06-stream.md index a2e1d1a1dac5af44e053aa314ede45005e2de587..4a5ebb96e8add2a8756b3e21d29f4ba40d5b927f 100644 --- a/docs/zh/07-develop/06-stream.md +++ b/docs/zh/07-develop/06-stream.md @@ -52,7 +52,7 @@ CREATE TABLE d1004 USING meters TAGS ("California.LosAngeles", 3); ### 创建流 ```sql -create stream current_stream into current_stream_output_stb as select _wstart as start, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s); +create stream current_stream trigger at_once into current_stream_output_stb as select _wstart as wstart, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s); ``` ### 写入数据 @@ -70,8 +70,8 @@ insert into d1004 values("2018-10-03 14:38:06.500", 11.50000, 221, 0.35000); ### 查询以观察结果 ```sql -taos> select start, wend, max_current from current_stream_output_stb; - start | wend | max_current | +taos> select wstart, wend, max_current from current_stream_output_stb; + wstart | wend | max_current | =========================================================================== 2018-10-03 14:38:05.000 | 2018-10-03 14:38:10.000 | 10.30000 | 2018-10-03 14:38:15.000 | 2018-10-03 14:38:20.000 | 12.60000 | @@ -89,7 +89,7 @@ Query OK, 2 rows in database (0.018762s) ### 创建流 ```sql -create stream power_stream into power_stream_output_stb as select ts, concat_ws(".", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname; +create stream power_stream trigger at_once into power_stream_output_stb as select ts, concat_ws(".", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname; ``` ### 写入数据 diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 1d834cfff020ce782d4821cfc055c0cff08b4584..036c9aa86e5c91903a74bffd1f9c59a13a97c116 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -25,6 +25,7 @@ import CDemo from "./_sub_c.mdx"; 本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。 +注意：默认是从wal消费数据，如果wal被删除，消费到的数据会不全，此时可以将参数 experimental.snapshot.enable 设置为true，从tsdb获取全部数据，但是这样的话就不能保证数据的消费顺序。所以建议根据自己的消费情况合理的设置wal的保留策略，保证可以从wal里订阅到全部数据。 ## 主要数据结构和 API 不同语言下， TMQ 订阅相关的 API 及数据结构如下： @@ -283,18 +284,17 @@ CREATE TOPIC topic_name AS DATABASE db_name; | 参数名称 | 类型 | 参数说明 | 备注 | | :----------------------------: | :-----: | -------------------------------------------------------- | ------------------------------------------- | -| `td.connect.ip` | string | 用于创建连接，同 `taos_connect` | | -| `td.connect.user` | string | 用于创建连接，同 `taos_connect` | | -| `td.connect.pass` | string | 用于创建连接，同 `taos_connect` | | -| `td.connect.port` | integer | 用于创建连接，同 `taos_connect` | | +| `td.connect.ip` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | +| `td.connect.user` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | +| `td.connect.pass` | string | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | +| `td.connect.port` | integer | 用于创建连接，同 `taos_connect` | 仅用于建立原生连接 | | `group.id` | string | 消费组 ID，同一消费组共享消费进度 | **必填项**。最大长度：192。 | | `client.id` | string | 客户端 ID | 最大长度：192。 | | `auto.offset.reset` | enum | 消费组订阅的初始位置 |
`earliest`: default;从头开始订阅;
`latest`: 仅从最新数据开始订阅;
`none`: 没有提交的 offset 无法订阅 | -| `enable.auto.commit` | boolean | 是否启用消费位点自动提交 | 合法值：`true`, `false`。 | -| `auto.commit.interval.ms` | integer | 以毫秒为单位的消费记录自动提交消费位点时间间隔 | 默认 5000 m | -| `enable.heartbeat.background` | boolean | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 默认开启 | -| `experimental.snapshot.enable` | boolean | 是否允许从 TSDB 消费数据 | 实验功能，默认关闭 | -| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） | | +| `enable.auto.commit` | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit | 默认值为 true | +| `auto.commit.interval.ms` | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒 | 默认值为 5000 | +| `experimental.snapshot.enable` | boolean | 是否允许从 TSDB 消费数据。当其关闭时，只能消费依据 WAL 保留策略仍然在WAL中的数据；当其打开时，除WAL中的数据以外，也能够消费已经从WAL中删除但落盘到TSDB中的数据 | 实验功能，默认关闭 | +| `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） |默认关闭 | 对于不同编程语言，其设置方式如下： @@ -367,7 +367,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } @@ -419,7 +418,6 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) | `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms | | `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | | `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值：`true`, `false` | -| `enable.heartbeat.background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` | diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 2eb4b3dee32d7326476bbca350c951ccd0482d71..8a8ef82009c512b550cd514f3d52716f79d8e0a1 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -231,7 +231,7 @@ bit_add 实现多列的按位与功能。如果只有一列，返回这一列。 -### 聚合函数示例 [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) +### 聚合函数示例1 返回值为数值类型 [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 @@ -243,3 +243,29 @@ l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先 ``` + +### 聚合函数示例2 返回值为字符串类型 [max_vol](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/max_vol.c) + +max_vol 实现了从多个输入的电压列中找到最大电压，返回由设备ID + 最大电压所在（行，列）+ 最大电压值 组成的组合字符串值 + +创建表： +```bash +create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId varchar(16)); +``` +创建自定义函数： +```bash +create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; +``` +使用自定义函数： +```bash +select max_vol(vol1,vol2,vol3,deviceid) from battery; +``` + +
+max_vol.c + +```c +{{#include tests/script/sh/max_vol.c}} +``` + +
\ No newline at end of file diff --git a/docs/zh/08-connector/02-rest-api.mdx b/docs/zh/08-connector/02-rest-api.mdx index a081595bca5f66bff3c60808126516633f92614b..f3f1e087d809b7ce98d27721ada13b7a5fdf7159 100644 --- a/docs/zh/08-connector/02-rest-api.mdx +++ b/docs/zh/08-connector/02-rest-api.mdx @@ -383,6 +383,133 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata } ``` +## TDengine 2.x 和 3.0 之间 REST API 的差异 + +### URI + +| URI | TDengine 2.x | TDengine 3.0 | +| :--------------------| :------------------: | :--------------------------------------------------: | +| /rest/sql | 支持 | 支持 （响应代码和消息体不同） | +| /rest/sqlt | 支持 | 不再支持 | +| /rest/sqlutc | 支持 | 不再支持 | + + +### HTTP code + +| HTTP code | TDengine 2.x | TDengine 3.0 | 备注 | +| :--------------------| :------------------: | :----------: | :-----------------------------------: | +| 200 | 支持 | 支持 | 正确返回和 taosc 接口错误返回 | +| 400 | 不支持 | 支持 | 参数错误返回 | +| 401 | 不支持 | 支持 | 鉴权失败 | +| 404 | 支持 | 支持 | 接口不存在 | +| 500 | 不支持 | 支持 | 内部错误 | +| 503 | 支持 | 支持 | 系统资源不足 | + + +### 响应代码和消息体 + +#### TDengine 2.x 响应代码和消息体 + +```JSON +{ + "status": "succ", + "head": [ + "name", + "created_time", + "ntables", + "vgroups", + "replica", + "quorum", + "days", + "keep1,keep2,keep(D)", + "cache(MB)", + "blocks", + "minrows", + "maxrows", + "wallevel", + "fsync", + "comp", + "precision", + "status" + ], + "data": [ + [ + "log", + "2020-09-02 17:23:00.039", + 4, + 1, + 1, + 1, + 10, + "30,30,30", + 1, + 3, + 100, + 4096, + 1, + 3000, + 2, + "us", + "ready" + ] + ], + "rows": 1 +} +``` +``` + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], +``` + +#### TDengine 3.0 响应代码和消息体 + + +```JSON +{ + "code": 0, + "column_meta": [ + [ + "name", + "VARCHAR", + 64 + ], + [ + "ntables", + "BIGINT", + 8 + ], + [ + "status", + "VARCHAR", + 10 + ] + ], + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], + "rows": 2 +} +``` + ## 参考 [taosAdapter](/reference/taosadapter/) diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 061475f51e0cf9c25ba09d670e9ceef0bd2bab71..d1c12583659e2f6dd2a63357669b92c3b1ff29b1 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -17,7 +17,7 @@ import TabItem from '@theme/TabItem'; - JDBC 原生连接：Java 应用在物理节点 1（pnode1）上使用 TSDBDriver 直接调用客户端驱动（libtaos.so 或 taos.dll）的 API 将写入和查询请求发送到位于物理节点 2（pnode2）上的 taosd 实例。 - JDBC REST 连接：Java 应用通过 RestfulDriver 将 SQL 封装成一个 REST 请求，发送给物理节点 2 的 REST 服务器（taosAdapter），通过 REST 服务器请求 taosd 并返回结果。 -使用 REST 连接，不依赖 TDengine 客户端驱动，可以跨平台，更加方便灵活，但性能比原生连接器低约 30%。 +使用 REST 连接，不依赖 TDengine 客户端驱动，可以跨平台，更加方便灵活。 :::info TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致，但 TDengine 与关系对象型数据库的使用场景和技术特征存在差异，所以`taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点： @@ -728,7 +728,7 @@ consumer.close() 详情请参考：[数据订阅](../../../develop/tmq) -### 使用示例如下： +#### 完整示例 diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 2aa1a58e49f34b412f12bd0d67586dc6e56cf0bc..fd6df992b530fdce4427e34ef8bc227f449eaac7 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -122,7 +122,7 @@ _taosSql_ 通过 cgo 实现了 Go 的 `database/sql/driver` 接口。只需要 使用 `taosSql` 作为 `driverName` 并且使用一个正确的 [DSN](#DSN) 作为 `dataSourceName`，DSN 支持的参数： -* configPath 指定 taos.cfg 目录 +* cfg 指定 taos.cfg 目录 示例： diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index b57f02ac80c8d8cee8577b1a1d33b22317d7bdef..1cff142e11d39e6afe86fab187697d222f37a9dd 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -10,7 +10,9 @@ import TabItem from "@theme/TabItem"; `taospy` 是 TDengine 的官方 Python 连接器。`taospy` 提供了丰富的 API， 使得 Python 应用可以很方便地使用 TDengine。`taospy` 对 TDengine 的[原生接口](../cpp)和 [REST 接口](../rest-api)都进行了封装， 分别对应 `taospy` 包的 `taos` 模块 和 `taosrest` 模块。 除了对原生接口和 REST 接口的封装，`taospy` 还提供了符合 [Python 数据访问规范(PEP 249)](https://peps.python.org/pep-0249/) 的编程接口。这使得 `taospy` 和很多第三方工具集成变得简单，比如 [SQLAlchemy](https://www.sqlalchemy.org/) 和 [pandas](https://pandas.pydata.org/)。 -使用客户端驱动提供的原生接口直接与服务端建立的连接的方式下文中称为“原生连接”；使用 taosAdapter 提供的 REST 接口与服务端建立的连接的方式下文中称为“REST 连接”。 +`taos-ws-py` 是使用 WebSocket 方式连接 TDengine 的 Python 连接器包。可以选装。 + +使用客户端驱动提供的原生接口直接与服务端建立的连接的方式下文中称为“原生连接”；使用 taosAdapter 提供的 REST 接口或 WebSocket 接口与服务端建立的连接的方式下文中称为“REST 连接”或“WebSocket 连接”。 Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-python)。 @@ -115,6 +117,15 @@ import taos import taosrest ``` + + + +对于 WebSocket 连接，只需验证是否能成功导入 `taosws` 模块。可在 Python 交互式 Shell 中输入： + +```python +import taosws +``` + @@ -183,6 +194,27 @@ curl -u root:taosdata http://:/rest/sql -d "select server_version()" } ``` + + + +对于 WebSocket 连接， 除了确保集群已经启动，还要确保 taosAdapter 组件已经启动。可以使用如下 curl 命令测试： + +``` +curl -i -N -d "show databases" -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -H "Connection: Upgrade" -H "Upgrade: websocket" -H "Host: :" -H "Origin: http://:" http://:/rest/sql +``` + +上面的 FQDN 为运行 taosAdapter 的机器的 FQDN， PORT 为 taosAdapter 配置的监听端口， 默认为 6041。 +如果测试成功，会输出服务器版本信息，比如： + +```json +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Tue, 21 Mar 2023 09:29:17 GMT +Transfer-Encoding: chunked + +{"status":"succ","head":["server_version()"],"column_meta":[["server_version()",8,8]],"data":[["2.6.0.27"]],"rows":1} +``` + @@ -321,6 +353,85 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 与 req_id 一起使用 + +使用可选的 req_id 参数，指定请求 id，可以用于 tracing + + + + +##### TaosConnection 类的使用 + +`TaosConnection` 类既包含对 PEP249 Connection 接口的实现(如：`cursor`方法和 `close` 方法)，也包含很多扩展功能（如： `execute`、 `query`、`schemaless_insert` 和 `subscribe` 方法。 + +```python title="execute 方法" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:insert}} +``` + +```python title="query 方法" +{{#include docs/examples/python/connection_usage_native_reference_with_req_id.py:query}} +``` + +:::tip +查询结果只能获取一次。比如上面的示例中 `fetch_all()` 和 `fetch_all_into_dict()` 只能用一个。重复获取得到的结果为空列表。 +::: + +##### TaosResult 类的使用 + +上面 `TaosConnection` 类的使用示例中，我们已经展示了两种获取查询结果的方法： `fetch_all()` 和 `fetch_all_into_dict()`。除此之外 `TaosResult` 还提供了按行迭代(`rows_iter`)或按数据块迭代(`blocks_iter`)结果集的方法。在查询数据量较大的场景，使用这两个方法会更高效。 + +```python title="blocks_iter 方法" +{{#include docs/examples/python/result_set_with_req_id_examples.py}} +``` +##### TaosCursor 类的使用 + +`TaosConnection` 类和 `TaosResult` 类已经实现了原生接口的所有功能。如果你对 PEP249 规范中的接口比较熟悉也可以使用 `TaosCursor` 类提供的方法。 + +```python title="TaosCursor 的使用" +{{#include docs/examples/python/cursor_usage_native_reference_with_req_id.py}} +``` + +:::note +TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。 + +::: + + + + +##### TaosRestCursor 类的使用 + +`TaosRestCursor` 类是对 PEP249 Cursor 接口的实现。 + +```python title="TaosRestCursor 的使用" +{{#include docs/examples/python/connect_rest_with_req_id_examples.py:basic}} +``` +- `cursor.execute` ： 用来执行任意 SQL 语句。 +- `cursor.rowcount`： 对于写入操作返回写入成功记录数。对于查询操作，返回结果集行数。 +- `cursor.description` ： 返回字段的描述信息。关于描述信息的具体格式请参考[TaosRestCursor](https://docs.taosdata.com/api/taospy/taosrest/cursor.html)。 + +##### RestClient 类的使用 + +`RestClient` 类是对于 [REST API](../rest-api) 的直接封装。它只包含一个 `sql()` 方法用于执行任意 SQL 语句， 并返回执行结果。 + +```python title="RestClient 的使用" +{{#include docs/examples/python/rest_client_with_req_id_example.py}} +``` + +对于 `sql()` 方法更详细的介绍， 请参考 [RestClient](https://docs.taosdata.com/api/taospy/taosrest/restclient.html)。 + + + +```python +{{#include docs/examples/python/connect_websocket_with_req_id_examples.py:basic}} +``` + +- `conn.execute`: 用来执行任意 SQL 语句，返回影响的行数 +- `conn.query`: 用来执行查询 SQL 语句，返回查询结果 + + + + ### 与 pandas 一起使用 @@ -373,6 +484,56 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 无模式写入 + +连接器支持无模式写入功能。 + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### 其它示例程序 | 示例程序链接 | 示例程序内容 | diff --git a/docs/zh/08-connector/_linux_install.mdx b/docs/zh/08-connector/_linux_install.mdx index 0b1f415f54d28e65a519d35ab94a84b3aa8338ba..b16ab2ac5b0696ee8e4077706e80f6e4267faf1b 100644 --- a/docs/zh/08-connector/_linux_install.mdx +++ b/docs/zh/08-connector/_linux_install.mdx @@ -14,7 +14,7 @@ import PkgListV3 from "/components/PkgListV3"; 解压软件包之后，会在解压目录下看到以下文件(目录)： - _ install_client.sh_：安装脚本，用于应用驱动程序 - - _ taos.tar.gz_：应用驱动安装包 + - _ package.tar.gz_：应用驱动安装包 - _ driver_：TDengine 应用驱动 driver - _examples_: 各种编程语言的示例程序(c/C#/go/JDBC/MATLAB/python/R) 运行 install_client.sh 进行安装。 diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 9e346fdca8ca82a046b02f3d49a1b27ad8457b9d..dd306356605929e521ee4416c7694bc0731af957 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -35,7 +35,6 @@ database_option: { | TABLE_SUFFIX value | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value - | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value | WAL_SEGMENT_SIZE value } @@ -75,11 +74,10 @@ database_option: { - TABLE_PREFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 - TABLE_SUFFIX：内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 - TSDB_PAGESIZE：一个 VNODE 中时序数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB到 16 MB。 -- WAL_RETENTION_PERIOD：数据订阅已消费WAL日志，WAL文件的最大额外保留的时长策略。单位为 s。默认为 0，表示无需额外保留。-1, 表示额外保留，时间无上限。 -- WAL_RETENTION_SIZE：数据订阅已消费WAL日志，WAL文件的最大额外保留的累计大小策略。单位为 KB。默认为 0，表示无需额外保留。-1, 表示额外保留，累计大小无上限。 +- WAL_RETENTION_PERIOD: 为了数据订阅消费，需要WAL日志文件额外保留的最大时长策略。WAL日志清理，不受订阅客户端消费状态影响。单位为 s。默认为 0，表示无需为订阅保留。新建订阅，应先设置恰当的时长策略。 +- WAL_RETENTION_SIZE：为了数据订阅消费，需要WAL日志文件额外保留的最大累计大小策略。单位为 KB。默认为 0，表示累计大小无上限。 - WAL_ROLL_PERIOD：wal 文件切换时长，单位为 s。当WAL文件创建并写入后，经过该时间，会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 - WAL_SEGMENT_SIZE：wal 单个文件大小，单位为 KB。当前写入文件大小超过上限后会自动创建一个新的WAL文件。默认为 0，即仅在TSDB落盘时创建新文件。 - ### 创建数据库示例 ```sql @@ -179,6 +177,14 @@ TRIM DATABASE db_name; 删除过期数据，并根据多级存储的配置归整数据。 +## 落盘内存数据 + +```sql +FLUSH DATABASE db_name; +``` + +落盘内存中的数据。在关闭节点之前，执行这条命令可以避免重启后的数据回放，加速启动过程。 + ## 调整VGROUP中VNODE的分布 ```sql @@ -194,3 +200,11 @@ BALANCE VGROUP ``` 自动调整集群所有vgroup中的vnode分布，相当于在vnode级别对集群进行数据的负载均衡操作。 + +## 查看数据库工作状态 + +```sql +SHOW db_name.ALIVE; +``` + +查询数据库 db_name 的可用状态，返回值 0：不可用 1：完全可用 2：部分可用（即数据库包含的 VNODE 部分节点可用，部分节点不可用） diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index c5933228decfb3eb865cad058fa6fbba77fefb58..74ef52ee7c53d6c619a47d7e6453b56f152104a2 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -13,12 +13,11 @@ create_definition: col_name column_definition column_definition: - type_name [COMMENT 'string_value'] + type_name ``` **使用说明** - 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 -- 建表时可以给列或标签附加注释。 - TAGS语法指定超级表的标签列，标签列需要遵循以下约定： - TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。 - TAGS 列名不能与其他列名相同。 diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index dcc085b3bd714116455badbdc268c41b717b818f..e4ba5b76e487e2ac739d0797a32f2a6a04d849e3 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -248,11 +248,11 @@ NULLS 语法用来指定 NULL 值在排序中输出的位置。NULLS LAST 是升 LIMIT 控制输出条数，OFFSET 指定从第几条之后开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。LIMIT 5 OFFSET 2 可以简写为 LIMIT 2, 5，都输出第 3 行到第 7 行数据。 -在有 PARTITION BY 子句时，LIMIT 控制的是每个切分的分片中的输出，而不是总的结果集输出。 +在有 PARTITION BY/GROUP BY 子句时，LIMIT 控制的是每个切分的分片中的输出，而不是总的结果集输出。 ## SLIMIT -SLIMIT 和 PARTITION BY 子句一起使用，用来控制输出的分片的数量。SLIMIT 5 SOFFSET 2 可以简写为 SLIMIT 2, 5，都表示输出第 3 个到第 7 个分片。 +SLIMIT 和 PARTITION BY/GROUP BY 子句一起使用，用来控制输出的分片的数量。SLIMIT 5 SOFFSET 2 可以简写为 SLIMIT 2, 5，都表示输出第 3 个到第 7 个分片。 需要注意，如果有 ORDER BY 子句，则输出只有一个分片。 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 94f805205183aca6a7a08d9a836b0df33c7ed1f0..50e82e6b902d0beddbc205a7fde5867389c1cbc3 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 -**返回结果数据类型**：BIGINT。 +**返回结果数据类型**：BIGINT, TIMESTAMP。 **应用字段**：VARCHAR, NCHAR。 @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index 53689197efa1fa2b03ddc60868168d092064f014..f750124049c5b0dabecb359fe8c1b9d28ca1f431 100644 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -69,6 +69,16 @@ FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 +以上填充模式中，除了 NONE 模式默认不填充值之外，其他模式在查询的整个时间范围内如果没有数据 FILL 子句将被忽略，即不产生填充数据，查询结果为空。这种行为在部分模式（PREV、NEXT、LINEAR）下具有合理性，因为在这些模式下没有数据意味着无法产生填充数值。而对另外一些模式（NULL、VALUE）来说，理论上是可以产生填充数值的，至于需不需要输出填充数值，取决于应用的需求。所以为了满足这类需要强制填充数据或 NULL 的应用的需求，同时不破坏现有填充模式的行为兼容性，从 3.0.3.0 版本开始，增加了两种新的填充模式： + +7. NULL_F: 强制填充 NULL 值 +8. VALUE_F: 强制填充 VALUE 值 + +NULL, NULL_F, VALUE, VALUE_F 这几种填充模式针对不同场景区别如下： +- INTERVAL 子句： NULL_F, VALUE_F 为强制填充模式；NULL, VALUE 为非强制模式。在这种模式下下各自的语义与名称相符 +- 流计算中的 INTERVAL 子句：NULL_F 与 NULL 行为相同，均为非强制模式；VALUE_F 与 VALUE 行为相同，均为非强制模式。即流计算中的 INTERVAL 没有强制模式 +- INTERP 子句：NULL 与 NULL_F 行为相同，均为强制模式；VALUE 与 VALUE_F 行为相同，均为强制模式。即 INTERP 中没有非强制模式。 + :::info 1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 @@ -153,7 +163,7 @@ SELECT COUNT(*), FIRST(ts) FROM temp_tb_1 SESSION(ts, tol_val); 以下面的 SQL 语句为例，事件窗口切分如图所示： ```sql -select _wstart, _wend, count(*) from t start with c1 > 0 end with c2 < 10 +select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c2 < 10 ``` ![TDengine Database 事件窗口示意图](./event_window.webp) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 9e2fde526e1e63e3fbfb6124420caee3491cf965..9bbb551c8781f7a8fd17f9d91cd820b1e3863550 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -8,10 +8,13 @@ description: 流式计算的相关 SQL 的详细语法 ## 创建流式计算 ```sql -CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery +CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery stream_options: { - TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] - WATERMARK time + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time + IGNORE EXPIRED [0|1] + DELETE_MARK time + FILL_HISTORY [0|1] } ``` @@ -28,6 +31,15 @@ subquery: SELECT select_list 支持会话窗口、状态窗口与滑动窗口，其中，会话窗口与状态窗口搭配超级表时必须与partition by tbname一起使用 +stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表 + +TAGS 字句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG +```sql +create_definition: + col_name column_definition +column_definition: + type_name [COMMENT 'string_value'] +``` subtable 子句定义了流式计算中创建的子表的命名规则，详见 流式计算的 partition 部分。 @@ -114,7 +126,7 @@ SELECT * from information_schema.`ins_streams`; 在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 -对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式，默认为 AT_ONCE： +对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式，默认为 WINDOW_CLOSE： 1. AT_ONCE：写入立即触发 @@ -163,9 +175,67 @@ T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的 TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定： -1. 重新计算，即 IGNORE EXPIRED 0：默认配置，从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 +1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 -2. 直接丢弃, 即 IGNORE EXPIRED 1：忽略过期数据 +2. 直接丢弃, 即 IGNORE EXPIRED 1：默认配置，忽略过期数据 无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 + +## 写入已存在的超级表 +```sql +[field1_name,...] +``` +用来指定stb_name的列与subquery输出结果的对应关系。如果stb_name的列与subquery输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果stb_name的列与subquery输出结果的数据类型不匹配，会把subquery输出结果的类型转换成对应的stb_name的列的类型。 + +对于已经存在的超级表，检查列的schema信息 +1. 检查列的schema信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 +2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与subquery的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 +3. 至少自定义一个tag，否则报错。详见 自定义TAG + +## 自定义TAG + +用户可以为每个 partition 对应的子表生成自定义的TAG值。 +```sql +CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s)); +``` + +PARTITION 子句中，为 concat("tag-", tbname)定义了一个别名cc, 对应超级表st1的自定义TAG的名字。在上述示例中，流新创建的子表的TAG将以前缀 'new-' 连接原表名作为TAG的值。 + +会对TAG信息进行如下检查 +1.检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 +2.检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 + +## 清理中间状态 + +``` +DELETE_MARK time +``` +DELETE_MARK用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是10年 +T = 最新事件时间 - DELETE_MARK + +## 流式计算支持的函数 + +1. 所有的单行函数均可用于流计算中。 +2. 以下 19 个聚合函数不能在创建流计算的 SQL 语句中使用 +``` +leastsquares +percentile +top +bottom +elapsed +interp +derivative +irate +twa +histogram +diff +statecount +stateduration +csum +mavg +sample +tail +unique +mode +``` diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8fd704ef551c9ebda61678bd129917e67cbfbef8..d416febd55521853812fbe6952d403f4d265ac4e 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -269,7 +269,7 @@ description: TDengine 保留关键字的详细列表 - SPLIT - STABLE - STABLES -- STAR +- START - STATE - STATE_WINDOW - STATEMENT diff --git a/docs/zh/12-taos-sql/21-node.md b/docs/zh/12-taos-sql/21-node.md index 4116343730eee014047de99291c868b792153cd7..778d699099b3a564d57cbe0d0374dbdcb0fde0dc 100644 --- a/docs/zh/12-taos-sql/21-node.md +++ b/docs/zh/12-taos-sql/21-node.md @@ -119,6 +119,14 @@ DROP QNODE ON DNODE dnode_id; 删除 ID 为 dnode_id 的 DNODE 上的 QNODE，但并不会影响该 dnode 的状态。 +## 查询集群状态 + +```sql +SHOW CLUSTER ALIVE; +``` + +查询当前集群的状态是否可用，返回值： 0：不可用 1：完全可用 2：部分可用（集群中部分节点下线，但其它节点仍可以正常使用） + ## 修改客户端配置 如果将客户端也看作广义的集群的一部分，可以通过如下命令动态修改客户端配置参数。 diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index d4ee0e178c02e65eb3f1ceaa73e170893f65cc88..fc0b8072a7ea3722990b76bea2002bae1f12d7d3 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其 | 1 | consumer_id | BIGINT | 消费者的唯一 ID | | 2 | consumer_group | BINARY(192) | 消费者组 | | 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | -| 4 | status | BINARY(20) | 消费者当前状态 | +| 4 | status | BINARY(20) | 消费者当前状态。消费者状态包括：ready(正常可用)、 lost(连接已丢失)、 rebalancing(消费者所属 vgroup 正在分配中)、unknown(未知状态)| | 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | | 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | | 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index 5348233d5af20f0bb65360775e7c6497beb62f21..c85efa2376f3f3d78b2775a6e1408a307b212607 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -253,8 +253,9 @@ Query OK, 24 row(s) in set (0.002444s) - 上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。 +上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。 +需要注意，这里只会显示 data 文件中数据块的信息，stt 文件中的数据的信息不会被显示。 ## SHOW TAGS diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 9d67533cdea5c055c8d3ff58000fb80ac640e271..a797966f57a5d9c8c942de1cce3dd3209c067de6 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ description: "TDengine 3.0 版本的语法变更说明" | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 | 2 | ALTER ALL DNODES | 新增 | 修改所有DNODE的参数。 -| 3 | ALTER DATABASE | 调整 | 废除
  • QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。3.0.0版本STRICT暂不支持修改。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。
    新增
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
    调整
  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
+| 3 | ALTER DATABASE | 调整 | 废除
  • QUORUM：写入需要的副本确认数。3.0 版本默认行为是强一致性，且不支持修改为弱一致性。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。

    • 新增
  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
    调整
  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
| 4 | ALTER STABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改超级表的注释。
| 5 | ALTER TABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改表的注释。
  • TTL：修改表的生命周期。
| 6 | ALTER USER | 调整 | 废除
  • PRIVILEGE：修改用户权限。3.0版本使用GRANT和REVOKE来授予和回收权限。
    新增
  • ENABLE：启用或停用此用户。
  • SYSINFO：修改用户是否可查看系统信息。
diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index b8c5f9d647fcf920e454fabbf4da9c527b038a76..a10b5b55bc067518056b66346484f3e37bdcdc3c 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ taosAdapter 支持通过命令行参数、环境变量和配置文件来进行 ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` 备注： @@ -331,6 +328,10 @@ taosAdapter 通过参数 `restfulRowLimit` 来控制结果的返回条数，-1 taosAdapter 通过参数 `httpCodeServerError` 来设置当 C 接口返回错误时是否返回非 200 的 http 状态码。当设置为 true 时将根据 C 返回的错误码返回不同 http 状态码。具体见 [HTTP 响应码](../../connector/rest-api/#http-响应码)。 +## 配置 schemaless 写入是否自动创建 DB + +taosAdapter 从 3.0.4.0 版本开始，提供参数 `smlAutoCreateDB` 来控制在 schemaless 协议写入时是否自动创建 DB。默认值为 false 不自动创建 DB，需要用户手动创建 DB 后进行 schemaless 写入。 + ## 故障解决 您可以通过命令 `systemctl status taosadapter` 来检查 taosAdapter 运行状态。 diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index e2e51efec56fbf6ad312783c71bb9fe4f99043e6..c5d98767f9134c8eeb9bbc1ee2fb887370d9da54 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -400,11 +400,11 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) #### 执行指定查询语句的配置参数 -查询子表或者普通表的配置参数在 `specified_table_query` 中设置。 +查询指定表（可以指定超级表、子表或普通表）的配置参数在 `specified_table_query` 中设置。 - **query_interval** : 查询时间间隔，单位是秒，默认值为 0。 -- **threads** : 执行查询 SQL 的线程数，默认值为 1。 +- **threads/concurrent** : 执行查询 SQL 的线程数，默认值为 1。 - **sqls**： - **sql**: 执行的 SQL 命令，必填。 @@ -431,7 +431,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) #### 执行指定订阅语句的配置参数 -订阅子表或者普通表的配置参数在 `specified_table_query` 中设置。 +订阅指定表（可以指定超级表、子表或者普通表）的配置参数在 `specified_table_query` 中设置。 - **threads/concurrent** : 执行 SQL 的线程数，默认为 1。 diff --git a/docs/zh/14-reference/07-tdinsight/index.mdx b/docs/zh/14-reference/07-tdinsight/index.mdx index 8783d47f001249ea74a6f11a7f2709ae318152a0..b4eda61ebdb4f6ab9b8b650571a35196c76eda56 100644 --- a/docs/zh/14-reference/07-tdinsight/index.mdx +++ b/docs/zh/14-reference/07-tdinsight/index.mdx @@ -43,8 +43,6 @@ sudo apt-get update sudo apt-get install grafana ``` -### 在 CentOS / RHEL 上安装 Grafana - @@ -79,7 +77,37 @@ sudo yum install \ - +### 安装 TDengine 数据源插件 + + + + +从 GitHub 安装 TDengine 最新版数据源插件。 + +```bash +get_latest_release() { + curl --silent "https://api.github.com/repos/taosdata/grafanaplugin/releases/latest" | + grep '"tag_name":' | + sed -E 's/.*"v([^"]+)".*/\1/' +} +TDENGINE_PLUGIN_VERSION=$(get_latest_release) +sudo grafana-cli \ + --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v$TDENGINE_PLUGIN_VERSION/tdengine-datasource-$TDENGINE_PLUGIN_VERSION.zip \ + plugins install tdengine-datasource +``` + +:::note +3.1.6 和更早版本插件需要在配置文件 `/etc/grafana/grafana.ini` 中添加如下设置，以启用未签名插件。 + +```ini +[plugins] +allow_loading_unsigned_plugins = tdengine-datasource +``` + +::: + + + 我们提供了一个自动化安装脚本 [`TDinsight.sh`](https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh) 脚本以便用户快速进行安装配置。 @@ -175,33 +203,7 @@ sudo ./TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata - 特别地，当您使用 Grafana Cloud 或其他组织时，`-O` 可用于设置组织 ID。 `-G` 可指定 Grafana 插件安装目录。 `-e` 参数将仪表盘设置为可编辑。 - - -### 安装 TDengine 数据源插件 - -从 GitHub 安装 TDengine 最新版数据源插件。 - -```bash -get_latest_release() { - curl --silent "https://api.github.com/repos/taosdata/grafanaplugin/releases/latest" | - grep '"tag_name":' | - sed -E 's/.*"v([^"]+)".*/\1/' -} -TDENGINE_PLUGIN_VERSION=$(get_latest_release) -sudo grafana-cli \ - --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v$TDENGINE_PLUGIN_VERSION/tdengine-datasource-$TDENGINE_PLUGIN_VERSION.zip \ - plugins install tdengine-datasource -``` - -:::note -3.1.6 和更早版本插件需要在配置文件 `/etc/grafana/grafana.ini` 中添加如下设置，以启用未签名插件。 - -```ini -[plugins] -allow_loading_unsigned_plugins = tdengine-datasource -``` - -::: + ### 启动 Grafana 服务 @@ -233,8 +235,7 @@ sudo systemctl enable grafana-server ![TDengine Database TDinsight 数据源测试](./assets/howto-add-datasource-test.webp) - - + ### 导入仪表盘 diff --git a/docs/zh/14-reference/08-taos-shell.md b/docs/zh/14-reference/08-taos-shell.md index 5ac5f6f479e52aa7dd9a606a8eeb19e8c11a8a3c..3423cf35bbeb54c526e2bbad7ec3eae230cce07b 100644 --- a/docs/zh/14-reference/08-taos-shell.md +++ b/docs/zh/14-reference/08-taos-shell.md @@ -61,12 +61,14 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; - -c CONFIGDIR: 指定配置文件目录，Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg` - -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 - -d DATABASE: 指定连接到服务端时使用的数据库 +- -E dsn: 使用 WebSocket DSN 连接云服务或者提供 WebSocket 连接的服务端 - -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 - -k: 测试服务端运行状态，0: unavailable，1: network ok，2: service ok，3: service degraded，4: exiting - -l PKTLEN: 网络测试时使用的测试包大小 - -n NETROLE: 网络连接测试时的测试范围，默认为 `client`, 可选值为 `client`、`server` - -N PKTNUM: 网络测试时使用的测试包数量 - -r: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) +- -R: 使用 RESTful 模式连接服务端 - -s COMMAND: 以非交互模式执行的 SQL 命令 - -t: 测试服务端启动状态，状态同-k - -w DISPLAYWIDTH: 客户端列显示宽度 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 6eeb577ab597baf7d7547d15a5048018ee8ef497..fe23684fde2501ea60cccba589527fdf590d3cef 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -99,6 +99,9 @@ taos --dump-config ## 监控相关 +:::note +请注意，完整的监控功能需要安装并运行 `taoskeeper` 服务。taoskeeper 负责接收监控指标数据并创建 `log` 库。 + ### monitor | 属性 | 说明 | @@ -623,6 +626,15 @@ charset 的有效值是 UTF-8。 | 缺省值 | 1 | | 补充说明 | 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下
2、手动启动，就在 taosd 执行目录下。 | +### enableScience + +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| 适用范围 | 仅客户端 TAOS-CLI 适用 | +| 含义 | 是否开启科学计数法显示浮点数 | +| 取值范围 | 0：否，1：是 | +| 缺省值 | 0 | + ### udf | 属性 | 说明 | diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index ac0c8754b5a200ec571c956dc0308df6c1b026a3..b4d35fb2400be9407e32182c3d632a52538b95bd 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -4,23 +4,25 @@ title: taosKeeper description: TDengine 3.0 版本监控指标的导出工具 --- +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + ## 简介 taosKeeper 是 TDengine 3.0 版本监控指标的导出工具，通过简单的几项配置即可获取 TDengine 的运行状态。taosKeeper 使用 TDengine RESTful 接口，所以不需要安装 TDengine 客户端即可使用。 ## 安装 - +taosKeeper 有两种安装方式： taosKeeper 安装方式： - +- 安装 TDengine 官方安装包的同时会自动安装 taosKeeper, 详情请参考[ TDengine 安装](/operation/pkg-install)。 - - 单独编译 taosKeeper 并安装，详情请参考 [taosKeeper](https://github.com/taosdata/taoskeeper) 仓库。 -## 运行 +## 配置和运行方式 -### 配置和运行方式 +### 配置 taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：[命令行参数](#命令行参数启动)、[环境变量](#环境变量启动) 和 [配置文件](#配置文件启动)。优先级为：命令行参数、环境变量、配置文件参数。 @@ -34,29 +36,82 @@ monitorFqdn localhost # taoskeeper 服务的 FQDN TDengine 监控配置相关，具体请参考：[TDengine 监控配置](../config/#监控相关)。 -### 命令行参数启动 +### 启动 -在使用命令行参数运行 taosKeeper 并控制其行为。 + + -```shell -$ taosKeeper +安装后，请使用 `systemctl` 命令来启动 taoskeeper 的服务进程。 + +```bash +systemctl start taoskeeper ``` -### 环境变量启动 +检查服务是否正常工作： -通过设置环境变量达到控制启动参数的目的，通常在容器中运行时使用。 +```bash +systemctl status taoskeeper +``` -```shell -$ export TAOS_KEEPER_TDENGINE_HOST=192.168.64.3 - -$ taoskeeper +如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： + +``` +Active: active (running) ``` -具体参数列表请参照 `taoskeeper -h` 输入结果。 +如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： + +``` +Active: inactive (dead) +``` + +如下 `systemctl` 命令可以帮助你管理 taoskeeper 服务： + +- 启动服务进程：`systemctl start taoskeeper` + +- 停止服务进程：`systemctl stop taoskeeper` + +- 重启服务进程：`systemctl restart taoskeeper` + +- 查看服务状态：`systemctl status taoskeeper` + +:::info + +- `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。 +- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taoskeeper` 方式启动 taoskeeper 服务。 +- 故障排查： +- 如果服务异常请查看系统日志获取更多信息。 +::: + + + + +安装后，可以运行 `sudo launchctl start com.tdengine.taoskeeper` 来启动 taoskeeper 服务进程。 + +如下 `launchctl` 命令用于管理 taoskeeper 服务： + +- 启动服务进程：`sudo launchctl start com.tdengine.taoskeeper` + +- 停止服务进程：`sudo launchctl stop com.tdengine.taoskeeper` + +- 查看服务状态：`sudo launchctl list | grep taoskeeper` + +:::info + +- `launchctl` 命令管理`com.tdengine.taoskeeper`需要管理员权限，务必在前面加 `sudo` 来增强安全性。 +- `sudo launchctl list | grep taoskeeper` 指令返回的第一列是 `taoskeeper` 程序的 PID，若为 `-` 则说明 taoskeeper 服务未运行。 +- 故障排查： +- 如果服务异常请查看系统日志获取更多信息。 -### 配置文件启动 +::: -执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 + + + + +#### 配置文件启动 + +执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/taoskeeper.toml` 配置，否则将使用默认配置。 ```shell $ taoskeeper -c @@ -101,6 +156,10 @@ database = "log" # 指定需要监控的普通表 tables = [] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### 获取监控指标 @@ -134,19 +193,62 @@ $ curl http://127.0.0.1:6043/metrics 部分结果集： ```shell -# HELP taos_cluster_info_connections_total +# HELP taos_cluster_info_connections_total # TYPE taos_cluster_info_connections_total counter taos_cluster_info_connections_total{cluster_id="5981392874047724755"} 16 -# HELP taos_cluster_info_dbs_total +# HELP taos_cluster_info_dbs_total # TYPE taos_cluster_info_dbs_total counter taos_cluster_info_dbs_total{cluster_id="5981392874047724755"} 2 -# HELP taos_cluster_info_dnodes_alive +# HELP taos_cluster_info_dnodes_alive # TYPE taos_cluster_info_dnodes_alive counter taos_cluster_info_dnodes_alive{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_dnodes_total +# HELP taos_cluster_info_dnodes_total # TYPE taos_cluster_info_dnodes_total counter taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 -# HELP taos_cluster_info_first_ep +# HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` + +### check\_health + +``` +$ curl -i http://127.0.0.1:6043/check_health +``` + +返回结果： + +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Mon, 03 Apr 2023 07:20:38 GMT +Content-Length: 19 + +{"version":"1.0.0"} +``` + +### 集成 Prometheus + +taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 + +#### 抽取配置 + +Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。 + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +我们提供了 `TaosKeeper Prometheus Dashboard for 3.x` dashboard，提供了和 TDinsight 类似的监控 dashboard。 + +在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。 + diff --git a/docs/zh/17-operation/01-pkg-install.md b/docs/zh/17-operation/01-pkg-install.md index 6d93c1697b1e0936b3f6539d3b1fb95db0baa956..6272c14db79537dd65c93e424097b9c31e88ddb9 100644 --- a/docs/zh/17-operation/01-pkg-install.md +++ b/docs/zh/17-operation/01-pkg-install.md @@ -141,8 +141,20 @@ taos tools is uninstalled successfully! ``` + 在 C:\TDengine 目录下，通过运行 unins000.exe 卸载程序来卸载 TDengine。 + + + + +卸载 TDengine 命令如下: + +``` +$ rmtaos +TDengine is removed successfully! +``` + diff --git a/docs/zh/17-operation/03-tolerance.md b/docs/zh/17-operation/03-tolerance.md index 79cf10c39a7028e04e7c1ebbea54738dcdc528af..bf2a651c1a9a68411a2348eccda7e0e44db75214 100644 --- a/docs/zh/17-operation/03-tolerance.md +++ b/docs/zh/17-operation/03-tolerance.md @@ -19,12 +19,8 @@ TDengine 接收到应用的请求数据包时，先将请求的原始数据包 ## 灾备 -TDengine 的集群通过多个副本的机制，来提供系统的高可用性，同时具备一定的灾备能力。 +TDengine 灾备是通过在异地的两个数据中心中设置两个 TDengine 集群并利用 taosX 的数据复制能力来实现的。假定两个集群为集群 A 和集群 B，其中集群 A 为源集群，承担写入请求并提供查询服务。则在集群 A 所在数据中心中可以配置 taosX 利用 TDengine 提供的数据订阅能力，实时消费集群 A 中新写入的数据，并同步到集群 B。如果发生了灾难导致集群 A 所在数据中心不可用，则可以启用集群 B 作为数据写入和查询的主节点，并在集群 B 所处数据中心中配置 taosX 将数据复制到已经恢复的集群 A 或者新建的集群 C。 -TDengine 集群是由 mnode 负责管理的，为保证 mnode 的高可靠，可以配置 三个 mnode 副本。为保证元数据的强一致性，mnode 副本之间通过同步方式进行数据复制，保证了元数据的强一致性。 +利用 taosX 的数据复制能力也可以构造出更复杂的灾备方案。 -TDengine 集群中的时序数据的副本数是与数据库关联的，一个集群里可以有多个数据库，每个数据库可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数。为了支持高可靠，需要设置副本数为 3。 - -TDengine 集群的节点数必须大于等于副本数，否则创建表时将报错。 - -当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。 +taosX 只在 TDengine 企业版中提供，关于其具体细节，请联系 business@taosdata.com diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index e936f35dcac544ad94035b5e5c9716c4aa50562e..7def90916c9ba5b75c4d1e1be8a2836a5251eec6 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -38,3 +38,303 @@ chmod +x TDinsight.sh 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。 更多使用场景和限制请参考[TDinsight](/reference/tdinsight/) 文档。 + +## log 库 + +TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 + +### cluster\_info 表 + +`cluster_info` 表记录集群信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||集群 first ep| +|first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id| +|version|VARCHAR||tdengine version。例如：3.0.4.0| +|master\_uptime|FLOAT||当前 master 节点的uptime。单位：天| +|monitor\_interval|INT||monitor interval。单位：秒| +|dbs\_total|INT||database 总数| +|tbs\_total|BIGINT||当前集群 table 总数| +|stbs\_total|INT||当前集群 stable 总数| +|dnodes\_total|INT||当前集群 dnode 总数| +|dnodes\_alive|INT||当前集群 dnode 存活总数| +|mnodes\_total|INT||当前集群 mnode 总数| +|mnodes\_alive|INT||当前集群 mnode 存活总数| +|vgroups\_total|INT||当前集群 vgroup 总数| +|vgroups\_alive|INT||当前集群 vgroup 存活总数| +|vnodes\_total|INT||当前集群 vnode 总数| +|vnodes\_alive|INT||当前集群 vnode 存活总数| +|connections\_total|INT||当前集群连接总数| +|topics\_total|INT||当前集群 topic 总数| +|streams\_total|INT||当前集群 stream 总数| +|protocol|INT||协议版本，目前为 1| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info 表 + +`d_info` 表记录 dnode 状态信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode 状态| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info 表 + +`m_info` 表记录 mnode 角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||mnode 角色， leader 或 follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info 表 + +`dnodes_info` 记录 dnode 信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||taosd cpu 使用率，从 `/proc//stat` 读取| +|cpu\_system|FLOAT||服务器 cpu 使用率，从 `/proc/stat` 读取| +|cpu\_cores|FLOAT||服务器 cpu 核数| +|mem\_engine|INT||taosd 内存使用率，从 `/proc//status` 读取| +|mem\_system|INT||服务器可用内存| +|mem\_total|INT||服务器内存总量，单位 KB| +|disk\_engine|INT||| +|disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes| +|disk\_total|BIGINT||data dir 挂载的磁盘总容量，单位 bytes| +|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 kb/s| +|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 kb/s| +|io\_read|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_write|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 kb/s| +|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 kb/s| +|req\_select|INT||两个间隔内发生的查询请求数目| +|req\_select\_rate|FLOAT||两个间隔内的查询请求速度 = `req_select / monitorInterval`| +|req\_insert|INT||两个间隔内发生的写入请求，包含的单条数据数目| +|req\_insert\_success|INT||两个间隔内发生的处理成功的写入请求，包含的单条数据数目| +|req\_insert\_rate|FLOAT||两个间隔内的单条数据写入速度 = `req_insert / monitorInterval`| +|req\_insert\_batch|INT||两个间隔内发生的写入请求数目| +|req\_insert\_batch\_success|INT||两个间隔内发生的成功的批量写入请求数目| +|req\_insert\_batch\_rate|FLOAT||两个间隔内的写入请求数目的速度 = `req_insert_batch / monitorInterval`| +|errors|INT||两个间隔内的出错的写入请求的数目| +|vnodes\_num|INT||dnode 上 vnodes 数量| +|masters|INT||dnode 上 master node 数量| +|has\_mnode|INT||dnode 是否包含 mnode| +|has\_qnode|INT||dnode 是否包含 qnode| +|has\_snode|INT||dnode 是否包含 snode| +|has\_bnode|INT||dnode 是否包含 bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir 表 + +`data_dir` 表记录 data 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data 目录，一般为 `/var/lib/taos`| +|level|INT||0、1、2 多级存储级别| +|avail|BIGINT||data 目录可用空间| +|used|BIGINT||data 目录已使用空间| +|total|BIGINT||data 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir 表 + +`log_dir` 表记录 log 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log 目录名，一般为 `/var/log/taos/`| +|avail|BIGINT||log 目录可用空间| +|used|BIGINT||log 目录已使用空间| +|total|BIGINT||log 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir 表 + +`temp_dir` 表记录 temp 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp 目录名，一般为 `/tmp/`| +|avail|BIGINT||temp 目录可用空间| +|used|BIGINT||temp 目录已使用空间| +|total|BIGINT||temp 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info 表 + +`vgroups_info` 表记录虚拟节点组信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||vgroup 所属的 database 名字| +|tables\_num|BIGINT||vgroup 中 table 数量| +|status|VARCHAR||vgroup 状态| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role 表 + +`vnodes_role` 表记录虚拟节点角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||vnode 角色，leader 或 follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs 表 + +`logs` 表记录登录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content，长度不超过1024字节| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary 表 + +`log_summary` 记录日志统计信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error 总数| +|info|INT||info 总数| +|debug|INT||debug 总数| +|trace|INT||trace 总数| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info 表 + +`grants_info` 记录授权信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||认证过期时间，企业版有效，社区版为 bigint 最大值| +|timeseries\_used|BIGINT||已用测点数| +|timeseries\_total|BIGINT||总测点数，开源版本为 bigint 最大值| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor 表 + +`keeper_monitor` 记录 taoskeeper 监控数据。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu 使用率| +|mem|FLOAT||内存使用率| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total 表 + +`taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail 表 + +`taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight 表 + +`taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds 表 + +`taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent 表 + +`taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent 表 + +`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx index 5927dc4fcacc0b947e3a4c44778d9d5979bfc196..7980736a1c62150df1c2742c788aabde9b1dbc3b 100644 --- a/docs/zh/20-third-party/01-grafana.mdx +++ b/docs/zh/20-third-party/01-grafana.mdx @@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource 或者从 [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) 或 [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) 下载 .zip 文件到本地并解压到 Grafana 插件目录。命令行下载示例如下： ```bash -GF_VERSION=3.2.9 +GF_VERSION=3.3.1 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 5c8c5edd0519764ec893b551a2dfc934ad44e5c4..0974289c1fa5a1e220589af3a5d43e237b47acb6 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,18 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + +## 3.0.3.2 + + + +## 3.0.3.1 + + + ## 3.0.3.0 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index a9c9ea5ac7d153df6353ffebf59c77b7a5c5867b..78926555f1f4e1d579e8b2593575c84f67d18f6a 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,18 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.12 + + + +## 2.4.11 + + + +## 2.4.10 + + + ## 2.4.9 diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml index 68224bbad5f719b74ff7ed103d4024985bbb2887..4731d8e2370687a0fce0cb68d09ff4948266c30b 100644 --- a/examples/JDBC/taosdemo/pom.xml +++ b/examples/JDBC/taosdemo/pom.xml @@ -10,7 +10,7 @@ Demo project for TDengine - 5.3.20 + 5.3.26 diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 83b40c4f20a332b94abe45dd5c4fb7afd820861a..94545dfaad943a3ec5e35ae6859213835d8c4540 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -20,7 +20,8 @@ #include #include "taos.h" -static int running = 1; +static int running = 1; +const char* topic_name = "topicname"; static int32_t msg_process(TAOS_RES* msg) { char buf[1024]; @@ -243,7 +244,7 @@ _end: tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, topic_name); if (code) { tmq_list_destroy(topicList); return NULL; @@ -269,6 +270,31 @@ void basic_consume_loop(tmq_t* tmq) { fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +void consume_repeatly(tmq_t* tmq) { + int32_t numOfAssignment = 0; + tmq_topic_assignment* pAssign = NULL; + + int32_t code = tmq_get_topic_assignment(tmq, topic_name, &pAssign, &numOfAssignment); + if (code != 0) { + fprintf(stderr, "failed to get assignment, reason:%s", tmq_err2str(code)); + } + + // seek to the earliest offset + for(int32_t i = 0; i < numOfAssignment; ++i) { + tmq_topic_assignment* p = &pAssign[i]; + + code = tmq_offset_seek(tmq, topic_name, p->vgId, p->begin); + if (code != 0) { + fprintf(stderr, "failed to seek to %ld, reason:%s", p->begin, tmq_err2str(code)); + } + } + + free(pAssign); + + // let's do it again + basic_consume_loop(tmq); +} + int main(int argc, char* argv[]) { int32_t code; @@ -294,10 +320,13 @@ int main(int argc, char* argv[]) { if ((code = tmq_subscribe(tmq, topic_list))) { fprintf(stderr, "Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); } + tmq_list_destroy(topic_list); basic_consume_loop(tmq); + consume_repeatly(tmq); + code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "Failed to close consumer: %s\n", tmq_err2str(code)); diff --git a/examples/lua/OpenResty/rest/tdpool/init.lua b/examples/lua/OpenResty/rest/tdpool/init.lua index ebf8e91756539cc8af5db38232a40bf42aeaa245..c0c6d5659013b4a4586c286741a20867fafd9c9f 100644 --- a/examples/lua/OpenResty/rest/tdpool/init.lua +++ b/examples/lua/OpenResty/rest/tdpool/init.lua @@ -1,16 +1,15 @@ local _M = {} local driver = require "luaconnector51" -local water_mark = 0 -local occupied = 0 -local connection_pool = {} +td_pool_watermark = 0 +td_pool_occupied = 0 +td_connection_pool = {} -function _M.new(o,config) +function _M.new(o, config) o = o or {} - o.connection_pool = connection_pool - o.water_mark = water_mark - o.occupied = occupied - if #connection_pool == 0 then - + o.connection_pool = td_connection_pool + o.watermark = td_pool_watermark + o.occupied = td_pool_occupied + if #td_connection_pool == 0 then for i = 1, config.connection_pool_size do local res = driver.connect(config) if res.code ~= 0 then @@ -18,8 +17,8 @@ function _M.new(o,config) return nil else local object = {obj = res.conn, state = 0} - table.insert(o.connection_pool,i, object) - ngx.log(ngx.INFO, "add connection, now pool size:"..#(o.connection_pool)) + table.insert(td_connection_pool, i, object) + ngx.log(ngx.INFO, "add connection, now pool size:"..#(td_connection_pool)) end end @@ -32,13 +31,13 @@ function _M:get_connection() local connection_obj - for i = 1, #connection_pool do - connection_obj = connection_pool[i] + for i = 1, #td_connection_pool do + connection_obj = td_connection_pool[i] if connection_obj.state == 0 then connection_obj.state = 1 - occupied = occupied +1 - if occupied > water_mark then - water_mark = occupied + td_pool_occupied = td_pool_occupied + 1 + if td_pool_occupied > td_pool_watermark then + td_pool_watermark = td_pool_occupied end return connection_obj["obj"] end @@ -49,21 +48,27 @@ function _M:get_connection() return nil end -function _M:get_water_mark() +function _M:get_watermark() + + return td_pool_watermark +end + + +function _M:get_current_load() - return water_mark + return td_pool_occupied end function _M:release_connection(conn) local connection_obj - for i = 1, #connection_pool do - connection_obj = connection_pool[i] + for i = 1, #td_connection_pool do + connection_obj = td_connection_pool[i] if connection_obj["obj"] == conn then connection_obj["state"] = 0 - occupied = occupied -1 + td_pool_occupied = td_pool_occupied -1 return end end diff --git a/examples/lua/OpenResty/rest/test.lua b/examples/lua/OpenResty/rest/test.lua index 48aeef3fb4dd8c9a0dc18e8039b4b8c781760666..cb4a1479f34600f6e43802c6c7d82754a93433ad 100644 --- a/examples/lua/OpenResty/rest/test.lua +++ b/examples/lua/OpenResty/rest/test.lua @@ -4,8 +4,21 @@ local Pool = require "tdpool" local config = require "config" ngx.say("start time:"..os.time()) -local pool = Pool.new(Pool,config) -local conn = pool:get_connection() +local pool = Pool.new(Pool, config) +local another_pool = Pool.new(Pool, config) +local conn, conn1, conn2 +conn = pool:get_connection() +conn1 = pool:get_connection() +conn2 = pool:get_connection() +local temp_conn = another_pool:get_connection() +ngx.say("pool size:"..config.connection_pool_size) +ngx.say("pool watermark:"..pool:get_watermark()) +ngx.say("pool current load:"..pool:get_current_load()) +pool:release_connection(conn1) +pool:release_connection(conn2) +another_pool:release_connection(temp_conn) +ngx.say("pool watermark:"..pool:get_watermark()) +ngx.say("pool current load:"..pool:get_current_load()) local res = driver.query(conn,"drop database if exists nginx") if res.code ~=0 then @@ -31,7 +44,6 @@ end res = driver.query(conn,"create table m1 (ts timestamp, speed int,owner binary(20))") if res.code ~=0 then ngx.say("create table---failed: "..res.error) - else ngx.say("create table--- pass.") end @@ -83,8 +95,5 @@ while not flag do -- ngx.say("i am here once...") ngx.sleep(0.001) -- time unit is second end - -ngx.say("pool water_mark:"..pool:get_water_mark()) - pool:release_connection(conn) ngx.say("end time:"..os.time()) diff --git a/examples/lua/build.sh b/examples/lua/build.sh index 1e6fef632ceb828c9e0eca1a02dc4a6654906474..5e856f3e9ef0c18b309c1b6fa66f8c79b7dec2dd 100755 --- a/examples/lua/build.sh +++ b/examples/lua/build.sh @@ -4,5 +4,5 @@ if [ "$lua_header_installed" = "0" ]; then sudo apt install -y liblua5.3-dev fi -gcc -std=c99 lua_connector.c -fPIC -shared -o luaconnector.so -Wall -ltaos -I/usr/include/lua5.3 -I../../include/client +gcc -g -std=c99 lua_connector.c -fPIC -shared -o luaconnector.so -Wall -ltaos -I/usr/include/lua5.3 -I../../include/client diff --git a/examples/lua/lua51/lua_connector51.c b/examples/lua/lua51/lua_connector51.c index 8a9051dd0cb3da0fd82e752132d499a5935be758..c5ef9126a644cae83a730f5d39d501713aff9db5 100644 --- a/examples/lua/lua51/lua_connector51.c +++ b/examples/lua/lua51/lua_connector51.c @@ -67,7 +67,7 @@ static int l_connect(lua_State *L){ taos = taos_connect(host, user,password,database, port); if (taos == NULL) { - printf("failed to connect server, reason:%s\n", taos_errstr(taos)); + //printf("failed to connect server, reason:%s\n", taos_errstr(taos)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); @@ -79,7 +79,7 @@ static int l_connect(lua_State *L){ // printf("success to connect server\n"); lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); + lua_pushstring(L, "success"); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,taos); lua_setfield(L, table_index, "conn"); diff --git a/examples/lua/lua_connector.c b/examples/lua/lua_connector.c index c3d8bcb99548c787953cd27bcbb90b6c63ee5a4a..5d8226f2623f6451ef77b71dac1a6583ba6c460b 100644 --- a/examples/lua/lua_connector.c +++ b/examples/lua/lua_connector.c @@ -67,8 +67,7 @@ static int l_connect(lua_State *L){ taos = taos_connect(host, user,password,database, port); if (taos == NULL) { - printf("failed to connect server, reason:%s\n", taos_errstr(taos)); - +// printf("failed to connect server, reason:%s\n", taos_errstr(NULL)); lua_pushinteger(L, -1); lua_setfield(L, table_index, "code"); lua_pushstring(L, taos_errstr(taos)); @@ -79,7 +78,7 @@ static int l_connect(lua_State *L){ // printf("success to connect server\n"); lua_pushinteger(L, 0); lua_setfield(L, table_index, "code"); - lua_pushstring(L, taos_errstr(taos)); + lua_pushstring(L, "success"); lua_setfield(L, table_index, "error"); lua_pushlightuserdata(L,taos); lua_setfield(L, table_index, "conn"); diff --git a/examples/python/taosdemo/taosdemo.py b/examples/python/taosdemo/taosdemo.py index d55023bdbf119544a788aa6246c9d63dbf024872..13896bf54117d80d67a840fdb1158e3cbf793229 100755 --- a/examples/python/taosdemo/taosdemo.py +++ b/examples/python/taosdemo/taosdemo.py @@ -21,7 +21,7 @@ import json import random import time import datetime -from multiprocessing import Manager, Pool, Lock +from multiprocessing import Manager, Pool from multipledispatch import dispatch from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED @@ -102,12 +102,7 @@ def restful_execute(host: str, port: int, user: str, password: str, cmd: str): v_print("resp status: %d", resp.status_code) if debug: - v_print( - "resp text: %s", - json.dumps( - resp.json(), - sort_keys=True, - indent=2)) + v_print("resp text: %s", json.dumps(resp.json(), sort_keys=True, indent=2)) else: print("resp: %s" % json.dumps(resp.json())) @@ -115,34 +110,29 @@ def restful_execute(host: str, port: int, user: str, password: str, cmd: str): def query_func(process: int, thread: int, cmd: str): v_print("%d process %d thread cmd: %s", process, thread, cmd) - if oneMoreHost != "NotSupported" and random.randint( - 0, 1) == 1: + if oneMoreHost != "NotSupported" and random.randint(0, 1) == 1: v_print("%s", "Send to second host") if native: - cursor2.execute(cmd) + cursor.execute(cmd) else: - restful_execute( - oneMoreHost, port, user, password, cmd) + restful_execute(oneMoreHost, port, user, password, cmd) else: v_print("%s%s%s", "Send ", cmd, " to the host") if native: pass -# cursor.execute(cmd) + # cursor.execute(cmd) else: - restful_execute( - host, port, user, password, cmd) + restful_execute(host, port, user, password, cmd) def query_data_process(cmd: str): # establish connection if native if native: - v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + v_print("host:%s, user:%s passwd:xxxxxx configDir:%s ", host, user, configDir) try: conn = taos.connect( - host=host, - user=user, - password=password, - config=configDir) + host=host, user=user, password=password, config=configDir + ) v_print("conn: %s", str(conn.__class__)) except Exception as e: print("Error: %s" % e.args[0]) @@ -160,6 +150,7 @@ def query_data_process(cmd: str): try: cursor.execute(cmd) cols = cursor.description + print(cols) data = cursor.fetchall() for col in data: @@ -170,12 +161,7 @@ def query_data_process(cmd: str): sys.exit(1) else: - restful_execute( - host, - port, - user, - password, - cmd) + restful_execute(host, port, user, password, cmd) if native: cursor.close() @@ -186,21 +172,21 @@ def create_stb(): for i in range(0, numOfStb): if native: cursor.execute( - "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % - (stbName, i)) + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" + % (stbName, i) + ) else: restful_execute( host, port, user, password, - "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % - (stbName, i) + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" + % (stbName, i), ) def use_database(): - if native: cursor.execute("USE %s" % current_db) else: @@ -212,15 +198,15 @@ def create_databases(): v_print("will create database db%d", int(i)) if native: - cursor.execute( - "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + cursor.execute("CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) else: restful_execute( host, port, user, password, - "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i), + ) def drop_tables(): @@ -243,17 +229,11 @@ def drop_databases(): v_print("will drop database db%d", int(i)) if native: - cursor.execute( - "DROP DATABASE IF EXISTS %s%d" % - (dbName, i)) + cursor.execute("DROP DATABASE IF EXISTS %s%d" % (dbName, i)) else: restful_execute( - host, - port, - user, - password, - "DROP DATABASE IF EXISTS %s%d" % - (dbName, i)) + host, port, user, password, "DROP DATABASE IF EXISTS %s%d" % (dbName, i) + ) def insert_func(process: int, thread: int): @@ -266,13 +246,11 @@ def insert_func(process: int, thread: int): # establish connection if native if native: - v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + v_print("host:%s, user:%s passwd:xxxxxx configDir:%s ", host, user, configDir) try: conn = taos.connect( - host=host, - user=user, - password=password, - config=configDir) + host=host, user=user, password=password, config=configDir + ) v_print("conn: %s", str(conn.__class__)) except Exception as e: print("Error: %s" % e.args[0]) @@ -291,26 +269,29 @@ def insert_func(process: int, thread: int): row = 0 while row < numOfRec: v_print("row: %d", row) - sqlCmd = ['INSERT INTO '] + sqlCmd = ["INSERT INTO "] try: - sqlCmd.append( - "%s.%s%d " % (current_db, tbName, thread)) + sqlCmd.append("%s.%s%d " % (current_db, tbName, thread)) - if (numOfStb > 0 and autosubtable): - sqlCmd.append("USING %s.%s%d TAGS('%s') " % - (current_db, stbName, numOfStb - 1, uuid)) + if numOfStb > 0 and autosubtable: + sqlCmd.append( + "USING %s.%s%d TAGS('%s') " + % (current_db, stbName, numOfStb - 1, uuid) + ) - start_time = datetime.datetime( - 2021, 1, 25) + datetime.timedelta(seconds=row) + start_time = datetime.datetime(2021, 1, 25) + datetime.timedelta( + seconds=row + ) sqlCmd.append("VALUES ") for batchIter in range(0, batch): - sqlCmd.append("('%s', %f) " % - ( - start_time + - datetime.timedelta( - milliseconds=batchIter), - random.random())) + sqlCmd.append( + "('%s', %f) " + % ( + start_time + datetime.timedelta(milliseconds=batchIter), + random.random(), + ) + ) row = row + 1 if row >= numOfRec: v_print("BREAK, row: %d numOfRec:%d", row, numOfRec) @@ -319,23 +300,21 @@ def insert_func(process: int, thread: int): except Exception as e: print("Error: %s" % e.args[0]) - cmd = ' '.join(sqlCmd) + cmd = " ".join(sqlCmd) if measure: exec_start_time = datetime.datetime.now() if native: affectedRows = cursor.execute(cmd) + print("affectedRows: %d" % affectedRows) else: - restful_execute( - host, port, user, password, cmd) + restful_execute(host, port, user, password, cmd) if measure: exec_end_time = datetime.datetime.now() exec_delta = exec_end_time - exec_start_time - v_print( - "consume %d microseconds", - exec_delta.microseconds) + v_print("consume %d microseconds", exec_delta.microseconds) v_print("cmd: %s, length:%d", cmd, len(cmd)) @@ -355,51 +334,39 @@ def create_tb(): if native: cursor.execute("USE %s%d" % (dbName, i)) else: - restful_execute( - host, port, user, password, "USE %s%d" % - (dbName, i)) + restful_execute(host, port, user, password, "USE %s%d" % (dbName, i)) for j in range(0, numOfTb): if native: cursor.execute( - "CREATE TABLE %s%d (ts timestamp, value float)" % - (tbName, j)) + "CREATE TABLE %s%d (ts timestamp, value float)" % (tbName, j) + ) else: restful_execute( host, port, user, password, - "CREATE TABLE %s%d (ts timestamp, value float)" % - (tbName, j)) + "CREATE TABLE %s%d (ts timestamp, value float)" % (tbName, j), + ) def insert_data_process(lock, i: int, begin: int, end: int): lock.acquire() tasks = end - begin - v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) + v_print( + "insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks + ) - if (threads < (end - begin)): + if threads < (end - begin): for j in range(begin, end, threads): with ThreadPoolExecutor(max_workers=threads) as executor: k = end if ((j + threads) > end) else (j + threads) - workers = [ - executor.submit( - insert_func, - i, - n) for n in range( - j, - k)] + workers = [executor.submit(insert_func, i, n) for n in range(j, k)] wait(workers, return_when=ALL_COMPLETED) else: with ThreadPoolExecutor(max_workers=threads) as executor: - workers = [ - executor.submit( - insert_func, - i, - j) for j in range( - begin, - end)] + workers = [executor.submit(insert_func, i, j) for j in range(begin, end)] wait(workers, return_when=ALL_COMPLETED) lock.release() @@ -409,22 +376,18 @@ def query_db(i): if native: cursor.execute("USE %s%d" % (dbName, i)) else: - restful_execute( - host, port, user, password, "USE %s%d" % - (dbName, i)) + restful_execute(host, port, user, password, "USE %s%d" % (dbName, i)) for j in range(0, numOfTb): if native: - cursor.execute( - "SELECT COUNT(*) FROM %s%d" % (tbName, j)) + cursor.execute("SELECT COUNT(*) FROM %s%d" % (tbName, j)) else: restful_execute( - host, port, user, password, "SELECT COUNT(*) FROM %s%d" % - (tbName, j)) + host, port, user, password, "SELECT COUNT(*) FROM %s%d" % (tbName, j) + ) def printConfig(): - print("###################################################################") print("# Use native interface: %s" % native) print("# Server IP: %s" % host) @@ -435,7 +398,6 @@ def printConfig(): print("# Configuration Dir: %s" % configDir) print("# User: %s" % user) - print("# Password: %s" % password) print("# Number of Columns per record: %s" % colsPerRecord) print("# Number of Threads: %s" % threads) print("# Number of Processes: %s" % processes) @@ -455,13 +417,14 @@ def printConfig(): print("# Query command: %s" % queryCmd) print("# Insert Only: %s" % insertOnly) print("# Verbose output %s" % verbose) - print("# Test time: %s" % - datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + print( + "# Test time: %s" + % datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S") + ) print("###################################################################") if __name__ == "__main__": - native = False verbose = False debug = False @@ -497,73 +460,131 @@ if __name__ == "__main__": skipPrompt = False try: - opts, args = getopt.gnu_getopt(sys.argv[1:], - 'Nh:p:u:P:d:a:m:Ms:Q:T:C:r:l:t:n:c:xOR:D:vgyH', - [ - 'native', 'host', 'port', 'user', 'password', 'dbname', 'replica', 'tbname', - 'stable', 'stbname', 'query', 'threads', 'processes', - 'recPerReq', 'colsPerRecord', 'numOfTb', 'numOfRec', 'config', - 'insertOnly', 'outOfOrder', 'rateOOOO', 'deleteMethod', - 'verbose', 'debug', 'skipPrompt', 'help' - ]) + opts, args = getopt.gnu_getopt( + sys.argv[1:], + "Nh:p:u:P:d:a:m:Ms:Q:T:C:r:l:t:n:c:xOR:D:vgyH", + [ + "native", + "host", + "port", + "user", + "password", + "dbname", + "replica", + "tbname", + "stable", + "stbname", + "query", + "threads", + "processes", + "recPerReq", + "colsPerRecord", + "numOfTb", + "numOfRec", + "config", + "insertOnly", + "outOfOrder", + "rateOOOO", + "deleteMethod", + "verbose", + "debug", + "skipPrompt", + "help", + ], + ) except getopt.GetoptError as err: - print('ERROR:', err) - print('Try `taosdemo.py --help` for more options.') + print("ERROR:", err) + print("Try `taosdemo.py --help` for more options.") sys.exit(1) if bool(opts) is False: - print('Try `taosdemo.py --help` for more options.') + print("Try `taosdemo.py --help` for more options.") sys.exit(1) for key, value in opts: - if key in ['-H', '--help']: - print('') + if key in ["-H", "--help"]: + print("") + print("taosdemo.py for TDengine") + print("") + print("Author: Shuduo Sang ") + print("") + + print("\t-H, --help Show usage.") + print("") + + print( + "\t-N, --native flag, Use native interface if set. Default is using RESTful interface." + ) + print( + "\t-h, --host host, The host to connect to TDengine. Default is localhost." + ) + print( + "\t-p, --port port, The TCP/IP port number to use for the connection. Default is 0." + ) print( - 'taosdemo.py for TDengine') - print('') - print('Author: Shuduo Sang ') - print('') - - print('\t-H, --help Show usage.') - print('') - - print('\t-N, --native flag, Use native interface if set. Default is using RESTful interface.') - print('\t-h, --host host, The host to connect to TDengine. Default is localhost.') - print('\t-p, --port port, The TCP/IP port number to use for the connection. Default is 0.') - print('\t-u, --user user, The user name to use when connecting to the server. Default is \'root\'.') - print('\t-P, --password password, The password to use when connecting to the server. Default is \'taosdata\'.') - print('\t-l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3.') + "\t-u, --user user, The user name to use when connecting to the server. Default is 'root'." + ) + print( + "\t-P, --password password, The password to use when connecting to the server. Default is 'taosdata'." + ) + print( + "\t-l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3." + ) + print( + "\t-d, --dbname database, Destination database. Default is 'test'." + ) + print( + "\t-a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5." + ) print( - '\t-d, --dbname database, Destination database. Default is \'test\'.') - print('\t-a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5.') + "\t-m, --tbname table_prefix, Table prefix name. Default is 't'." + ) print( - '\t-m, --tbname
table_prefix, Table prefix name. Default is \'t\'.') + "\t-M, --stable flag, Use super table. Default is no" + ) print( - '\t-M, --stable flag, Use super table. Default is no') + "\t-s, --stbname stable_prefix, STable prefix name. Default is 'st'" + ) print( - '\t-s, --stbname stable_prefix, STable prefix name. Default is \'st\'') - print('\t-Q, --query [NO|EACHTB|command] query, Execute query command. set \'EACHTB\' means select * from each table') + "\t-Q, --query [NO|EACHTB|command] query, Execute query command. set 'EACHTB' means select * from each table" + ) print( - '\t-T, --threads num_of_threads, The number of threads. Default is 1.') + "\t-T, --threads num_of_threads, The number of threads. Default is 1." + ) print( - '\t-C, --processes num_of_processes, The number of threads. Default is 1.') - print('\t-r, --batch num_of_records_per_req, The number of records per request. Default is 1000.') + "\t-C, --processes num_of_processes, The number of threads. Default is 1." + ) + print( + "\t-r, --batch num_of_records_per_req, The number of records per request. Default is 1000." + ) print( - '\t-t, --numOfTb num_of_tables, The number of tables. Default is 1.') - print('\t-n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1.') - print('\t-c, --config config_directory, Configuration directory. Default is \'/etc/taos/\'.') - print('\t-x, --inserOnly flag, Insert only flag.') - print('\t-O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order.') - print('\t-R, --rateOOOO rate, Out of order data\'s rate--if order=1 Default 10, min: 0, max: 50.') - print('\t-D, --deleteMethod Delete data methods 0: don\'t delete, 1: delete by table, 2: delete by stable, 3: delete by database.') - print('\t-v, --verbose Print verbose output') - print('\t-g, --debug Print debug output') + "\t-t, --numOfTb num_of_tables, The number of tables. Default is 1." + ) print( - '\t-y, --skipPrompt Skip read key for continous test, default is not skip') - print('') + "\t-n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1." + ) + print( + "\t-c, --config config_directory, Configuration directory. Default is '/etc/taos/'." + ) + print("\t-x, --inserOnly flag, Insert only flag.") + print( + "\t-O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order." + ) + print( + "\t-R, --rateOOOO rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50." + ) + print( + "\t-D, --deleteMethod Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database." + ) + print("\t-v, --verbose Print verbose output") + print("\t-g, --debug Print debug output") + print( + "\t-y, --skipPrompt Skip read key for continous test, default is not skip" + ) + print("") sys.exit(0) - if key in ['-N', '--native']: + if key in ["-N", "--native"]: try: import taos except Exception as e: @@ -571,104 +592,104 @@ if __name__ == "__main__": sys.exit(1) native = True - if key in ['-h', '--host']: + if key in ["-h", "--host"]: host = value - if key in ['-p', '--port']: + if key in ["-p", "--port"]: port = int(value) - if key in ['-u', '--user']: + if key in ["-u", "--user"]: user = value - if key in ['-P', '--password']: + if key in ["-P", "--password"]: password = value else: password = defaultPass - if key in ['-d', '--dbname']: + if key in ["-d", "--dbname"]: dbName = value - if key in ['-a', '--replica']: + if key in ["-a", "--replica"]: replica = int(value) if replica < 1: print("FATAL: number of replica need > 0") sys.exit(1) - if key in ['-m', '--tbname']: + if key in ["-m", "--tbname"]: tbName = value - if key in ['-M', '--stable']: + if key in ["-M", "--stable"]: useStable = True numOfStb = 1 - if key in ['-s', '--stbname']: + if key in ["-s", "--stbname"]: stbName = value - if key in ['-Q', '--query']: + if key in ["-Q", "--query"]: queryCmd = str(value) - if key in ['-T', '--threads']: + if key in ["-T", "--threads"]: threads = int(value) if threads < 1: print("FATAL: number of threads must be larger than 0") sys.exit(1) - if key in ['-C', '--processes']: + if key in ["-C", "--processes"]: processes = int(value) if processes < 1: print("FATAL: number of processes must be larger than 0") sys.exit(1) - if key in ['-r', '--batch']: + if key in ["-r", "--batch"]: batch = int(value) - if key in ['-l', '--colsPerRec']: + if key in ["-l", "--colsPerRec"]: colsPerRec = int(value) - if key in ['-t', '--numOfTb']: + if key in ["-t", "--numOfTb"]: numOfTb = int(value) v_print("numOfTb is %d", numOfTb) - if key in ['-n', '--numOfRec']: + if key in ["-n", "--numOfRec"]: numOfRec = int(value) v_print("numOfRec is %d", numOfRec) if numOfRec < 1: print("FATAL: number of records must be larger than 0") sys.exit(1) - - if key in ['-c', '--config']: + if key in ["-c", "--config"]: configDir = value v_print("config dir: %s", configDir) - if key in ['-x', '--insertOnly']: + if key in ["-x", "--insertOnly"]: insertOnly = True v_print("insert only: %d", insertOnly) - if key in ['-O', '--outOfOrder']: + if key in ["-O", "--outOfOrder"]: outOfOrder = int(value) v_print("out of order is %d", outOfOrder) - if key in ['-R', '--rateOOOO']: + if key in ["-R", "--rateOOOO"]: rateOOOO = int(value) v_print("the rate of out of order is %d", rateOOOO) - if key in ['-D', '--deleteMethod']: + if key in ["-D", "--deleteMethod"]: deleteMethod = int(value) if (deleteMethod < 0) or (deleteMethod > 3): print( - "inputed delete method is %d, valid value is 0~3, set to default 0" % - deleteMethod) + "inputed delete method is %d, valid value is 0~3, set to default 0" + % deleteMethod + ) deleteMethod = 0 v_print("the delete method is %d", deleteMethod) - if key in ['-v', '--verbose']: + if key in ["-v", "--verbose"]: verbose = True - if key in ['-g', '--debug']: + if key in ["-g", "--debug"]: debug = True - if key in ['-y', '--skipPrompt']: + if key in ["-y", "--skipPrompt"]: skipPrompt = True if verbose: @@ -679,13 +700,11 @@ if __name__ == "__main__": # establish connection first if native if native: - v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + v_print("host:%s, user:%s passwd:xxxxxx configDir:%s ", host, user, configDir) try: conn = taos.connect( - host=host, - user=user, - password=password, - config=configDir) + host=host, user=user, password=password, config=configDir + ) v_print("conn: %s", str(conn.__class__)) except Exception as e: print("Error: %s" % e.args[0]) @@ -705,7 +724,7 @@ if __name__ == "__main__": drop_tables() print("Drop tables done.") elif deleteMethod == 2: - drop_stables() + drop_stable() print("Drop super tables done.") elif deleteMethod == 3: drop_databases() @@ -725,7 +744,7 @@ if __name__ == "__main__": if numOfStb > 0: create_stb() - if (autosubtable == False): + if autosubtable is False: create_tb_using_stb() else: create_tb() @@ -734,7 +753,9 @@ if __name__ == "__main__": end_time = time.time() print( "Total time consumed {} seconds for create table.".format( - (end_time - start_time_begin))) + (end_time - start_time_begin) + ) + ) if native: cursor.close() @@ -758,10 +779,8 @@ if __name__ == "__main__": remainder = numOfTb % processes v_print( - "num of tables: %d, quotient: %d, remainder: %d", - numOfTb, - quotient, - remainder) + "num of tables: %d, quotient: %d, remainder: %d", numOfTb, quotient, remainder + ) for i in range(processes): begin = end @@ -770,7 +789,15 @@ if __name__ == "__main__": end = begin + quotient + 1 else: end = begin + quotient - pool.apply_async(insert_data_process, args=(lock, i, begin, end,)) + pool.apply_async( + insert_data_process, + args=( + lock, + i, + begin, + end, + ), + ) pool.close() pool.join() @@ -780,8 +807,9 @@ if __name__ == "__main__": end_time = time.time() print( "Total time consumed {} seconds for insert data.".format( - (end_time - start_time))) - + (end_time - start_time) + ) + ) # query data if queryCmd != "NO": @@ -790,8 +818,6 @@ if __name__ == "__main__": if measure: end_time = time.time() - print( - "Total time consumed {} seconds.".format( - (end_time - start_time_begin))) + print("Total time consumed {} seconds.".format((end_time - start_time_begin))) print("done") diff --git a/examples/rust/wrapper.h b/examples/rust/wrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..78857597a9dc0c9a0f35194339a0d132f788d592 --- /dev/null +++ b/examples/rust/wrapper.h @@ -0,0 +1 @@ +#include diff --git a/include/client/taos.h b/include/client/taos.h index cf410a42daf1e9c401af767497a603aa12c7a536..44703c2797d3e39bbc05f817a0cfd65acd6af2cf 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -101,6 +101,7 @@ typedef struct TAOS_FIELD_E { #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *res, int code); +typedef void (*__taos_notify_fn_t)(void *param, void *ext, int type); typedef struct TAOS_MULTI_BIND { int buffer_type; @@ -121,6 +122,10 @@ typedef enum { SET_CONF_RET_ERR_TOO_LONG = -6 } SET_CONF_RET_CODE; +typedef enum { + TAOS_NOTIFY_PASSVER = 0, +} TAOS_NOTIFY_TYPE; + #define RET_MSG_LENGTH 1024 typedef struct setConfRet { SET_CONF_RET_CODE retCode; @@ -225,6 +230,8 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type); + /* --------------------------schemaless INTERFACE------------------------------- */ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); @@ -262,6 +269,12 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst DLL_EXPORT const char *tmq_err2str(int32_t code); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ +typedef struct tmq_topic_assignment { + int32_t vgId; + int64_t currentOffset; + int64_t begin; + int64_t end; +} tmq_topic_assignment; DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); @@ -270,6 +283,8 @@ DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); +DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char* pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); +DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char* pTopicName, int32_t vgId, int64_t offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 2a40976a8b9a21e47ca49b0d62193f2c8fcff306..3d0dac7c3de095a60cb88c66bb82f70d66d2afdc 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,13 +25,6 @@ extern "C" { #endif -// TODO remove it -enum { - TMQ_CONF__RESET_OFFSET__NONE = -3, - TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, - TMQ_CONF__RESET_OFFSET__LATEST = -1, -}; - // clang-format off #define IS_META_MSG(x) ( \ x == TDMT_VND_CREATE_STB \ @@ -135,6 +128,7 @@ enum { TMQ_MSG_TYPE__POLL_META_RSP, TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__TAOSX_RSP, + TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__END_RSP, }; @@ -192,7 +186,7 @@ typedef struct SBlockID { typedef struct SDataBlockInfo { STimeWindow window; int32_t rowSize; - int32_t rows; // todo hide this attribute + int64_t rows; // todo hide this attribute uint32_t capacity; SBlockID id; int16_t hasVarCol; @@ -215,15 +209,12 @@ typedef struct SSDataBlock { } SSDataBlock; enum { - FETCH_TYPE__DATA = 1, - FETCH_TYPE__META, - FETCH_TYPE__SEP, + FETCH_TYPE__DATA = 0, FETCH_TYPE__NONE, }; typedef struct { int8_t fetchType; - STqOffsetVal offset; union { SSDataBlock data; void* meta; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 84d696e518c45c65d7fbb2f6d87689a656121dab..99fffa2cf1d6f28697fea0a96562707aeb3e3a71 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -178,7 +178,7 @@ int32_t getJsonValueLen(const char* data); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); -int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows); +int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, diff --git a/include/common/tglobal.h b/include/common/tglobal.h index ac75b8476299a8600ffa298e44b9f95b71640883..6fde7b48a29779b2a0c26fb58d360bbfc7ff221f 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -98,11 +98,13 @@ extern char *tsSvrCrashReportUri; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing -extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node +extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node +extern int32_t tsCacheLazyLoadThreshold; // cost threshold for last/last_row loading cache as much as possible // query client extern int32_t tsQueryPolicy; extern int32_t tsQueryRspPolicy; +extern int64_t tsQueryMaxConcurrentTables; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; @@ -110,6 +112,7 @@ extern int32_t tsQueryNodeChunkSize; extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; +extern bool tsEnableScience; extern int32_t tsRedirectPeriod; extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; @@ -145,10 +148,10 @@ extern char tsUdfdResFuncs[]; extern char tsUdfdLdLibPath[]; // schemaless -extern char tsSmlChildTableName[]; -extern char tsSmlTagName[]; -//extern bool tsSmlDataFormat; -//extern int32_t tsSmlBatchSize; +extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; +// extern bool tsSmlDataFormat; +// extern int32_t tsSmlBatchSize; // wal extern int64_t tsWalFsyncDataSizeLimit; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 17fde6a536777f255db3f2d62ba5042bfa34b231..f377ad0d63e140a8e5e4e7c46130b473d8bd684e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -77,7 +77,7 @@ static inline bool tmsgIsValid(tmsg_t type) { } static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT); } static inline bool syncUtilUserCommit(tmsg_t msgType) { @@ -106,6 +106,7 @@ enum { HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_TMQ, + HEARTBEAT_KEY_USER_PASSINFO, }; typedef enum _mgmt_table { @@ -177,6 +178,12 @@ typedef enum _mgmt_table { #define TSDB_ALTER_USER_SYSINFO 0xA #define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB #define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC +#define TSDB_ALTER_USER_ADD_READ_TABLE 0xD +#define TSDB_ALTER_USER_REMOVE_READ_TABLE 0xE +#define TSDB_ALTER_USER_ADD_WRITE_TABLE 0xF +#define TSDB_ALTER_USER_REMOVE_WRITE_TABLE 0x10 +#define TSDB_ALTER_USER_ADD_ALL_TABLE 0x11 +#define TSDB_ALTER_USER_REMOVE_ALL_TABLE 0x12 #define TSDB_ALTER_USER_PRIVILEGES 0x2 @@ -628,6 +635,7 @@ typedef struct { int8_t connType; SEpSet epSet; int32_t svrTimestamp; + int32_t passVer; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; @@ -669,13 +677,16 @@ int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); typedef struct { - int8_t alterType; - int8_t superUser; - int8_t sysInfo; - int8_t enable; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; - char objname[TSDB_DB_FNAME_LEN]; // db or topic + int8_t alterType; + int8_t superUser; + int8_t sysInfo; + int8_t enable; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; + char objname[TSDB_DB_FNAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; + char* tagCond; + int32_t tagCondLen; } SAlterUserReq; int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); @@ -698,12 +709,23 @@ typedef struct { SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; + SHashObj* readTbs; + SHashObj* writeTbs; + SHashObj* useDbs; } SGetUserAuthRsp; int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); +typedef struct SUserPassVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserPassVersion; + +typedef SGetUserAuthReq SGetUserPassReq; +typedef SUserPassVersion SGetUserPassRsp; + /* * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. @@ -845,6 +867,9 @@ typedef struct { int8_t cacheLast; int8_t replications; int32_t sstTrigger; + int32_t minRows; + int32_t walRetentionPeriod; + int32_t walRetentionSize; } SAlterDbReq; int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); @@ -1031,6 +1056,14 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserPassRsp +} SUserPassBatchRsp; + +int32_t tSerializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +int32_t tDeserializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +void tFreeSUserPassBatchRsp(SUserPassBatchRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; STimeWindow timeRange; @@ -1051,6 +1084,7 @@ typedef struct { int64_t signature; char* pComment; char* pCode; + int8_t orReplace; } SCreateFuncReq; int32_t tSerializeSCreateFuncReq(void* buf, int32_t bufLen, SCreateFuncReq* pReq); @@ -1089,9 +1123,15 @@ typedef struct { char* pCode; } SFuncInfo; +typedef struct { + int32_t funcVersion; + int64_t funcCreatedTime; +} SFuncExtraInfo; + typedef struct { int32_t numOfFuncs; SArray* pFuncInfos; + SArray* pFuncExtraInfos; } SRetrieveFuncRsp; int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); @@ -1135,6 +1175,7 @@ typedef struct { int64_t numOfInsertSuccessReqs; int64_t numOfBatchInsertReqs; int64_t numOfBatchInsertSuccessReqs; + int32_t numOfCachedTables; } SVnodeLoad; typedef struct { @@ -1263,6 +1304,9 @@ typedef struct { int16_t hashSuffix; int32_t tsdbPageSize; int64_t reserved[8]; + int8_t learnerReplica; + int8_t learnerSelfIndex; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -1316,6 +1360,12 @@ typedef struct { int8_t strict; int8_t cacheLast; int64_t reserved[8]; + // 1st modification + int16_t sttTrigger; + int32_t minRows; + // 2nd modification + int32_t walRetentionPeriod; + int32_t walRetentionSize; } SAlterVnodeConfigReq; int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq); @@ -1328,7 +1378,10 @@ typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; int64_t reserved[8]; -} SAlterVnodeReplicaReq; + int8_t learnerSelfIndex; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; +} SAlterVnodeReplicaReq, SAlterVnodeTypeReq; int32_t tSerializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); int32_t tDeserializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); @@ -1600,7 +1653,10 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQno typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; -} SDCreateMnodeReq, SDAlterMnodeReq; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; + int64_t lastIndex; +} SDCreateMnodeReq, SDAlterMnodeReq, SDAlterMnodeTypeReq; int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); @@ -1651,6 +1707,20 @@ typedef struct { int32_t tSerializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq); +typedef struct { + int32_t useless; +} SBalanceVgroupLeaderReq; + +int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); +int32_t tDeserializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq); + +typedef struct { + int32_t vgId; +} SForceBecomeFollowerReq; + +int32_t tSerializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); +int32_t tDeserializeSForceBecomeFollowerReq(void* buf, int32_t bufLen, SForceBecomeFollowerReq* pReq); + typedef struct { int32_t vgId; } SSplitVgroupReq; @@ -2762,6 +2832,7 @@ typedef struct { } SMqOffset; typedef struct { + int64_t consumerId; int32_t num; SMqOffset* offsets; } SMqCMCommitOffsetReq; @@ -2829,6 +2900,14 @@ typedef struct { int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); +typedef struct SMqVgOffset { + int64_t consumerId; + STqOffset offset; +} SMqVgOffset; + +int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset); +int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; @@ -3058,18 +3137,19 @@ typedef struct { int32_t code; int32_t epoch; int64_t consumerId; + int64_t walsver; + int64_t walever; } SMqRspHead; typedef struct { - SMsgHead head; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int8_t withTbName; - int8_t useSnapshot; - int32_t epoch; - uint64_t reqId; - int64_t consumerId; - int64_t timeout; - // int64_t currentOffset; + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t withTbName; + int8_t useSnapshot; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t timeout; STqOffsetVal reqOffset; } SMqPollReq; @@ -3104,43 +3184,9 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; -static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeString(buf, pTopicEp->db); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); - } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeStringTo(buf, pTopicEp->db); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - taosMemoryFreeClear(pSubTopicEp->schema.pSchema); - pSubTopicEp->schema.nCols = 0; - taosArrayDestroy(pSubTopicEp->vgs); -} +int32_t tEncodeMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp); +void* tDecodeMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp); +void tDeleteMqSubTopicEp(SMqSubTopicEp* pSubTopicEp); typedef struct { SMqRspHead head; @@ -3150,8 +3196,8 @@ typedef struct { void* metaRsp; } SMqMetaRsp; -int32_t tEncodeSMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp); -int32_t tDecodeSMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); +int32_t tEncodeMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp); +int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); typedef struct { SMqRspHead head; @@ -3166,9 +3212,9 @@ typedef struct { SArray* blockSchema; } SMqDataRsp; -int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); -int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); -void tDeleteSMqDataRsp(SMqDataRsp* pRsp); +int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteMqDataRsp(SMqDataRsp* pRsp); typedef struct { SMqRspHead head; @@ -3181,9 +3227,10 @@ typedef struct { SArray* blockData; SArray* blockTbName; SArray* blockSchema; - int32_t createTableNum; - SArray* createTableLen; - SArray* createTableReq; + // the following attributes are extended from SMqDataRsp + int32_t createTableNum; + SArray* createTableLen; + SArray* createTableReq; } STaosxRsp; int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp); @@ -3203,7 +3250,7 @@ static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pR tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); - tlen += tEncodeSMqSubTopicEp(buf, pVgEp); + tlen += tEncodeMqSubTopicEp(buf, pVgEp); } return tlen; } @@ -3218,14 +3265,14 @@ static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { } for (int32_t i = 0; i < sz; i++) { SMqSubTopicEp topicEp; - buf = tDecodeSMqSubTopicEp(buf, &topicEp); + buf = tDecodeMqSubTopicEp(buf, &topicEp); taosArrayPush(pRsp->topics, &topicEp); } return buf; } static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { - taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteSMqSubTopicEp); + taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteMqSubTopicEp); } #define TD_AUTO_CREATE_TABLE 0x1 diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index eca8740d2864612ff604601d089b206690854290..9b709272b2f5473828eee762a0834b999c2ec4bd 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -34,7 +34,7 @@ typedef enum { WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, - SYNC_CTRL_QUEUE, + SYNC_RD_QUEUE, STREAM_QUEUE, QUEUE_MAX, } EQueueType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 46ca814e50ac6d33f24456e6ed9ecb7e8769dd41..ffad6f66467fbdc90dcca4185a34d6a29d83e74b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -83,6 +83,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MND_MSG) TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -175,6 +177,7 @@ enum { // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) @@ -259,7 +262,7 @@ enum { TD_NEW_MSG_SEG(TDMT_SYNC_MSG) TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST_BATCH, "sync-client-request-batch", NULL, NULL) @@ -286,6 +289,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT, "sync-pre-snapshot", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT_REPLY, "sync-pre-snapshot-reply", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) @@ -298,9 +302,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK_TO_OFFSET, "vnode-tmq-seekto-offset", STqOffset, STqOffset) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) diff --git a/include/common/ttime.h b/include/common/ttime.h index 4a7c47d1725dcc3dd1fde1e5c17a021e992156fb..f189959f22ca4bee4b07518aff00e7bbc527fb87 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -64,7 +64,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) { : 1000000000; time_t t = taosTime(NULL); struct tm tm; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); tm.tm_hour = 0; tm.tm_min = 0; tm.tm_sec = 0; diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 1235dd344736d1245fd6cbc17ae7961554c6317d..641cbbb58885259502fd71ecf57dbf4690069afb 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -63,55 +63,55 @@ #define TK_READ 45 #define TK_WRITE 46 #define TK_NK_DOT 47 -#define TK_DNODE 48 -#define TK_PORT 49 -#define TK_DNODES 50 -#define TK_NK_IPTOKEN 51 -#define TK_FORCE 52 -#define TK_LOCAL 53 -#define TK_QNODE 54 -#define TK_BNODE 55 -#define TK_SNODE 56 -#define TK_MNODE 57 -#define TK_DATABASE 58 -#define TK_USE 59 -#define TK_FLUSH 60 -#define TK_TRIM 61 -#define TK_COMPACT 62 -#define TK_IF 63 -#define TK_NOT 64 -#define TK_EXISTS 65 -#define TK_BUFFER 66 -#define TK_CACHEMODEL 67 -#define TK_CACHESIZE 68 -#define TK_COMP 69 -#define TK_DURATION 70 -#define TK_NK_VARIABLE 71 -#define TK_MAXROWS 72 -#define TK_MINROWS 73 -#define TK_KEEP 74 -#define TK_PAGES 75 -#define TK_PAGESIZE 76 -#define TK_TSDB_PAGESIZE 77 -#define TK_PRECISION 78 -#define TK_REPLICA 79 -#define TK_VGROUPS 80 -#define TK_SINGLE_STABLE 81 -#define TK_RETENTIONS 82 -#define TK_SCHEMALESS 83 -#define TK_WAL_LEVEL 84 -#define TK_WAL_FSYNC_PERIOD 85 -#define TK_WAL_RETENTION_PERIOD 86 -#define TK_WAL_RETENTION_SIZE 87 -#define TK_WAL_ROLL_PERIOD 88 -#define TK_WAL_SEGMENT_SIZE 89 -#define TK_STT_TRIGGER 90 -#define TK_TABLE_PREFIX 91 -#define TK_TABLE_SUFFIX 92 -#define TK_NK_COLON 93 -#define TK_MAX_SPEED 94 -#define TK_START 95 -#define TK_WITH 96 +#define TK_WITH 48 +#define TK_DNODE 49 +#define TK_PORT 50 +#define TK_DNODES 51 +#define TK_NK_IPTOKEN 52 +#define TK_FORCE 53 +#define TK_LOCAL 54 +#define TK_QNODE 55 +#define TK_BNODE 56 +#define TK_SNODE 57 +#define TK_MNODE 58 +#define TK_DATABASE 59 +#define TK_USE 60 +#define TK_FLUSH 61 +#define TK_TRIM 62 +#define TK_COMPACT 63 +#define TK_IF 64 +#define TK_NOT 65 +#define TK_EXISTS 66 +#define TK_BUFFER 67 +#define TK_CACHEMODEL 68 +#define TK_CACHESIZE 69 +#define TK_COMP 70 +#define TK_DURATION 71 +#define TK_NK_VARIABLE 72 +#define TK_MAXROWS 73 +#define TK_MINROWS 74 +#define TK_KEEP 75 +#define TK_PAGES 76 +#define TK_PAGESIZE 77 +#define TK_TSDB_PAGESIZE 78 +#define TK_PRECISION 79 +#define TK_REPLICA 80 +#define TK_VGROUPS 81 +#define TK_SINGLE_STABLE 82 +#define TK_RETENTIONS 83 +#define TK_SCHEMALESS 84 +#define TK_WAL_LEVEL 85 +#define TK_WAL_FSYNC_PERIOD 86 +#define TK_WAL_RETENTION_PERIOD 87 +#define TK_WAL_RETENTION_SIZE 88 +#define TK_WAL_ROLL_PERIOD 89 +#define TK_WAL_SEGMENT_SIZE 90 +#define TK_STT_TRIGGER 91 +#define TK_TABLE_PREFIX 92 +#define TK_TABLE_SUFFIX 93 +#define TK_NK_COLON 94 +#define TK_MAX_SPEED 95 +#define TK_START 96 #define TK_TIMESTAMP 97 #define TK_END 98 #define TK_TABLE 99 @@ -127,24 +127,24 @@ #define TK_NK_EQ 109 #define TK_USING 110 #define TK_TAGS 111 -#define TK_COMMENT 112 -#define TK_BOOL 113 -#define TK_TINYINT 114 -#define TK_SMALLINT 115 -#define TK_INT 116 -#define TK_INTEGER 117 -#define TK_BIGINT 118 -#define TK_FLOAT 119 -#define TK_DOUBLE 120 -#define TK_BINARY 121 -#define TK_NCHAR 122 -#define TK_UNSIGNED 123 -#define TK_JSON 124 -#define TK_VARCHAR 125 -#define TK_MEDIUMBLOB 126 -#define TK_BLOB 127 -#define TK_VARBINARY 128 -#define TK_DECIMAL 129 +#define TK_BOOL 112 +#define TK_TINYINT 113 +#define TK_SMALLINT 114 +#define TK_INT 115 +#define TK_INTEGER 116 +#define TK_BIGINT 117 +#define TK_FLOAT 118 +#define TK_DOUBLE 119 +#define TK_BINARY 120 +#define TK_NCHAR 121 +#define TK_UNSIGNED 122 +#define TK_JSON 123 +#define TK_VARCHAR 124 +#define TK_MEDIUMBLOB 125 +#define TK_BLOB 126 +#define TK_VARBINARY 127 +#define TK_DECIMAL 128 +#define TK_COMMENT 129 #define TK_MAX_DELAY 130 #define TK_WATERMARK 131 #define TK_ROLLUP 132 @@ -208,142 +208,143 @@ #define TK_AGGREGATE 190 #define TK_BUFSIZE 191 #define TK_LANGUAGE 192 -#define TK_STREAM 193 -#define TK_INTO 194 -#define TK_TRIGGER 195 -#define TK_AT_ONCE 196 -#define TK_WINDOW_CLOSE 197 -#define TK_IGNORE 198 -#define TK_EXPIRED 199 -#define TK_FILL_HISTORY 200 -#define TK_UPDATE 201 -#define TK_SUBTABLE 202 -#define TK_KILL 203 -#define TK_CONNECTION 204 -#define TK_TRANSACTION 205 -#define TK_BALANCE 206 -#define TK_VGROUP 207 -#define TK_MERGE 208 -#define TK_REDISTRIBUTE 209 -#define TK_SPLIT 210 -#define TK_DELETE 211 -#define TK_INSERT 212 -#define TK_NULL 213 -#define TK_NK_QUESTION 214 -#define TK_NK_ARROW 215 -#define TK_ROWTS 216 -#define TK_QSTART 217 -#define TK_QEND 218 -#define TK_QDURATION 219 -#define TK_WSTART 220 -#define TK_WEND 221 -#define TK_WDURATION 222 -#define TK_IROWTS 223 -#define TK_ISFILLED 224 -#define TK_CAST 225 -#define TK_NOW 226 -#define TK_TODAY 227 -#define TK_TIMEZONE 228 -#define TK_CLIENT_VERSION 229 -#define TK_SERVER_VERSION 230 -#define TK_SERVER_STATUS 231 -#define TK_CURRENT_USER 232 -#define TK_CASE 233 -#define TK_WHEN 234 -#define TK_THEN 235 -#define TK_ELSE 236 -#define TK_BETWEEN 237 -#define TK_IS 238 -#define TK_NK_LT 239 -#define TK_NK_GT 240 -#define TK_NK_LE 241 -#define TK_NK_GE 242 -#define TK_NK_NE 243 -#define TK_MATCH 244 -#define TK_NMATCH 245 -#define TK_CONTAINS 246 -#define TK_IN 247 -#define TK_JOIN 248 -#define TK_INNER 249 -#define TK_SELECT 250 -#define TK_DISTINCT 251 -#define TK_WHERE 252 -#define TK_PARTITION 253 -#define TK_BY 254 -#define TK_SESSION 255 -#define TK_STATE_WINDOW 256 -#define TK_EVENT_WINDOW 257 -#define TK_SLIDING 258 -#define TK_FILL 259 -#define TK_VALUE 260 -#define TK_VALUE_F 261 -#define TK_NONE 262 -#define TK_PREV 263 -#define TK_NULL_F 264 -#define TK_LINEAR 265 -#define TK_NEXT 266 -#define TK_HAVING 267 -#define TK_RANGE 268 -#define TK_EVERY 269 -#define TK_ORDER 270 -#define TK_SLIMIT 271 -#define TK_SOFFSET 272 -#define TK_LIMIT 273 -#define TK_OFFSET 274 -#define TK_ASC 275 -#define TK_NULLS 276 -#define TK_ABORT 277 -#define TK_AFTER 278 -#define TK_ATTACH 279 -#define TK_BEFORE 280 -#define TK_BEGIN 281 -#define TK_BITAND 282 -#define TK_BITNOT 283 -#define TK_BITOR 284 -#define TK_BLOCKS 285 -#define TK_CHANGE 286 -#define TK_COMMA 287 -#define TK_CONCAT 288 -#define TK_CONFLICT 289 -#define TK_COPY 290 -#define TK_DEFERRED 291 -#define TK_DELIMITERS 292 -#define TK_DETACH 293 -#define TK_DIVIDE 294 -#define TK_DOT 295 -#define TK_EACH 296 -#define TK_FAIL 297 -#define TK_FILE 298 -#define TK_FOR 299 -#define TK_GLOB 300 -#define TK_ID 301 -#define TK_IMMEDIATE 302 -#define TK_IMPORT 303 -#define TK_INITIALLY 304 -#define TK_INSTEAD 305 -#define TK_ISNULL 306 -#define TK_KEY 307 -#define TK_MODULES 308 -#define TK_NK_BITNOT 309 -#define TK_NK_SEMI 310 -#define TK_NOTNULL 311 -#define TK_OF 312 -#define TK_PLUS 313 -#define TK_PRIVILEGE 314 -#define TK_RAISE 315 -#define TK_REPLACE 316 -#define TK_RESTRICT 317 -#define TK_ROW 318 -#define TK_SEMI 319 -#define TK_STAR 320 -#define TK_STATEMENT 321 -#define TK_STRICT 322 -#define TK_STRING 323 -#define TK_TIMES 324 -#define TK_VALUES 325 -#define TK_VARIABLE 326 -#define TK_VIEW 327 -#define TK_WAL 328 +#define TK_REPLACE 193 +#define TK_STREAM 194 +#define TK_INTO 195 +#define TK_TRIGGER 196 +#define TK_AT_ONCE 197 +#define TK_WINDOW_CLOSE 198 +#define TK_IGNORE 199 +#define TK_EXPIRED 200 +#define TK_FILL_HISTORY 201 +#define TK_UPDATE 202 +#define TK_SUBTABLE 203 +#define TK_KILL 204 +#define TK_CONNECTION 205 +#define TK_TRANSACTION 206 +#define TK_BALANCE 207 +#define TK_VGROUP 208 +#define TK_LEADER 209 +#define TK_MERGE 210 +#define TK_REDISTRIBUTE 211 +#define TK_SPLIT 212 +#define TK_DELETE 213 +#define TK_INSERT 214 +#define TK_NULL 215 +#define TK_NK_QUESTION 216 +#define TK_NK_ARROW 217 +#define TK_ROWTS 218 +#define TK_QSTART 219 +#define TK_QEND 220 +#define TK_QDURATION 221 +#define TK_WSTART 222 +#define TK_WEND 223 +#define TK_WDURATION 224 +#define TK_IROWTS 225 +#define TK_ISFILLED 226 +#define TK_CAST 227 +#define TK_NOW 228 +#define TK_TODAY 229 +#define TK_TIMEZONE 230 +#define TK_CLIENT_VERSION 231 +#define TK_SERVER_VERSION 232 +#define TK_SERVER_STATUS 233 +#define TK_CURRENT_USER 234 +#define TK_CASE 235 +#define TK_WHEN 236 +#define TK_THEN 237 +#define TK_ELSE 238 +#define TK_BETWEEN 239 +#define TK_IS 240 +#define TK_NK_LT 241 +#define TK_NK_GT 242 +#define TK_NK_LE 243 +#define TK_NK_GE 244 +#define TK_NK_NE 245 +#define TK_MATCH 246 +#define TK_NMATCH 247 +#define TK_CONTAINS 248 +#define TK_IN 249 +#define TK_JOIN 250 +#define TK_INNER 251 +#define TK_SELECT 252 +#define TK_DISTINCT 253 +#define TK_WHERE 254 +#define TK_PARTITION 255 +#define TK_BY 256 +#define TK_SESSION 257 +#define TK_STATE_WINDOW 258 +#define TK_EVENT_WINDOW 259 +#define TK_SLIDING 260 +#define TK_FILL 261 +#define TK_VALUE 262 +#define TK_VALUE_F 263 +#define TK_NONE 264 +#define TK_PREV 265 +#define TK_NULL_F 266 +#define TK_LINEAR 267 +#define TK_NEXT 268 +#define TK_HAVING 269 +#define TK_RANGE 270 +#define TK_EVERY 271 +#define TK_ORDER 272 +#define TK_SLIMIT 273 +#define TK_SOFFSET 274 +#define TK_LIMIT 275 +#define TK_OFFSET 276 +#define TK_ASC 277 +#define TK_NULLS 278 +#define TK_ABORT 279 +#define TK_AFTER 280 +#define TK_ATTACH 281 +#define TK_BEFORE 282 +#define TK_BEGIN 283 +#define TK_BITAND 284 +#define TK_BITNOT 285 +#define TK_BITOR 286 +#define TK_BLOCKS 287 +#define TK_CHANGE 288 +#define TK_COMMA 289 +#define TK_CONCAT 290 +#define TK_CONFLICT 291 +#define TK_COPY 292 +#define TK_DEFERRED 293 +#define TK_DELIMITERS 294 +#define TK_DETACH 295 +#define TK_DIVIDE 296 +#define TK_DOT 297 +#define TK_EACH 298 +#define TK_FAIL 299 +#define TK_FILE 300 +#define TK_FOR 301 +#define TK_GLOB 302 +#define TK_ID 303 +#define TK_IMMEDIATE 304 +#define TK_IMPORT 305 +#define TK_INITIALLY 306 +#define TK_INSTEAD 307 +#define TK_ISNULL 308 +#define TK_KEY 309 +#define TK_MODULES 310 +#define TK_NK_BITNOT 311 +#define TK_NK_SEMI 312 +#define TK_NOTNULL 313 +#define TK_OF 314 +#define TK_PLUS 315 +#define TK_PRIVILEGE 316 +#define TK_RAISE 317 +#define TK_RESTRICT 318 +#define TK_ROW 319 +#define TK_SEMI 320 +#define TK_STAR 321 +#define TK_STATEMENT 322 +#define TK_STRICT 323 +#define TK_STRING 324 +#define TK_TIMES 325 +#define TK_VALUES 326 +#define TK_VARIABLE 327 +#define TK_VIEW 328 +#define TK_WAL 329 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index cdb1642a5c313767fe6a6d5ad53d6202f8c677a6..6c3c7497b1f5b123143e99283657f55a49f2eb63 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tmsgcb.h" #include "trpc.h" +#include "sync.h" #ifdef __cplusplus extern "C" { @@ -33,8 +34,11 @@ typedef struct { bool deploy; int8_t selfIndex; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + int8_t numOfTotalReplicas; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int32_t nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SMsgCb msgCb; + int64_t lastIndex; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -69,6 +73,9 @@ int32_t mndStart(SMnode *pMnode); */ void mndStop(SMnode *pMnode); +int32_t mndIsCatchUp(SMnode *pMnode); +ESyncRole mndGetRole(SMnode *pMnode); + /** * @brief Get mnode monitor info. * diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index fbb24d28623d99932966e625d6499ebbc836ac0a..2c684f8f76a30b9700956c470de49687daedaeb9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -29,6 +29,7 @@ extern "C" { #include "tmsg.h" #include "tname.h" #include "transport.h" +#include "nodes.h" typedef struct SCatalog SCatalog; @@ -49,10 +50,15 @@ typedef enum { typedef struct SUserAuthInfo { char user[TSDB_USER_LEN]; - char dbFName[TSDB_DB_FNAME_LEN]; + SName tbName; AUTH_TYPE type; } SUserAuthInfo; +typedef struct SUserAuthRes { + bool pass; + SNode* pCond; +} SUserAuthRes; + typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; @@ -96,7 +102,7 @@ typedef struct SMetaData { SArray* pTableIndex; // pRes = SArray* SArray* pUdfList; // pRes = SFuncInfo* SArray* pIndex; // pRes = SIndexInfo* - SArray* pUser; // pRes = bool* + SArray* pUser; // pRes = SUserAuthRes* SArray* pQnodeList; // pRes = SArray* SArray* pTableCfg; // pRes = STableCfg* SArray* pDnodeList; // pRes = SArray* @@ -312,11 +318,9 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp); int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); -int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass); +int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, - bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ed7cbc812504ae09099abd79020c6b9d761d6674..ce7d038d42c8f37bec730111d6d26d946c47a168 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -29,7 +29,6 @@ extern "C" { #define DS_BUF_FULL 2 #define DS_BUF_EMPTY 3 -struct SDataSink; struct SSDataBlock; typedef struct SDeleterRes { diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index c3d20103512ae0ffa34e7dda4157912f184fb181..34372dc2ff23bacf0e21ee459025112c28b813bb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -26,6 +26,7 @@ extern "C" { typedef void* qTaskInfo_t; typedef void* DataSinkHandle; + struct SRpcMsg; struct SSubplan; @@ -67,9 +68,10 @@ typedef enum { * Create the exec task for stream mode * @param pMsg * @param SReadHandle + * @param vgId * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId); /** * Create the exec task for queue mode @@ -77,9 +79,22 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); * @param SReadHandle * @return */ -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, + uint64_t id); + +/** + * set the task Id, usually used by message queue process + * @param tinfo + * @param taskId + * @param queryId + */ +void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); + +// todo refactor +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId); + /** * Set multiple input data blocks for the stream scan. * @param tinfo @@ -108,7 +123,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -140,7 +155,6 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ - int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); @@ -154,6 +168,8 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); +int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode); + bool qTaskIsExecuting(qTaskInfo_t qinfo); /** @@ -162,14 +178,6 @@ bool qTaskIsExecuting(qTaskInfo_t qinfo); */ void qDestroyTask(qTaskInfo_t tinfo); -/** - * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. - * - * @param iter the table iterator to traverse all tables belongs to a super table, or an invert index - * @return - */ -int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); - void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/); @@ -179,28 +187,18 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); -/** - * return the scan info, in the form of tuple of two items, including table uid and current timestamp - * @param tinfo - * @param uid - * @param ts - * @return - */ -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); -// int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); +void qStreamSetOpen(qTaskInfo_t tinfo); -SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo); +SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index fb6ef26a8acf92c76cd022440a5b33ad3ca469e8..aa5c78195aff3bcb2b685921228cf843a7ea1e9a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -112,7 +112,7 @@ typedef struct SResultDataInfo { typedef struct SInputColumnInfoData { int32_t totalRows; // total rows in current columnar data int32_t startRowIndex; // handle started row index - int32_t numOfRows; // the number of rows needs to be handled + int64_t numOfRows; // the number of rows needs to be handled int32_t numOfInputCols; // PTS is not included bool colDataSMAIsSet; // if agg is set or not SColumnInfoData *pPTS; // primary timestamp column diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7b65c06b85f990b87c0c819f1d7627d8f28e8cc0..42bc89f0b71aeba488e0e27f899c754caf8f1df9 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -189,7 +189,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsVectorFunc(int32_t funcId); bool fmIsIndefiniteRowsFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); -bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsDateTimeFunc(int32_t funcId); bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); @@ -241,6 +241,7 @@ int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmSetInvertFunc(int32_t funcId, SFuncExecFuncs* pFpSet); int32_t fmSetNormalFunc(int32_t funcId, SFuncExecFuncs* pFpSet); bool fmIsInvertible(int32_t funcId); +char* fmGetFuncName(int32_t funcId); #ifdef __cplusplus } diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 1b1339340b1dc13d85740e953400ac6c0ee85f40..c825574fa6e3aea8936a3a9e5cd16224956a94a6 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -80,10 +80,6 @@ typedef struct SUdfInterBuf { } SUdfInterBuf; typedef void *UdfcFuncHandle; -// dynamic lib init and destroy -typedef int32_t (*TUdfInitFunc)(); -typedef int32_t (*TUdfDestroyFunc)(); - #define UDF_MEMORY_EXP_GROWTH 1.5 #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) @@ -153,6 +149,8 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne allocCapacity *= UDF_MEMORY_EXP_GROWTH; } + int32_t existedRows = data->numOfRows; + if (IS_VAR_DATA_TYPE(meta->type)) { char *tmp = (char *)realloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); if (tmp == NULL) { @@ -160,14 +158,20 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne } data->varLenCol.varOffsets = (int32_t *)tmp; data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; + memset(&data->varLenCol.varOffsets[existedRows], 0, sizeof(int32_t) * (allocCapacity - existedRows)); // for payload, add data in udfColDataAppend } else { char *tmp = (char *)realloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + uint32_t extend = BitmapLen(allocCapacity) - BitmapLen(data->rowsAlloc); + memset(tmp + BitmapLen(data->rowsAlloc), 0, extend); data->fixLenCol.nullBitmap = tmp; data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); + int32_t oldLen = BitmapLen(existedRows); + memset(&data->fixLenCol.nullBitmap[oldLen], 0, BitmapLen(allocCapacity) - oldLen); + if (meta->type == TSDB_DATA_TYPE_NULL) { return TSDB_CODE_SUCCESS; } @@ -194,6 +198,7 @@ static FORCE_INLINE void udfColDataSetNull(SUdfColumn *pColumn, int32_t row) { udfColDataSetNull_f(pColumn, row); } pColumn->hasNull = true; + pColumn->colData.numOfRows = ((int32_t)(row + 1) > pColumn->colData.numOfRows) ? (int32_t)(row + 1) : pColumn->colData.numOfRows; } static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentRow, const char *pData, bool isNull) { @@ -252,6 +257,10 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR return 0; } +// dynamic lib init and destroy for C UDF +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); + typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock *block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); @@ -259,6 +268,43 @@ typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock *block, SUdfInterBuf *interB typedef int32_t (*TUdfAggMergeFunc)(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf); typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf *buf, SUdfInterBuf *resultData); +////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +typedef struct SScriptUdfEnvItem { + const char *name; + const char *value; +} SScriptUdfEnvItem; + +typedef enum EUdfFuncType { UDF_FUNC_TYPE_SCALAR = 1, UDF_FUNC_TYPE_AGG = 2 } EUdfFuncType; + +typedef struct SScriptUdfInfo { + const char *name; + int32_t version; + int64_t createdTime; + + EUdfFuncType funcType; + int8_t scriptType; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + + const char *path; +} SScriptUdfInfo; + +typedef int32_t (*TScriptUdfScalarProcFunc)(SUdfDataBlock *block, SUdfColumn *resultCol, void *udfCtx); + +typedef int32_t (*TScriptUdfAggStartFunc)(SUdfInterBuf *buf, void *udfCtx); +typedef int32_t (*TScriptUdfAggProcessFunc)(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf, + void *udfCtx); +typedef int32_t (*TScriptUdfAggMergeFunc)(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf, + void *udfCtx); +typedef int32_t (*TScriptUdfAggFinishFunc)(SUdfInterBuf *buf, SUdfInterBuf *resultData, void *udfCtx); +typedef int32_t (*TScriptUdfInitFunc)(SScriptUdfInfo *info, void **pUdfCtx); +typedef int32_t (*TScriptUdfDestoryFunc)(void *udfCtx); + +// the following function is for open/close script plugin. +typedef int32_t (*TScriptOpenFunc)(SScriptUdfEnvItem *items, int numItems); +typedef int32_t (*TScriptCloseFunc)(); + #ifdef __cplusplus } #endif diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 3dee59ab14a0ac780f18f731a5abb6dbe559b6fd..607179c4464f157aca7d757403775b634b694d8b 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -116,6 +116,8 @@ typedef struct { int32_t vnodes_total; int32_t vnodes_alive; int32_t connections_total; + int32_t topics_toal; + int32_t streams_total; SArray *dnodes; // array of SMonDnodeDesc SArray *mnodes; // array of SMonMnodeDesc } SMonClusterInfo; @@ -147,9 +149,9 @@ typedef struct { } SMonStbInfo; typedef struct { - int32_t expire_time; - int64_t timeseries_used; - int64_t timeseries_total; + uint32_t expire_time; + int64_t timeseries_used; + int64_t timeseries_total; } SMonGrantInfo; typedef struct { diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index fefcc28bc29213d6aad0b9e9f93c9d76193cf4e4..2323d044ec4d2c79dad571682026f62f09631588 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -438,6 +438,7 @@ typedef struct SDropStreamStmt { typedef struct SCreateFunctionStmt { ENodeType type; + bool orReplace; bool ignoreExists; char funcName[TSDB_FUNC_NAME_LEN]; bool isAgg; @@ -457,7 +458,9 @@ typedef struct SGrantStmt { ENodeType type; char userName[TSDB_USER_LEN]; char objName[TSDB_DB_NAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; int64_t privileges; + SNode* pTagCond; } SGrantStmt; typedef SGrantStmt SRevokeStmt; @@ -466,6 +469,10 @@ typedef struct SBalanceVgroupStmt { ENodeType type; } SBalanceVgroupStmt; +typedef struct SBalanceVgroupLeaderStmt { + ENodeType type; +} SBalanceVgroupLeaderStmt; + typedef struct SMergeVgroupStmt { ENodeType type; int32_t vgId1; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index a2e4c76adccc91d5a1f8689643f3d7a93089eba0..3556a8ac9520bdfd7b58fa490c3b333aa034e9ed 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -209,7 +209,8 @@ typedef enum ENodeType { QUERY_NODE_INSERT_STMT, QUERY_NODE_QUERY, QUERY_NODE_SHOW_DB_ALIVE_STMT, - QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, + QUERY_NODE_BALANCE_VGROUP_LEADER_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1a9700907effa002ce4b2acf69196eb08fd7e6d9..480912a8cfd043eeeb985d88d034f2f342150766 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -298,6 +298,7 @@ typedef struct SSelectStmt { bool hasUniqueFunc; bool hasTailFunc; bool hasInterpFunc; + bool hasInterpPseudoColFunc; bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cb547ee6b3732e391ca943bcb9b3e9783d587e36..cfc6ef2025d87e5dfc8e025886c62b81cf9f7cbc 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -33,6 +33,7 @@ typedef enum { JOB_TASK_STATUS_INIT, JOB_TASK_STATUS_EXEC, JOB_TASK_STATUS_PART_SUCC, + JOB_TASK_STATUS_FETCH, JOB_TASK_STATUS_SUCC, JOB_TASK_STATUS_FAIL, JOB_TASK_STATUS_DROP, @@ -193,6 +194,7 @@ typedef struct SRequestConnInfo { typedef void (*__freeFunc)(void* param); +// todo add creator/destroyer function typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 2be0561ce7940e2c17dd49d71f62486d80538fc0..60ed94d4de5a9c43dc64ba2041073eb67889608e 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -106,6 +106,8 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId, void **pRsp, SArray *explainRes); +int32_t qWorkerDbgEnableDebug(char *option); + #ifdef __cplusplus } #endif diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 6b09bf4899ba0f444b80449c2a0d67df160b59ef..42a7261f3889f41befa4545471bbffd40d355f4e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -23,29 +23,29 @@ extern "C" { #ifndef _STREAM_STATE_H_ #define _STREAM_STATE_H_ -typedef struct SStreamTask SStreamTask; - typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); typedef struct STdbState { - SStreamTask* pOwner; - TDB* db; - TTB* pStateDb; - TTB* pFuncStateDb; - TTB* pFillStateDb; // todo refactor - TTB* pSessionStateDb; - TTB* pParNameDb; - TTB* pParTagDb; - TXN* txn; + struct SStreamTask* pOwner; + + TDB* db; + TTB* pStateDb; + TTB* pFuncStateDb; + TTB* pFillStateDb; // todo refactor + TTB* pSessionStateDb; + TTB* pParNameDb; + TTB* pParTagDb; + TXN* txn; } STdbState; // incremental state storage typedef struct { STdbState* pTdbState; int32_t number; + int64_t checkPointId; } SStreamState; -SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1d301623b1c325a7393a219140187aead37677f7..aade34e965a705bd201fcca59d341d152d989a7e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "executor.h" #include "os.h" +#include "executor.h" #include "query.h" #include "streamState.h" #include "tdatablock.h" @@ -50,6 +50,7 @@ enum { TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER2, + TASK_STATUS__RESTORE, // only available for source task to replay WAL from the checkpoint }; enum { @@ -103,21 +104,8 @@ typedef struct { int8_t type; } SStreamQueueItem; -#if 0 -typedef struct { - int8_t type; - int64_t ver; - int32_t* dataRef; - SSubmitReq* data; -} SStreamDataSubmit; - -typedef struct { - int8_t type; - int64_t ver; - SArray* dataRefs; // SArray - SArray* reqs; // SArray -} SStreamMergedSubmit; -#endif +typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); typedef struct { int8_t type; @@ -219,36 +207,20 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { } static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { - // return queue->qItem; } -static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { - int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); - if (dequeueFlag == STREAM_QUEUE__FAILED) { - ASSERT(queue->qItem != NULL); - return streamQueueCurItem(queue); - } else { - queue->qItem = NULL; - taosGetQitem(queue->qall, &queue->qItem); - if (queue->qItem == NULL) { - taosReadAllQitems(queue->queue, queue->qall); - taosGetQitem(queue->qall, &queue->qItem); - } - return streamQueueCurItem(queue); - } -} - -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); +void* streamQueueNextItem(SStreamQueue* queue); -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); typedef struct { - char* qmsg; - // followings are not applicable to encoder and decoder - void* executor; + char* qmsg; + void* pExecutor; // not applicable to encoder and decoder + struct SWalReader* pWalReader; // not applicable to encoder and decoder } STaskExec; typedef struct { @@ -263,16 +235,13 @@ typedef struct { SUseDbRsp dbInfo; } STaskDispatcherShuffle; -typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); - typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; - // not applicable to encoder and decoder - void* vnode; - FTbSink* tbSinkFunc; - STSchema* pTSchema; + void* vnode; // not available to encoder and decoder + FTbSink* tbSinkFunc; + STSchema* pTSchema; } STaskSinkTb; typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); @@ -295,24 +264,34 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -typedef struct SStreamTask { - int64_t streamId; - int32_t taskId; - int32_t totalLevel; - int8_t taskLevel; - int8_t outputType; - int16_t dispatchMsgType; +typedef struct SStreamId { + int64_t streamId; + int32_t taskId; + const char* idStr; +} SStreamId; +typedef struct SCheckpointInfo { + int64_t id; + int64_t version; // offset in WAL +} SCheckpointInfo; + +typedef struct SStreamStatus { int8_t taskStatus; int8_t schedStatus; - - // node info - int32_t selfChildId; - int32_t nodeId; - SEpSet epSet; - - int64_t recoverSnapVer; - int64_t startVer; +} SStreamStatus; + +struct SStreamTask { + SStreamId id; + int32_t totalLevel; + int8_t taskLevel; + int8_t outputType; + int16_t dispatchMsgType; + SStreamStatus status; + int32_t selfChildId; + int32_t nodeId; + SEpSet epSet; + SCheckpointInfo chkInfo; + STaskExec exec; // fill history int8_t fillHistory; @@ -322,9 +301,6 @@ typedef struct SStreamTask { int32_t nextCheckId; SArray* checkpointInfo; // SArray - // exec - STaskExec exec; - // output union { STaskDispatcherFixedEp fixedEpDispatcher; @@ -334,85 +310,53 @@ typedef struct SStreamTask { STaskSinkFetch fetchSink; }; - int8_t inputStatus; - int8_t outputStatus; - - // STaosQueue* inputQueue1; - // STaosQall* inputQall; + int8_t inputStatus; + int8_t outputStatus; SStreamQueue* inputQueue; SStreamQueue* outputQueue; // trigger - int8_t triggerStatus; - int64_t triggerParam; - void* timer; - - // msg handle - SMsgCb* pMsgCb; - - // state backend - SStreamState* pState; - - // do not serialize - int32_t recoverTryingDownstream; - int32_t recoverWaitingUpstream; - int64_t checkReqId; - SArray* checkReqIds; // shuffle - int32_t refCnt; - - int64_t checkpointingId; - int32_t checkpointAlignCnt; + int8_t triggerStatus; + int64_t triggerParam; + void* timer; + SMsgCb* pMsgCb; // msg handle + SStreamState* pState; // state backend + + // the followings attributes don't be serialized + int32_t recoverTryingDownstream; + int32_t recoverWaitingUpstream; + int64_t checkReqId; + SArray* checkReqIds; // shuffle + int32_t refCnt; + int64_t checkpointingId; + int32_t checkpointAlignCnt; + struct SStreamMeta* pMeta; +}; -} SStreamTask; +// meta +typedef struct SStreamMeta { + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int8_t walScan; +} SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewSStreamTask(int64_t streamId); -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); - -static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { - int8_t type = pItem->type; - if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); - if (pSubmitClone == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); - return -1; - } - qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, - pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); - taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - // qStreamInput(pTask->exec.executor, pSubmitClone); - } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || - type == STREAM_INPUT__REF_DATA_BLOCK) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__CHECKPOINT) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__GET_RES) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } - - if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { - atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); - } - -#if 0 - // TODO: back pressure - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); -#endif - return 0; -} - -static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); -} +SStreamTask* tNewStreamTask(int64_t streamId); +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); +void tFreeStreamTask(SStreamTask* pTask); +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem); +bool tInputQueueIsFull(const SStreamTask* pTask); typedef struct { SMsgHead head; @@ -587,8 +531,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); -int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); +// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); +void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); @@ -616,40 +561,22 @@ int32_t streamAggRecoverPrepare(SStreamTask* pTask); // int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); -// expand and deploy -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); - -// meta -typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - SHashObj* pRecoverStatus; - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; -} SStreamMeta; - SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); -// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); -int32_t streamMetaBegin(SStreamMeta* pMeta); -int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); -int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); +int32_t streamMetaBegin(SStreamMeta* pMeta); +int32_t streamMetaCommit(SStreamMeta* pMeta); +int32_t streamMetaRollBack(SStreamMeta* pMeta); +int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 189484d1a6c98f298c5e8e7910cacba12d309b39..e86a4f96905512dd08af6acc4b9563206b93f985 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -48,13 +48,15 @@ extern "C" { #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 #define SYNC_SNAP_RESEND_MS 1000 * 60 -#define SYNC_VND_COMMIT_MIN_MS 1000 +#define SYNC_VND_COMMIT_MIN_MS 3000 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 #define SYNC_TERM_INVALID -1 +#define SYNC_LEARNER_CATCHUP 10 + typedef enum { SYNC_STRATEGY_NO_SNAPSHOT = 0, SYNC_STRATEGY_STANDARD_SNAPSHOT = 1, @@ -76,19 +78,29 @@ typedef enum { TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, TAOS_SYNC_STATE_ERROR = 103, + TAOS_SYNC_STATE_LEARNER = 104, } ESyncState; +typedef enum { + TAOS_SYNC_ROLE_VOTER = 0, + TAOS_SYNC_ROLE_LEARNER = 1, + TAOS_SYNC_ROLE_ERROR = 2, +} ESyncRole; + typedef struct SNodeInfo { - int64_t clusterId; - int32_t nodeId; - uint16_t nodePort; - char nodeFqdn[TSDB_FQDN_LEN]; + int64_t clusterId; + int32_t nodeId; + uint16_t nodePort; + char nodeFqdn[TSDB_FQDN_LEN]; + ESyncRole nodeRole; } SNodeInfo; typedef struct SSyncCfg { + int32_t totalReplicaNum; int32_t replicaNum; int32_t myIndex; - SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncCfg; typedef struct SFsmCbMeta { @@ -143,10 +155,11 @@ typedef struct SSyncFSM { void* data; int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + SyncIndex (*FpAppliedIndexCb)(const struct SSyncFSM* pFsm); int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); + void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm, const SyncIndex commitIdx); void (*FpReConfigCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SReConfigCbMeta* pMeta); void (*FpLeaderTransferCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); bool (*FpApplyQueueEmptyCb)(const struct SSyncFSM* pFsm); @@ -154,6 +167,7 @@ typedef struct SSyncFSM { void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm); void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm); + void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm); int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); void (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot); @@ -235,6 +249,8 @@ void syncStop(int64_t rid); void syncPreStop(int64_t rid); void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncIsCatchUp(int64_t rid); +ESyncRole syncGetRole(int64_t rid); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); @@ -245,6 +261,7 @@ bool syncIsReadyForRead(int64_t rid); bool syncSnapshotSending(int64_t rid); bool syncSnapshotRecving(int64_t rid); int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); +int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 014ed518a397e97424f189527ac4ac2304288bc2..1e3974f5cc93bfef99a54173e92b14d6a1d8c689 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -138,7 +138,8 @@ typedef struct { int8_t enableRef; } SWalFilterCond; -typedef struct { +// todo hide this struct +typedef struct SWalReader { SWal *pWal; int64_t readerId; TdFilePtr pLogFile; @@ -146,8 +147,8 @@ typedef struct { int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; - int8_t curInvalid; - int8_t curStopped; +// int8_t curInvalid; +// int8_t curStopped; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it @@ -189,13 +190,15 @@ int32_t walApplyVer(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); -// read +// wal reader SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); +int64_t walReaderGetCurrentVer(const SWalReader* pReader); +void walReaderValidVersionRange(SWalReader* pReader, int64_t *sver, int64_t *ever); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); diff --git a/include/os/osDir.h b/include/os/osDir.h index 73871602c5e7818f0d9b050280d7326ee39678ef..55c7a15764cf68a15e37602d1182a56d859091b1 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -31,21 +31,49 @@ extern "C" { #endif +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #ifdef WINDOWS + #define TD_TMP_DIR_PATH "C:\\Windows\\Temp\\" +#ifdef CUS_NAME +#define TD_CFG_DIR_PATH "C:\\"CUS_NAME"\\cfg\\" +#define TD_DATA_DIR_PATH "C:\\"CUS_NAME"\\data\\" +#define TD_LOG_DIR_PATH "C:\\"CUS_NAME"\\log\\" +#else #define TD_CFG_DIR_PATH "C:\\TDengine\\cfg\\" #define TD_DATA_DIR_PATH "C:\\TDengine\\data\\" #define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" +#endif // CUS_NAME + #elif defined(_TD_DARWIN_64) + +#ifdef CUS_PROMPT +#define TD_TMP_DIR_PATH "/tmp/"CUS_PROMPT"d/" +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_TMP_DIR_PATH "/tmp/taosd/" #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT + #else + #define TD_TMP_DIR_PATH "/tmp/" +#ifdef CUS_PROMPT +#define TD_CFG_DIR_PATH "/etc/"CUS_PROMPT"/" +#define TD_DATA_DIR_PATH "/var/lib/"CUS_PROMPT"/" +#define TD_LOG_DIR_PATH "/var/log/"CUS_PROMPT"/" +#else #define TD_CFG_DIR_PATH "/etc/taos/" #define TD_DATA_DIR_PATH "/var/lib/taos/" #define TD_LOG_DIR_PATH "/var/log/taos/" +#endif // CUS_PROMPT #endif typedef struct TdDir *TdDirPtr; diff --git a/include/os/osTime.h b/include/os/osTime.h index 0a0a54119b4a40b79e54b7183b0cdc57cac03ca7..51a285a1393171ea59940498a4c8134afa270366 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -91,7 +91,7 @@ static FORCE_INLINE int64_t taosGetMonoTimestampMs() { } char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm); -struct tm *taosLocalTime(const time_t *timep, struct tm *result); +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf); struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst); time_t taosTime(time_t *t); time_t taosMktime(struct tm *timep); diff --git a/include/util/cus_name.h b/include/util/cus_name.h new file mode 100644 index 0000000000000000000000000000000000000000..16f677f855ba07403d292e4e64dd6060fb2ccf31 --- /dev/null +++ b/include/util/cus_name.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _CUS_NAME_H_ +#define _CUS_NAME_H_ + +#ifndef CUS_NAME +#define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT +#define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL +#define CUS_EMAIL "" +#endif + +#endif // _CUS_NAME_H_ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 5106196ccdc0b96903eeb10fc93e4d2d002fc9a5..b22ce30894e93bbda986c829c8567138c61d31ea 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -223,6 +223,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) // 2.x // #define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGUREDTAOS_DEF_ERROR_CODE(0, 0x033D) // 2.x // #define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGUREDTAOS_DEF_ERROR_CODE(0, 0x033E) // 2.x +#define TSDB_CODE_MND_DNODE_DIFF_CLUSTER TAOS_DEF_ERROR_CODE(0, 0x033F) // internal // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) @@ -240,6 +241,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) #define TSDB_CODE_MND_INVALID_ALTER_OPER TAOS_DEF_ERROR_CODE(0, 0x0356) #define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357) +#define TSDB_CODE_MND_USER_NOT_AVAILABLE TAOS_DEF_ERROR_CODE(0, 0x0358) +#define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359) // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) @@ -287,6 +290,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DB_ACCT TAOS_DEF_ERROR_CODE(0, 0x0389) // internal #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) +#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x // #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) // 2.x @@ -399,6 +403,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F) #define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410) #define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_MNODE_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0412) // internal +#define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x @@ -433,6 +439,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) #define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) +#define TSDB_CODE_VND_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0532) // internal +#define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -541,7 +549,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) #define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal -#define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) // +#define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) +#define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917) +#define TSDB_CODE_SYN_NEGO_WIN_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0X0918) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq @@ -641,7 +651,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634) #define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL TAOS_DEF_ERROR_CODE(0, 0x2635) #define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637) -#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) +#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_GROUP_BY TAOS_DEF_ERROR_CODE(0, 0x2638) #define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) #define TSDB_CODE_PAR_INVALID_DROP_STABLE TAOS_DEF_ERROR_CODE(0, 0x263A) #define TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE TAOS_DEF_ERROR_CODE(0, 0x263B) @@ -686,6 +696,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_NOT_SUPPORT_JOIN TAOS_DEF_ERROR_CODE(0, 0x2664) #define TSDB_CODE_PAR_INVALID_TAGS_PC TAOS_DEF_ERROR_CODE(0, 0x2665) #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY TAOS_DEF_ERROR_CODE(0, 0x2666) +#define TSDB_CODE_PAR_INVALID_OPTR_USAGE TAOS_DEF_ERROR_CODE(0, 0x2667) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner @@ -705,13 +716,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_PIPE_READ_ERR TAOS_DEF_ERROR_CODE(0, 0x2902) #define TSDB_CODE_UDF_PIPE_CONNECT_ERR TAOS_DEF_ERROR_CODE(0, 0x2903) -#define TSDB_CODE_UDF_PIPE_NO_PIPE TAOS_DEF_ERROR_CODE(0, 0x2904) +#define TSDB_CODE_UDF_PIPE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2904) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) -#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) -#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) -#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908) -#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2909) -#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x290A) +#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2906) +#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2907) +#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x2908) +#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909) +#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A) // sml #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) @@ -755,6 +766,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) +#define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/include/util/tarray.h b/include/util/tarray.h index a8510e4bc8295e7e40092ea5ae687717bda20676..4bf24b46b9d913c00d11b5ef53f8630f3ad4dbf5 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -69,14 +69,6 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles); */ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); -/** - * - * @param pArray - * @param comparFn - * @param fp - */ -void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); - /** * add all element from the source array list into the destination * @param pArray @@ -216,6 +208,8 @@ void taosArrayDestroyP(SArray* pArray, FDelete fp); void taosArrayDestroyEx(SArray* pArray, FDelete fp); +void taosArraySwap(SArray* a, SArray* b); + /** * sort the array * @param pArray @@ -252,14 +246,6 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); -/** - * swap array - * @param a - * @param b - * @return - */ -void taosArraySwap(SArray* a, SArray* b); - #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index f2da4096d3ed3457adc1bfa9ba733d8daf4e9dc9..664f22eeed7a2b167a2053739d78676afb71d9ac 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -198,11 +198,12 @@ typedef enum ELogicConditionType { #define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_PRIVILEDGE_CONDITION_LEN 200 #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 #define TSDB_FUNC_CODE_LEN 10 * 1024 * 1024 -#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_BUF_SIZE 4096 * 64 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 #define TSDB_FUNC_SCRIPT_BIN_LIB 0 @@ -248,7 +249,7 @@ typedef enum ELogicConditionType { #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 #define TSDB_USET_PASSWORD_LEN 129 -#define TSDB_VERSION_LEN 12 +#define TSDB_VERSION_LEN 32 #define TSDB_LABEL_LEN 8 #define TSDB_JOB_STATUS_LEN 32 @@ -284,8 +285,11 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_LEARNER_REPLICA 10 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 -#define TSDB_SYNC_LOG_BUFFER_RETENTION (TSDB_SYNC_LOG_BUFFER_SIZE >> 4) +#define TSDB_SYNC_LOG_BUFFER_RETENTION 256 +#define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 +#define TSDB_SYNC_NEGOTIATION_WIN 512 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta @@ -316,10 +320,10 @@ typedef enum ELogicConditionType { #define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 -#define TSDB_MAX_MINROWS_FBLOCK 1000 +#define TSDB_MAX_MINROWS_FBLOCK 1000000 #define TSDB_DEFAULT_MINROWS_FBLOCK 100 #define TSDB_MIN_MAXROWS_FBLOCK 200 -#define TSDB_MAX_MAXROWS_FBLOCK 10000 +#define TSDB_MAX_MAXROWS_FBLOCK 10000000 #define TSDB_DEFAULT_MAXROWS_FBLOCK 4096 #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond diff --git a/include/util/tlrucache.h b/include/util/tlrucache.h index 19009342484bce74c5173e7bf61ab52d6ddae767..c9cf71c2fd99398bd3f5e55bd08127f4278ed1f4 100644 --- a/include/util/tlrucache.h +++ b/include/util/tlrucache.h @@ -55,6 +55,8 @@ void *taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle); size_t taosLRUCacheGetUsage(SLRUCache *cache); size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache); +int32_t taosLRUCacheGetElems(SLRUCache *cache); + void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity); size_t taosLRUCacheGetCapacity(SLRUCache *cache); diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 1f6b205cdf8a2faa045a7a08b9008cb271a741a1..d05b5418b37d1506d52f7b211c3c92db53dabc85 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -61,7 +61,7 @@ typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); typedef struct STaosQnode STaosQnode; -typedef struct STaosQnode { +struct STaosQnode { STaosQnode *next; STaosQueue *queue; int64_t timestamp; @@ -70,9 +70,9 @@ typedef struct STaosQnode { int8_t itype; int8_t reserved[3]; char item[]; -} STaosQnode; +}; -typedef struct STaosQueue { +struct STaosQueue { STaosQnode *head; STaosQnode *tail; STaosQueue *next; // for queue set @@ -84,22 +84,22 @@ typedef struct STaosQueue { int64_t memOfItems; int32_t numOfItems; int64_t threadId; -} STaosQueue; +}; -typedef struct STaosQset { +struct STaosQset { STaosQueue *head; STaosQueue *current; TdThreadMutex mutex; tsem_t sem; int32_t numOfQueues; int32_t numOfItems; -} STaosQset; +}; -typedef struct STaosQall { +struct STaosQall { STaosQnode *current; STaosQnode *start; int32_t numOfItems; -} STaosQall; +}; STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); diff --git a/include/util/tsimplehash.h b/include/util/tsimplehash.h index c9df9114763549da1b88c551f9efcc421e0e5c14..987a9fe2a847549bd9eb645fc33b880ed9773186 100644 --- a/include/util/tsimplehash.h +++ b/include/util/tsimplehash.h @@ -48,6 +48,13 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn); */ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj); +/** + * set the free function pointer + * @param pHashObj + * @param freeFp + */ +void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp); + int32_t tSimpleHashPrint(const SSHashObj *pHashObj); /** diff --git a/include/util/version.h b/include/util/version.h index bac29e5baf6f7a754aa6d1b0bcfd2a06b841b0da..b241dd248b9a673591bbc94e1a71a78ad721919e 100644 --- a/include/util/version.h +++ b/include/util/version.h @@ -23,6 +23,7 @@ extern "C" { extern char version[]; extern char compatible_version[]; extern char gitinfo[]; +extern char gitinfoOfInternal[]; extern char buildinfo[]; #ifdef __cplusplus diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 3d3dfc8e7322ac11ca7bc30b95127c2d4590271d..2159899aa277f550adf76eb7c107bf3da17c9ab1 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -1,19 +1,18 @@ ######################################################## # # -# TDengine Configuration # -# Any questions, please email support@taosdata.com # +# Configuration # # # ######################################################## ######### 0. Client only configurations ############# -# The interval for TDengine CLI to send heartbeat to mnode +# The interval for CLI to send heartbeat to mnode # shellActivityTimer 3 ############### 1. Cluster End point ############################ -# The end point of the first dnode in the cluster to be connected to when this dnode or a TDengine CLI `taos` is started +# The end point of the first dnode in the cluster to be connected to when this dnode or the CLI utility is started # firstEp hostname:6030 # The end point of the second dnode to be connected to if the firstEp is not available @@ -25,7 +24,7 @@ # The FQDN of the host on which this dnode will be started. It can be IP address # fqdn hostname -# The port for external access after this dnode is started +# The port for external access after this dnode is started # serverPort 6030 # The maximum number of connections a dnode can accept @@ -40,10 +39,10 @@ # temporary file's directory, if you are using Windows platform please change to Windows path # tempDir /tmp/ -# Switch for allowing TDengine to collect and report service usage information +# Switch for allowing to collect and report service usage information # telemetryReporting 1 -# Switch for allowing TDengine to collect and report crash information +# Switch for allowing to collect and report crash information # crashReporting 1 # The maximum number of vnodes supported by this dnode @@ -96,7 +95,7 @@ # if free disk space is less than this value, this dnode will fail to start # minimalDataDirGB 2.0 -# enable/disable system monitor +# enable/disable system monitor # monitor 1 # The following parameter is used to limit the maximum number of lines in log files. @@ -114,8 +113,8 @@ # The following parameters are used for debug purpose only by this dnode. # debugFlag is a 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# Available debug levels are: -# 131: output warning and error +# Available debug levels are: +# 131: output warning and error # 135: output debug, warning and error # 143: output trace, debug, warning and error to log # 199: output debug, warning and error to both screen and file @@ -130,7 +129,7 @@ # debug flag for util # uDebugFlag 131 -# debug flag for rpc +# debug flag for rpc # rpcDebugFlag 131 # debug flag for jni @@ -139,7 +138,7 @@ # debug flag for query # qDebugFlag 131 -# debug flag for taosc driver +# debug flag for client driver # cDebugFlag 131 # debug flag for dnode messages diff --git a/packaging/cfg/taosd.service b/packaging/cfg/taosd.service index fff4b74e62a6da8f2bda9a6306a79132d7585e42..52c4b1d1e2d7aeb3447633353f48b2363c16efaa 100644 --- a/packaging/cfg/taosd.service +++ b/packaging/cfg/taosd.service @@ -1,5 +1,5 @@ [Unit] -Description=TDengine server service +Description=server service After=network-online.target Wants=network-online.target diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh index 3579f813e5b6ce91f0daa1fd230af14a4bf3d4b9..8fadffe4c68d05f9bb0e6f3ed160ead7d8bdb00d 100755 --- a/packaging/debRpmAutoInstall.sh +++ b/packaging/debRpmAutoInstall.sh @@ -1,15 +1,15 @@ #!/usr/bin/expect -set packgeName [lindex $argv 0] +set packageName [lindex $argv 0] set packageSuffix [lindex $argv 1] -set timeout 3 +set timeout 30 if { ${packageSuffix} == "deb" } { - spawn dpkg -i ${packgeName} + spawn dpkg -i ${packageName} } elseif { ${packageSuffix} == "rpm"} { - spawn rpm -ivh ${packgeName} + spawn rpm -ivh ${packageName} } expect "*one:" send "\r" expect "*skip:" send "\r" -expect eof \ No newline at end of file +expect eof diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 35bea0e65ccc5070fe9d4e82adadc7132ae7cc81..7d90beac1c9e12792a4d9a1de76c06b91157049f 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,7 +12,7 @@ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini ENV DEBIAN_FRONTEND=noninteractive WORKDIR /root/ -RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini +RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat curl gdb vim tmux less net-tools valgrind && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ LC_CTYPE=en_US.UTF-8 \ diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index e9d86219cb635d29cfc360314d122362d3285c0c..c21063e6a42a8642b0195640c1d7044be700c7da 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -90,45 +90,33 @@ cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepat cp -r %{_compiledir}/../examples/* %{buildroot}%{homepath}/examples if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then - mkdir -p %{buildroot}%{userlocalpath}/bin - mkdir -p %{buildroot}%{userlocalpath}/lib - mkdir -p %{buildroot}%{userlocalpath}/lib/pkgconfig - mkdir -p %{buildroot}%{userlocalpath}/include - mkdir -p %{buildroot}%{userlocalpath}/include/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share - mkdir -p %{buildroot}%{userlocalpath}/share/doc - mkdir -p %{buildroot}%{userlocalpath}/share/doc/jemalloc - mkdir -p %{buildroot}%{userlocalpath}/share/man - mkdir -p %{buildroot}%{userlocalpath}/share/man/man3 - - cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{userlocalpath}/bin/ + mkdir -p %{buildroot}%{homepath}/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/include/jemalloc/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/ ||: + mkdir -p %{buildroot}%{homepath}/jemalloc/lib/pkgconfig ||: + + cp %{_compiledir}/build/bin/jemalloc-config %{buildroot}%{homepath}/jemalloc/bin if [ -f %{_compiledir}/build/bin/jemalloc.sh ]; then - cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jemalloc.sh %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/bin/jeprof ]; then - cp %{_compiledir}/build/bin/jeprof %{buildroot}%{userlocalpath}/bin/ + cp %{_compiledir}/build/bin/jeprof %{buildroot}%{homepath}/jemalloc/bin fi if [ -f %{_compiledir}/build/include/jemalloc/jemalloc.h ]; then - cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{userlocalpath}/include/jemalloc/ + cp %{_compiledir}/build/include/jemalloc/jemalloc.h %{buildroot}%{homepath}/jemalloc/include/jemalloc/ fi if [ -f %{_compiledir}/build/lib/libjemalloc.so.2 ]; then - cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/ - ln -sf libjemalloc.so.2 %{buildroot}%{userlocalpath}/lib/libjemalloc.so + cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib + ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so fi if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then - cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then - cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{userlocalpath}/lib/ + cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib fi if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then - cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{userlocalpath}/lib/pkgconfig/ - fi - if [ -f %{_compiledir}/build/share/doc/jemalloc/jemalloc.html ]; then - cp %{_compiledir}/build/share/doc/jemalloc/jemalloc.html %{buildroot}%{userlocalpath}/share/doc/jemalloc/ - fi - if [ -f %{_compiledir}/build/share/man/man3/jemalloc.3 ]; then - cp %{_compiledir}/build/share/man/man3/jemalloc.3 %{buildroot}%{userlocalpath}/share/man/man3/ + cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi fi diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 78d5043b0cf664e02d840fe307b93dc9cc8d9a68..081383f89b358325f55cc1a7641015efcb0a4eed 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -25,7 +25,7 @@ sourcePath="nas" cpuType="x64" lite="true" packageType="tar" -subFile="taos.tar.gz" +subFile="package.tar.gz" while getopts "m:c:f:l:s:o:t:v:h" opt; do case $opt in m) @@ -79,9 +79,9 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' if [[ ${verMode} = "enterprise" ]];then - prePackag="TDengine-enterprise-${testFile}" + prePackage="TDengine-enterprise-${testFile}" elif [ ${verMode} = "community" ];then - prePackag="TDengine-${testFile}" + prePackage="TDengine-${testFile}" fi if [ ${lite} = "true" ];then packageLite="-Lite" @@ -92,10 +92,10 @@ if [[ "$packageType" = "tar" ]] ;then packageType="tar.gz" fi -tdPath="${prePackag}-${version}" -originTdpPath="${prePackag}-${originversion}" +tdPath="${prePackage}-${version}" +originTdpPath="${prePackage}-${originversion}" -packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" +packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" if [ "$testFile" == "server" ] ;then @@ -105,13 +105,13 @@ elif [ ${testFile} = "client" ];then elif [ ${testFile} = "tools" ];then tdPath="taosTools-${version}" originTdpPath="taosTools-${originversion}" - packgeName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" + packageName="${tdPath}-Linux-${cpuType}${packageLite}.${packageType}" originPackageName="${originTdpPath}-Linux-${cpuType}${packageLite}.${packageType}" installCmd="install-taostools.sh" fi -echo "tdPath:${tdPath},originTdpPath:${originTdpPath},packgeName:${packgeName},originPackageName:${originPackageName}" +echo "tdPath:${tdPath},originTdpPath:${originTdpPath},packageName:${packageName},originPackageName:${originPackageName}" function cmdInstall { command=$1 if command -v ${command} ;then @@ -206,7 +206,7 @@ else fi -if [[ ${packgeName} =~ "server" ]] ;then +if [[ ${packageName} =~ "server" ]] ;then echoColor BD " pkill -9 taosd " pkill -9 taosd fi @@ -232,25 +232,25 @@ if [ -d ${installPath}/${tdPath} ] ;then fi echoColor G "===== download installPackage =====" -cd ${installPath} && wgetFile ${packgeName} ${version} ${sourcePath} +cd ${installPath} && wgetFile ${packageName} ${version} ${sourcePath} cd ${oriInstallPath} && wgetFile ${originPackageName} ${originversion} ${sourcePath} cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . -packageSuffix=$(echo ${packgeName} | awk -F '.' '{print $NF}') +packageSuffix=$(echo ${packageName} | awk -F '.' '{print $NF}') if [ ! -f debRpmAutoInstall.sh ];then echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh - echo 'set packgeName [lindex $argv 0]' >> debRpmAutoInstall.sh + echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh - echo 'set timeout 3 ' >> debRpmAutoInstall.sh + echo 'set timeout 30 ' >> debRpmAutoInstall.sh echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh - echo ' spawn dpkg -i ${packgeName} ' >> debRpmAutoInstall.sh + echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh - echo ' spawn rpm -ivh ${packgeName}' >> debRpmAutoInstall.sh + echo ' spawn rpm -ivh ${packageName}' >> debRpmAutoInstall.sh echo '}' >> debRpmAutoInstall.sh echo 'expect "*one:"' >> debRpmAutoInstall.sh echo 'send "\r"' >> debRpmAutoInstall.sh @@ -261,25 +261,25 @@ fi echoColor G "===== instal Package =====" -if [[ ${packgeName} =~ "deb" ]];then +if [[ ${packageName} =~ "deb" ]];then cd ${installPath} dpkg -r taostools dpkg -r tdengine - if [[ ${packgeName} =~ "TDengine" ]];then - echoColor BD "./debRpmAutoInstall.sh ${packgeName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packgeName} ${packageSuffix} + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} else - echoColor BD "dpkg -i ${packgeName}" && dpkg -i ${packgeName} + echoColor BD "dpkg -i ${packageName}" && dpkg -i ${packageName} fi -elif [[ ${packgeName} =~ "rpm" ]];then +elif [[ ${packageName} =~ "rpm" ]];then cd ${installPath} sudo rpm -e tdengine sudo rpm -e taostools - if [[ ${packgeName} =~ "TDengine" ]];then - echoColor BD "./debRpmAutoInstall.sh ${packgeName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packgeName} ${packageSuffix} + if [[ ${packageName} =~ "TDengine" ]];then + echoColor BD "./debRpmAutoInstall.sh ${packageName} ${packageSuffix}" && chmod 755 debRpmAutoInstall.sh && ./debRpmAutoInstall.sh ${packageName} ${packageSuffix} else - echoColor BD "rpm -ivh ${packgeName}" && rpm -ivh ${packgeName} + echoColor BD "rpm -ivh ${packageName}" && rpm -ivh ${packageName} fi -elif [[ ${packgeName} =~ "tar" ]];then +elif [[ ${packageName} =~ "tar" ]];then echoColor G "===== check installPackage File of tar =====" cd ${oriInstallPath} if [ ! -f {originPackageName} ];then @@ -290,7 +290,7 @@ elif [[ ${packgeName} =~ "tar" ]];then echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} cd ${installPath} echoColor YD "unzip the new installation package" - echoColor BD "tar -xf ${packgeName}" && tar -xf ${packgeName} + echoColor BD "tar -xf ${packageName}" && tar -xf ${packageName} if [ ${testFile} != "tools" ] ;then cd ${installPath}/${tdPath} && tar xf ${subFile} @@ -326,7 +326,7 @@ fi cd ${installPath} -if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then +if [[ ${packageName} =~ "Lite" ]] || ([[ ${packageName} =~ "x64" ]] && [[ ${packageName} =~ "client" ]]) || ([[ ${packageName} =~ "deb" ]] && [[ ${packageName} =~ "server" ]]) || ([[ ${packageName} =~ "rpm" ]] && [[ ${packageName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then @@ -334,7 +334,7 @@ if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${pac tar xf taosTools-2.1.3-Linux-x64.tar.gz fi cd taosTools-2.1.3 && bash install-taostools.sh -elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then +elif ([[ ${packageName} =~ "arm64" ]] && [[ ${packageName} =~ "client" ]]);then echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then @@ -347,28 +347,28 @@ fi echoColor G "===== start TDengine =====" -if [[ ${packgeName} =~ "server" ]] ;then +if [[ ${packageName} =~ "server" ]] ;then echoColor BD " rm -rf /var/lib/taos/* && systemctl restart taosd " rm -rf /var/lib/taos/* systemctl restart taosd fi -rm -rf ${installPath}/${packgeName} +rm -rf ${installPath}/${packageName} rm -rf ${installPath}/${tdPath}/ -# if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then +# if ([[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "tar" ]]) || [[ ${packageName} =~ "client" ]] ;then # echoColor G "===== install taos-tools when package is lite or client =====" # cd ${installPath} # wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # tar xf taosTools-2.1.2-Linux-x64.tar.gz # cd taosTools-2.1.2 && bash install-taostools.sh -# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then +# elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "deb" ]] ;then # echoColor G "===== install taos-tools when package is lite or client =====" # cd ${installPath} # wgetFile taosTools-2.1.2-Linux-x64.tar.gz . # tar xf taosTools-2.1.2-Linux-x64.tar.gz # cd taosTools-2.1.2 && bash install-taostools.sh -# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then +# elif [[ ${packageName} =~ "Lite" ]] && [[ ${packageName} =~ "rpm" ]] ;then # echoColor G "===== install taos-tools when package is lite or client =====" # cd ${installPath} # wgetFile taosTools-2.1.2-Linux-x64.tar.gz . diff --git a/packaging/tools/com.taosdata.taoskeeper.plist b/packaging/tools/com.taosdata.taoskeeper.plist new file mode 100644 index 0000000000000000000000000000000000000000..338b5d8e79c4a6f529f70e85bdd2d8b64d67264f --- /dev/null +++ b/packaging/tools/com.taosdata.taoskeeper.plist @@ -0,0 +1,33 @@ + + + + + Label + com.tdengine.taoskeeper + ProgramArguments + + /usr/local/bin/taoskeeper + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taoskeeper + + \ No newline at end of file diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2495e177e1b0a8a20ecf6527862cc310c4ef34df..1b47b10520147664a3c2f3a558fed1208f84a2ca 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -4,7 +4,7 @@ # is required to use systemd to manage services at boot set -e -#set -x +# set -x verMode=edge pagMode=full @@ -24,7 +24,7 @@ productName="TDengine" emailName="taosdata.com" uninstallScript="rmtaos" historyFile="taos_history" -tarName="taos.tar.gz" +tarName="package.tar.gz" dataDir="/var/lib/taos" logDir="/var/log/taos" configDir="/etc/taos" @@ -34,21 +34,25 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" -explorerName="${clientName}-explorer" clientName2="taos" -serverName2="taosd" +serverName2="${clientName2}d" +configFile2="${clientName2}.cfg" productName2="TDengine" emailName2="taosdata.com" +xname2="${clientName2}x" +adapterName2="${clientName2}adapter" +explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" dumpName2="${clientName2}dump" uninstallScript2="rm${clientName2}" historyFile="${clientName2}_history" logDir="/var/log/${clientName2}" configDir="/etc/${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" data_dir=${dataDir} log_dir=${logDir} @@ -206,15 +210,15 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${serverName2} || : ${csudo}rm -f ${bin_link_dir}/${udfdName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/${demoName} || : - ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : - ${csudo}rm -f ${bin_link_dir}/${dumpName} || : - ${csudo}rm -f ${bin_link_dir}/${xname} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${xname2} || : ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -222,23 +226,22 @@ function install_bin() { ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -s ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : - [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : - [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -s ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : - [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${serverName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${serverName2} ${bin_link_dir}/${serverName2} || : + [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || : + [ -x ${install_main_dir}/bin/${adapterName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${adapterName2} ${bin_link_dir}/${adapterName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || : + [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : + [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : + if [ "$clientName2" == "${clientName}" ]; then + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + fi [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : - [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : - [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -250,14 +253,14 @@ function install_lib() { #${csudo}rm -rf ${v15_java_app_dir} || : ${csudo}cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* - ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then - ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : [ -f ${install_main_dir}/libtaosws.so ] && ${csudo}ln -sf ${install_main_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi @@ -347,10 +350,10 @@ function install_header() { [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + ${csudo}ln -sf ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -sf ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -sf ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -sf ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h [ -f ${install_main_dir}/include/taosws.h ] && ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : } @@ -399,7 +402,7 @@ function set_hostname() { ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : fi - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} serverFqdn=$newHostname if [[ -e /etc/hosts ]]; then @@ -433,7 +436,7 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn echo return @@ -455,7 +458,7 @@ function set_ipAsFqdn() { read -p "Please choose an IP from local IP list:" localFqdn else # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} serverFqdn=$localFqdn break fi @@ -511,7 +514,7 @@ function install_adapter_config() { fi [ -f ${cfg_install_dir}/${adapterName}.toml ] && - ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml + ${csudo}ln -sf ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml [ ! -z $1 ] && return 0 || : # only install client @@ -519,15 +522,15 @@ function install_adapter_config() { function install_config() { - if [ ! -f "${cfg_install_dir}/${configFile}" ]; then + if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir} + [ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} ${csudo}chmod 644 ${cfg_install_dir}/* else - ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new + ${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new fi - ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg + ${csudo}ln -sf ${cfg_install_dir}/${configFile2} ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client @@ -548,7 +551,7 @@ function install_config() { read firstEp while true; do if [ ! -z "$firstEp" ]; then - ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile} + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} break else break @@ -569,39 +572,57 @@ function install_config() { done } +function install_share_etc() { + [ ! -d ${script_dir}/share/etc ] && return + for c in `ls ${script_dir}/share/etc/`; do + if [ -e /etc/${clientName2}/$c ]; then + out=/etc/${clientName2}/$c.new.`date +%F` + ${csudo}cp -f ${script_dir}/share/etc/$c $out ||: + else + ${csudo}mkdir -p /etc/${clientName2} >/dev/null 2>/dev/null ||: + ${csudo}cp -f ${script_dir}/share/etc/$c /etc/${clientName2}/$c ||: + fi + done + + [ ! -d ${script_dir}/share/srv ] && return + ${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} ||: +} + function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - ${csudo}ln -s ${log_dir} ${install_main_dir}/log + ${csudo}ln -sf ${log_dir} ${install_main_dir}/log } function install_data() { ${csudo}mkdir -p ${data_dir} - ${csudo}ln -s ${data_dir} ${install_main_dir}/data + ${csudo}ln -sf ${data_dir} ${install_main_dir}/data } function install_connector() { - [ -d "${script_dir}/connector/" ] && ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ + if [ -d "${script_dir}/connector/" ]; then + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" + fi } function install_examples() { if [ -d ${script_dir}/examples ]; then - ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + ${csudo}cp -rf ${script_dir}/examples/* ${install_main_dir}/examples || echo "failed to copy examples" fi } function install_web() { if [ -d "${script_dir}/share" ]; then - ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share + ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||: fi } function clean_service_on_sysvinit() { - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - ${csudo}service ${serverName} stop || : + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then + ${csudo}service ${serverName2} stop || : fi if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then @@ -609,30 +630,30 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}chkconfig --del ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}chkconfig --del ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}chkconfig --del tarbitratord || : fi elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}insserv -r ${serverName} || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}insserv -r ${serverName2} || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}insserv -r tarbitratord || : fi elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/${serverName} ]; then - ${csudo}update-rc.d -f ${serverName} remove || : + if [ -e ${service_config_dir}/${serverName2} ]; then + ${csudo}update-rc.d -f ${serverName2} remove || : fi if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo}update-rc.d -f tarbitratord remove || : fi fi - ${csudo}rm -f ${service_config_dir}/${serverName} || : + ${csudo}rm -f ${service_config_dir}/${serverName2} || : ${csudo}rm -f ${service_config_dir}/tarbitratord || : if $(which init &>/dev/null); then @@ -653,24 +674,24 @@ function install_service_on_sysvinit() { fi if ((${initd_mod} == 1)); then - ${csudo}chkconfig --add ${serverName} || : - ${csudo}chkconfig --level 2345 ${serverName} on || : + ${csudo}chkconfig --add ${serverName2} || : + ${csudo}chkconfig --level 2345 ${serverName2} on || : elif ((${initd_mod} == 2)); then - ${csudo}insserv ${serverName} || : - ${csudo}insserv -d ${serverName} || : + ${csudo}insserv ${serverName2} || : + ${csudo}insserv -d ${serverName2} || : elif ((${initd_mod} == 3)); then - ${csudo}update-rc.d ${serverName} defaults || : + ${csudo}update-rc.d ${serverName2} defaults || : fi } function clean_service_on_systemd() { - taosd_service_config="${service_config_dir}/${serverName}.service" - if systemctl is-active --quiet ${serverName}; then + service_config="${service_config_dir}/${serverName2}.service" + if systemctl is-active --quiet ${serverName2}; then echo "${productName} is running, stopping it..." - ${csudo}systemctl stop ${serverName} &>/dev/null || echo &>/dev/null + ${csudo}systemctl stop ${serverName2} &>/dev/null || echo &>/dev/null fi - ${csudo}systemctl disable ${serverName} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${taosd_service_config} + ${csudo}systemctl disable ${serverName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${service_config} tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then @@ -682,24 +703,46 @@ function clean_service_on_systemd() { # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then # ${csudo}rm -f ${service_config_dir}/${serverName2}.service # fi + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function install_service_on_systemd() { clean_service_on_systemd - [ -f ${script_dir}/cfg/${serverName}.service ] && - ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + install_share_etc + + [ -f ${script_dir}/cfg/${serverName2}.service ] && + ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - # [ -f ${script_dir}/cfg/${serverName}.service ] && - # ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + # [ -f ${script_dir}/cfg/${serverName2}.service ] && + # ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ # ${service_config_dir}/${serverName2}.service || : # fi ${csudo}systemctl daemon-reload - ${csudo}systemctl enable ${serverName} + ${csudo}systemctl enable ${serverName2} ${csudo}systemctl daemon-reload } @@ -719,7 +762,7 @@ function install_service() { elif ((${service_mod} == 1)); then install_service_on_sysvinit else - kill_process ${serverName} + kill_process ${serverName2} fi } @@ -756,10 +799,10 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) else - min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 5) fi - exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + exist_version=$(${installDir}/bin/${serverName2} -V | head -1 | cut -d ' ' -f 3) vercomp $exist_version "3.0.0.0" case $? in 2) @@ -829,13 +872,13 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName2}...${NC}" # Stop the service if running - if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then if ((${service_mod} == 0)); then - ${csudo}systemctl stop ${serverName} || : + ${csudo}systemctl stop ${serverName2} || : elif ((${service_mod} == 1)); then - ${csudo}service ${serverName} stop || : + ${csudo}service ${serverName2} stop || : else - kill_process ${serverName} + kill_process ${serverName2} fi sleep 1 fi @@ -862,21 +905,21 @@ function updateProduct() { openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName} Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ./${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then @@ -887,7 +930,7 @@ function updateProduct() { if ((${prompt_force} == 1)); then echo "" - echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}" + echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" fi echo echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" @@ -899,7 +942,7 @@ function updateProduct() { echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function installProduct() { @@ -944,21 +987,21 @@ function installProduct() { # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" - [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}" + [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To configure ${clientName2} Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml" if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}systemctl start ${serverName2}${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" - [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${csudo}service ${serverName2} start${NC}" + [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}: ${serverName2}${NC}" + [ -f ${installDir}/bin/${clientName2}adapter ] && \ + echo -e "${GREEN_DARK}To start ${clientName2} Adapter ${NC}: ${clientName2}adapter &${NC}" fi if [ ! -z "$firstEp" ]; then @@ -970,28 +1013,28 @@ function installProduct() { tmpPort="" fi if [[ "$tmpPort" != "" ]]; then - echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" else - echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" fi echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access ${productName} ${NC}: ${clientName} -h $serverFqdn${GREEN_DARK} to login into ${productName} server${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}" echo fi - echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" + echo -e "\033[44;32;1m${productName2} is installed successfully!${NC}" echo else # Only install client install_bin install_config echo - echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}" fi touch ~/.${historyFile} - rm -rf $(tar -tf ${tarName} | grep -v "^\./$") + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } ## ==============================Main program starts from here============================ @@ -1002,7 +1045,7 @@ if [ "$verType" == "server" ]; then echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m" else # Install server and client - if [ -x ${bin_dir}/${serverName} ]; then + if [ -x ${bin_dir}/${serverName2} ]; then update_flag=1 updateProduct else @@ -1012,7 +1055,7 @@ if [ "$verType" == "server" ]; then elif [ "$verType" == "client" ]; then interactiveFqdn=no # Only install client - if [ -x ${bin_dir}/${clientName} ]; then + if [ -x ${bin_dir}/${clientName2} ]; then update_flag=1 updateProduct client else diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 1543c59297d663f0579d53f5c9778385cbef3ba8..53b9c80f10082a1aa70dc6aec54ea54b25be66f1 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -17,7 +17,7 @@ serverName="taosd" clientName="taos" uninstallScript="rmtaos" configFile="taos.cfg" -tarName="taos.tar.gz" +tarName="package.tar.gz" osType=Linux pagMode=full @@ -95,7 +95,7 @@ function install_main_path() { ${csudo}mkdir -p ${install_main_dir}/cfg ${csudo}mkdir -p ${install_main_dir}/bin ${csudo}mkdir -p ${install_main_dir}/driver - if [ $productName == "TDengine" ]; then + if [ "$productName2" == "TDengine" ]; then ${csudo}mkdir -p ${install_main_dir}/examples fi ${csudo}mkdir -p ${install_main_dir}/include @@ -118,18 +118,19 @@ function install_bin() { #Make link [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${clientName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName2} ${bin_link_dir}/${clientName2} || : if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : fi - [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : + [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : fi } @@ -305,7 +306,7 @@ function update_TDengine() { echo echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" - rm -rf $(tar -tf ${tarName}) + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function install_TDengine() { @@ -332,7 +333,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}" - rm -rf $(tar -tf ${tarName}) + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index a428c612b2607ea482f0a6ddc5b26636470b72b6..4ce2374b7fed8e9cdaf50c62231df8ae6a2032e4 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start com.tdengine.taosd -To start Taos Adapter : launchctl start com.tdengine.taosadapter -To access TDengine : use taos in shell +• To configure TDengine, edit /etc/taos/taos.cfg +• To start service, run launchctl start com.tdengine.taosd +• To start Taos Adapter, run launchctl start com.tdengine.taosadapter +• To access TDengine from your local machine, run taos If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. diff --git a/packaging/tools/mac_before_install_client.txt b/packaging/tools/mac_before_install_client.txt index 0457d73c49a0bdba49864477f32d07aaa512c375..cce8191667f7570591612944f611729e9e399c31 100644 --- a/packaging/tools/mac_before_install_client.txt +++ b/packaging/tools/mac_before_install_client.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -Once it's installed, please take the steps below: -1: open a terminal/shell in Mac -2: if connecting to Cloud Service, follow the instructions on your cloud service account and configure the environment variable -3: if connecting to another TDengine Service, you can also view help information via "taos --help" -4: execute command taos +After the installation process is complete, perform the following steps to start using TDengine: +1: Open Terminal on your Mac. +2: To connect to a TDengine server using the default settings and credentials, run the taos command. +3: To connect to a TDengine server using custom settings or credentials, run taos --help for more information. +4: To connect to TDengine Cloud, follow the instructions on the Tools - TDengine CLI page in your TDengine Cloud account. -If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. +If any issues occur during installation, check the /var/log/taos/tdengine_install.log file to troubleshoot. \ No newline at end of file diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 3996495aca6ea0c3367d38371838ffb688ca66e6..41113c9ae45ca73c9623bffff9b921fdf4525f3f 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -124,9 +124,18 @@ call :stop_delete call :check_svc taosd call :check_svc taosadapter -copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul -if exist C:\\TDengine\\driver\\taosws.dll ( - copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul +if exist c:\\windows\\sysnative ( + echo x86 + copy /y C:\\TDengine\\driver\\taos.dll %windir%\\sysnative > nul + if exist C:\\TDengine\\driver\\taosws.dll ( + copy /y C:\\TDengine\\driver\\taosws.dll %windir%\\sysnative > nul + ) +) else ( + echo x64 + copy /y C:\\TDengine\\driver\\taos.dll C:\\Windows\\System32 > nul + if exist C:\\TDengine\\driver\\taosws.dll ( + copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul + ) ) rem // create services diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 87c72c6a7c9985cffa13a112fbfbb5261d228632..98c5245cd39367203d7a8d8387b56846d1e19083 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -608,23 +608,23 @@ function update_TDengine() { echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi fi @@ -660,23 +660,23 @@ function install_TDengine() { echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi fi diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 208bfc183c91bba96be689fe6a475a5ae6e38649..1dbfc897bc7f11a5e5c302fbf08d73bc539808be 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -2,7 +2,7 @@ # # Generate tar.gz package for linux client in all os system set -e -# set -x +set -x curr_dir=$(pwd) compile_dir=$1 @@ -23,8 +23,11 @@ clientName2="${12}" productName="TDengine" clientName="taos" +benchmarkName="taosBenchmark" configFile="taos.cfg" -tarName="taos.tar.gz" +tarName="package.tar.gz" + +benchmarkName2="${clientName2}Benchmark" if [ "$osType" != "Darwin" ]; then script_dir="$(dirname $(readlink -f $0))" @@ -53,11 +56,12 @@ fi # Directories and files. -if [ "$verMode" == "cluster" ]; then - sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh - sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh - sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh -fi +#if [ "$verMode" == "cluster" ]; then +# sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh +# sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/configFile2=\"taos\"/configFile2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh +# sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh +#fi if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then @@ -66,6 +70,7 @@ if [ "$osType" != "Darwin" ]; then ${script_dir}/remove_client.sh" else bin_files="${build_dir}/bin/${clientName} \ + ${build_dir}/bin/${benchmarkName} \ ${script_dir}/remove_client.sh \ ${script_dir}/set_core.sh \ ${script_dir}/get_client.sh" @@ -153,6 +158,7 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_client_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh @@ -191,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 7ad3cf7b0aecf14bb984e98304bb78513bfb077b..a59083525714fa25724c5d8d93ffd799c0685091 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -28,7 +28,7 @@ productName="TDengine" serverName="taosd" clientName="taos" configFile="taos.cfg" -tarName="taos.tar.gz" +tarName="package.tar.gz" dumpName="taosdump" benchmarkName="taosBenchmark" toolsName="taostools" @@ -51,9 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - taostools_ver=$(git tag |grep -v taos | sort | tail -1) + taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) + [ -z "$taostools_ver" ] && taostools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} @@ -96,7 +96,7 @@ else ${taostools_bin_files} \ ${taosx_bin} \ ${explorer_bin_files} \ - ${build_dir}/bin/taosadapter \ + ${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ @@ -135,12 +135,12 @@ mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} -if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then - cp ${compile_dir}/test/cfg/taosadapter.toml ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.toml" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.toml ${install_dir}/cfg || : fi -if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then - cp ${compile_dir}/test/cfg/taosadapter.service ${install_dir}/cfg || : +if [ -f "${compile_dir}/test/cfg/${clientName}adapter.service" ]; then + cp ${compile_dir}/test/cfg/${clientName}adapter.service ${install_dir}/cfg || : fi if [ -f "${cfg_dir}/${serverName}.service" ]; then @@ -150,18 +150,19 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm +# mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: if [ $adapterName != "taosadapter" ]; then - mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml + mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml sed -i "s/path = \"\/var\/log\/taos\"/path = \"\/var\/log\/${productName}\"/g" ${install_dir}/cfg/$adapterName.toml sed -i "s/password = \"taosdata\"/password = \"${defaultPasswd}\"/g" ${install_dir}/cfg/$adapterName.toml - mv ${install_dir}/cfg/taosadapter.service ${install_dir}/cfg/$adapterName.service + mv ${install_dir}/cfg/${clientName2}adapter.service ${install_dir}/cfg/$adapterName.service sed -i "s/TDengine/${productName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosAdapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service - mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} + mv ${install_dir}/bin/${clientName2}adapter ${install_dir}/bin/${adapterName} mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb fi @@ -171,22 +172,22 @@ if [ -n "${taostools_bin_files}" ]; then && cp ${taostools_bin_files} ${taostools_install_dir}/bin \ && chmod a+x ${taostools_install_dir}/bin/* || : - if [ -f ${top_dir}/tools/taos-tools/packaging/tools/install-taostools.sh ]; then - cp ${top_dir}/tools/taos-tools/packaging/tools/install-taostools.sh \ + if [ -f ${top_dir}/tools/taos-tools/packaging/tools/install-tools.sh ]; then + cp ${top_dir}/tools/taos-tools/packaging/tools/install-tools.sh \ ${taostools_install_dir}/ > /dev/null \ - && chmod a+x ${taostools_install_dir}/install-taostools.sh \ - || echo -e "failed to copy install-taostools.sh" + && chmod a+x ${taostools_install_dir}/install-tools.sh \ + || echo -e "failed to copy install-tools.sh" else - echo -e "install-taostools.sh not found" + echo -e "install-tools.sh not found" fi - if [ -f ${top_dir}/tools/taos-tools/packaging/tools/uninstall-taostools.sh ]; then - cp ${top_dir}/tools/taos-tools/packaging/tools/uninstall-taostools.sh \ + if [ -f ${top_dir}/tools/taos-tools/packaging/tools/uninstall-tools.sh ]; then + cp ${top_dir}/tools/taos-tools/packaging/tools/uninstall-tools.sh \ ${taostools_install_dir}/ > /dev/null \ - && chmod a+x ${taostools_install_dir}/uninstall-taostools.sh \ - || echo -e "failed to copy uninstall-taostools.sh" + && chmod a+x ${taostools_install_dir}/uninstall-tools.sh \ + || echo -e "failed to copy uninstall-tools.sh" else - echo -e "uninstall-taostools.sh not found" + echo -e "uninstall-tools.sh not found" fi if [ -f ${build_dir}/lib/libavro.so.23.0.0 ]; then @@ -233,8 +234,10 @@ if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" remove_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi if [ "$verMode" == "cloud" ]; then @@ -262,8 +265,10 @@ if [ "$verMode" == "cluster" ]; then sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh + sed -i "s/configFile2=\"taos.cfg\"/configFile2=\"${clientName2}.cfg\"/g" install_temp.sh sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh - sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh + cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'` + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusDomain}\"/g" install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$verMode" == "cloud" ]; then @@ -317,6 +322,7 @@ if [[ $dbName == "taos" ]]; then mkdir -p ${install_dir}/share/ cp -Rfap ${web_dir}/admin ${install_dir}/share/ cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: else echo "directory not found for enterprise release: ${web_dir}/admin" fi @@ -332,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + tmp_pwd=`pwd` + cd ${install_dir}/connector + if [ ! -d taos-connector-jdbc ];then + git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + fi + cd taos-connector-jdbc + mvn clean package -Dmaven.test.skip=true + echo ${build_dir}/lib/ + cp target/*.jar ${build_dir}/lib/ + cd ${install_dir}/connector + rm -rf taos-connector-jdbc + cd ${tmp_pwd} + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index b2fc488a19bbf2c9425c4a48596326b2c7070dc2..3a013ade2cef2859151971d5da70ea4e0df6e681 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -145,6 +145,59 @@ function install_include() { log_print "install include success" } +function install_jemalloc() { + jemalloc_dir=${script_dir}/../jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo}/usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}/usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + + if [ -d /etc/ld.so.conf.d ]; then + echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" + ${csudo}ldconfig + else + echo "/etc/ld.so.conf.d not found!" + fi + fi +} + function install_lib() { log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : @@ -534,7 +587,7 @@ function install_service_on_sysvinit() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/taosd.service" - # taosd service already is stoped before install in preinst script + # taosd service already is stopped before install in preinst script #if systemctl is-active --quiet taosd; then # echo "TDengine is running, stopping it..." # ${csudo}systemctl stop taosd &> /dev/null || echo &> /dev/null @@ -586,6 +639,11 @@ function install_service_on_launchctl() { ${csudo}cp ${install_main_dir}/service/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : fi + if [ -f ${install_main_dir}/service/com.taosdata.taoskeeper.plist ]; then + ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || : + ${csudo}cp ${install_main_dir}/service/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : + ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist || : + fi } function install_taosadapter_service() { @@ -662,6 +720,7 @@ function install_TDengine() { # Install include, lib, binary and service install_include && install_lib && + install_jemalloc install_bin if [[ "$?" != 0 ]];then diff --git a/packaging/tools/release_note b/packaging/tools/release_note index 4578a4523c50f3b0764ce05add5920cf0be72172..81b09ba69c1485804e194a760e4aedde10014388 100644 --- a/packaging/tools/release_note +++ b/packaging/tools/release_note @@ -72,7 +72,7 @@ New Features: taos-1.4.13 (Released on 2018-12-14) Bugs Fixed: - - Clients failed to connect to server due to unexpected and invalid packets recieved by the server. + - Clients failed to connect to server due to unexpected and invalid packets received by the server. Features Added: - Add support to HikariCP in TSDB JDBC driver. diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 2479e48670bacb014c9209c8db11c8b0e09f5889..6c671473bfb5046ac18e7b813088ec6d6d3843d2 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -40,11 +40,16 @@ serverName2="taosd" clientName2="taos" productName2="TDengine" +adapterName2="${clientName2}adapter" +demoName2="${clientName2}demo" benchmarkName2="${clientName2}Benchmark" dumpName2="${clientName2}dump" +keeperName2="${clientName2}keeper" +xName2="${clientName2}x" +explorerName2="${clientName2}-explorer" uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -55,8 +60,8 @@ local_bin_link_dir="/usr/local/bin" service_config_dir="/etc/systemd/system" -taos_service_name=${serverName} -taosadapter_service_name="taosadapter" +taos_service_name=${serverName2} +taosadapter_service_name="${clientName2}adapter" tarbitrator_service_name="tarbitratord" csudo="" if command -v sudo >/dev/null; then @@ -84,14 +89,14 @@ else fi function kill_taosadapter() { - pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep "${adapterName2}" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi } function kill_taosd() { - pid=$(ps -ef | grep ${serverName} | grep -v "grep" | awk '{print $2}') + pid=$(ps -ef | grep ${serverName2} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : fi @@ -109,17 +114,17 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/udfd || : - ${csudo}rm -f ${bin_link_dir}/taosadapter || : - ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : + ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : - ${csudo}rm -f ${bin_link_dir}/taoskeeper || : - ${csudo}rm -f ${bin_link_dir}/taosx || : - ${csudo}rm -f ${bin_link_dir}/taos-explorer || : + ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : + ${csudo}rm -f ${bin_link_dir}/${xName2} || : + ${csudo}rm -f ${bin_link_dir}/${explorerName2} || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : @@ -130,8 +135,8 @@ function clean_bin() { } function clean_local_bin() { - ${csudo}rm -f ${local_bin_link_dir}/taosBenchmark || : - ${csudo}rm -f ${local_bin_link_dir}/taosdemo || : + ${csudo}rm -f ${local_bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${local_bin_link_dir}/${demoName2} || : } function clean_lib() { @@ -173,7 +178,7 @@ function clean_service_on_systemd() { ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null ${csudo}rm -f ${taosd_service_config} - taosadapter_service_config="${service_config_dir}/taosadapter.service" + taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then echo "${productName2} ${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -187,7 +192,27 @@ function clean_service_on_systemd() { ${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${tarbitratord_service_config} + + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function clean_service_on_sysvinit() { @@ -235,8 +260,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : - ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : - ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.${clientName2}adapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 10a0fb5e02a7fbc10b05765d4036e76b0bd1e4c8..2bdb56fac210982048d630469fbb10942bc15f69 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -15,11 +15,12 @@ uninstallScript="rmtaos" clientName2="taos" productName2="TDengine" -benchmarkName2="${clientName}Benchmark" -dumpName2="${clientName}dump" -uninstallScript2="rm${clientName}" +benchmarkName2="${clientName2}Benchmark" +demoName2="${clientName2}demo" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" -installDir="/usr/local/${clientName}" +installDir="/usr/local/${clientName2}" #install main path install_main_dir=${installDir} @@ -44,14 +45,17 @@ function kill_client() { function clean_bin() { # Remove link - ${csudo}rm -f ${bin_link_dir}/${clientName} || : - ${csudo}rm -f ${bin_link_dir}/taosdemo || : - ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/set_core || : if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${demoName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : fi diff --git a/packaging/tools/repair_link.sh b/packaging/tools/repair_link.sh index 7fd503f27013a9fce7208ece4335a1f427e05c9d..d71a16023eaabb3248283ed54a786a7cf0caf728 100755 --- a/packaging/tools/repair_link.sh +++ b/packaging/tools/repair_link.sh @@ -8,7 +8,7 @@ read -p "Please enter link directory such as /var/lib/taos/tsdb: " linkDir while true; do if [ ! -d $linkDir ]; then - read -p "Paht not exists, please enter the correct link path:" linkDir + read -p "Path not exists, please enter the correct link path:" linkDir continue fi break @@ -28,12 +28,12 @@ for linkFile in $(find -L $linkDir -xtype l); do if [ -z "${dirHash["$dirName"]}" ]; then read -p "Please enter the directory to replace ${dirName}:" newDir - read -p "Do you want to replcace all[y/N]?" replcaceAll - if [[ ( "${replcaceAll}" == "y") || ( "${replcaceAll}" == "Y") ]]; then + read -p "Do you want to replace all[y/N]?" replaceAll + if [[ ( "${replaceAll}" == "y") || ( "${replaceAll}" == "Y") ]]; then dirHash["$dirName"]="$newDir" fi fi - # Replcace the file + # Replace the file ln -sf "${newDir}/${baseName}" "${linkFile}" done diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 02f412f5d46fcc1ef2627c4dd6378775dd755995..8676fa2c51e891edc4077dfd2b6f9e5888af5f05 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -21,7 +21,7 @@ [Setup] VersionInfoVersion={#MyAppVersion} AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB} -AppName={#MyAppName} +AppName={#CusName} AppVersion={#MyAppVersion} ;AppVerName={#MyAppName} {#MyAppVersion} AppPublisher={#MyAppPublisher} @@ -64,8 +64,8 @@ Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt [run] -Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\{#CusName}\\taosd.exe --win_service""" ; Flags: runhidden -Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\{#CusName}\\taosadapter.exe""" ; Flags: runhidden +Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden +Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden diff --git a/packaging/tools/windows_before_install.txt b/packaging/tools/windows_before_install.txt index ef783bf10badedc733e87946a323d499b60fc5ac..91cb35d77e788d0705624da14d9e36c75403be3e 100644 --- a/packaging/tools/windows_before_install.txt +++ b/packaging/tools/windows_before_install.txt @@ -1,4 +1,4 @@ -TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. +TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. TDengine will be installed under C:\TDengine, users can modify configuration file C:\TDengine\cfg\taos.cfg, set the log file path or other parameters. To start/stop TDengine with administrator privileges: sc start/stop taosd diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index b10daa9c210ed1a39198a3cf1d0f59ab77be6bee..8e20d7d2752d143d59766ac6eca7887e90f08333 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -36,14 +36,6 @@ extern "C" { #include "tconfig.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - goto label; \ - } \ - } while (0) - #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms @@ -71,6 +63,7 @@ typedef struct { // statistics int32_t reportCnt; int32_t connKeyCnt; + int32_t passKeyCnt; // with passVer call back int64_t reportBytes; // not implemented int64_t startTime; // ctl @@ -133,6 +126,12 @@ typedef struct SAppInfo { TdThreadMutex mutex; } SAppInfo; +typedef struct { + int32_t ver; + void* param; + __taos_notify_fn_t fp; +} SPassInfo; + typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -148,6 +147,7 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; + SPassInfo passInfo; } STscObj; typedef struct STscDbg { @@ -286,25 +286,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { return (SReqResultInfo*)&msg->resInfo; } -static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* msg = (SMqRspObj*)res; - msg->resIter++; - if (msg->resIter < msg->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); - if (msg->rsp.withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter); - setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols); - taosMemoryFreeClear(msg->resInfo.row); - taosMemoryFreeClear(msg->resInfo.pCol); - taosMemoryFreeClear(msg->resInfo.length); - taosMemoryFreeClear(msg->resInfo.convertBuf); - taosMemoryFreeClear(msg->resInfo.convertJson); - } - setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false); - return &msg->resInfo; - } - return NULL; -} +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); @@ -317,7 +299,6 @@ extern int32_t clientConnRefPool; extern int32_t timestampDeltaLimit; extern int64_t lastClusterId; - __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); @@ -370,7 +351,6 @@ void taos_close_internal(void* taos); // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); -int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); @@ -381,10 +361,7 @@ void stopAllRequests(SHashObj* pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); - -// --- mq -void hbMgrInitMqHbRspHandle(); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* param); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index c29f495201d7b9ec0360abf6a5414798739b3da8..908e47083058b25bc0b6439e3e8888e1d2585328 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -26,6 +26,7 @@ extern "C" { #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarnL(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLongString("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index d8d41d8be6e75c32c7c5da06dbcfca5c508a8b62..92896e6f23ffd0b37e3387ac800417519f19dd6a 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -70,7 +70,7 @@ extern "C" { #define VALUE_LEN 6 #define OTD_JSON_FIELDS_NUM 4 -#define MAX_RETRY_TIMES 5 +#define MAX_RETRY_TIMES 100 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; typedef enum { diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 2b42de93e3f45ccb95c085c31825d4e0a117538c..cbef80b6daa63fd3192aae98cfb6170893381894 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -102,6 +102,7 @@ typedef struct STscStmt { SStmtBindInfo bInfo; int64_t reqid; + int32_t errCode; } STscStmt; extern char *gStmtStatusStr[]; @@ -121,6 +122,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ return _code; \ } \ } while (0) @@ -129,6 +131,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ } \ return _code; \ } while (0) @@ -137,9 +140,19 @@ extern char *gStmtStatusStr[]; code = c; \ if (code != TSDB_CODE_SUCCESS) { \ terrno = code; \ + pStmt->errCode = code; \ goto _return; \ } \ } while (0) +#define STMT_ERRI_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) + #define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) diff --git a/source/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/source/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index a88bf0f7a6bfd74bc92a82e57a44c5fc2957f6a8..2921afc3f827d233b2b4a6fd0d2886e97dfaf325 100644 --- a/source/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/source/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -75,6 +75,14 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_connectImp(JNIEn */ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp(JNIEnv *, jobject, jbyteArray, jlong); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: executeQueryWithReqId + * Signature: ([BJJ)I + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryWithReqId(JNIEnv *, jobject, jbyteArray, + jlong, jlong); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: getErrCodeImp @@ -186,6 +194,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTab */ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(JNIEnv *, jobject, jbyteArray, jlong); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: prepareStmtWithReqId + * Signature: ([BJJ)I + */ +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtWithReqId(JNIEnv *, jobject, jbyteArray, + jlong, jlong); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: setBindTableNameImp @@ -260,6 +276,32 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JN JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertImp(JNIEnv *, jobject, jobjectArray, jlong, jint, jint); +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithReqId(JNIEnv *, jobject, jlong, + jobjectArray, jint, jint, + jlong); + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithTtl(JNIEnv *, jobject, jlong, + jobjectArray, jint, jint, jint); + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithTtlAndReqId(JNIEnv *, jobject, + jlong, jobjectArray, + jint, jint, jint, + jlong); + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRaw(JNIEnv *, jobject, jlong, jstring, + jint, jint); + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithReqId(JNIEnv *, jobject, jlong, + jstring, jint, jint, + jlong); + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithTtl(JNIEnv *, jobject, jlong, + jstring, jint, jint, jint); + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithTtlAndReqId(JNIEnv *, jobject, + jlong, jstring, + jint, jint, jint, + jlong); /** * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: getTableVgID diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index e3e20ee85d4de996b664ccbaeed7514a992c9937..99569fdb574bd2860e8163c9a1ba0e2d30eb5750 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,6 +30,10 @@ #include "tsched.h" #include "ttime.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -103,6 +107,7 @@ static void deregisterRequest(SRequestObj *pRequest) { if (duration >= SLOW_QUERY_INTERVAL) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); + tscWarnL("slow query: %s, duration:%" PRId64, pRequest->sqlstr, duration); } releaseTscObj(pTscObj->id); @@ -234,7 +239,7 @@ void destroyTscObj(void *pObj) { tscTrace("begin to destroy tscObj %" PRIx64 " p:%p", tscId, pTscObj); SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; - hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); + hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, pTscObj->passInfo.fp); destroyAllRequests(pTscObj->pRequests); taosHashCleanup(pTscObj->pRequests); @@ -541,9 +546,15 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + char logDirName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logDirName, 64, "%slog", CUS_PROMPT); +#else + snprintf(logDirName, 64, "taoslog"); +#endif + if (taosCreateLog(logDirName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { // ignore create log failed, only print - printf(" WARING: Create taoslog failed. configDir=%s\n", configDir); + printf(" WARING: Create %s failed:%s. configDir=%s\n", logDirName, strerror(errno), configDir); } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index c9c2e7a5f82453035100f2e4a67d806c70a7ff57..79435da89fb5f85fa0fdace11e573e7c85c716b9 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -19,6 +19,15 @@ #include "scheduler.h" #include "trpc.h" +typedef struct { + union { + struct { + int64_t clusterId; + int32_t passKeyCnt; + }; + }; +} SHbParam; + static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); @@ -49,6 +58,52 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC return TSDB_CODE_SUCCESS; } +static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey, SAppHbMgr *pAppHbMgr) { + int32_t code = 0; + int32_t numOfBatchs = 0; + SUserPassBatchRsp batchRsp = {0}; + if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + + numOfBatchs = taosArrayGetSize(batchRsp.pArray); + + SClientHbReq *pReq = NULL; + while ((pReq = taosHashIterate(pAppHbMgr->activeInfo, pReq))) { + STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid); + if (!pTscObj) { + continue; + } + SPassInfo *passInfo = &pTscObj->passInfo; + if (!passInfo->fp) { + releaseTscObj(pReq->connKey.tscRid); + continue; + } + + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { + int32_t oldVer = atomic_load_32(&passInfo->ver); + if (oldVer < rsp->version) { + atomic_store_32(&passInfo->ver, rsp->version); + if (passInfo->fp) { + (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); + } + tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, rsp->user, oldVer, + atomic_load_32(&passInfo->ver), pTscObj->id); + } + break; + } + } + releaseTscObj(pReq->connKey.tscRid); + } + + taosArrayDestroy(batchRsp.pArray); + + return code; +} + static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { int32_t code = 0; SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); @@ -291,6 +346,15 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user pass info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey, pAppHbMgr); + break; + } default: tscError("invalid hb key type:%d", kv->key); break; @@ -472,6 +536,49 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { + STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); + if (!pTscObj) { + tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); + return TSDB_CODE_APP_ERROR; + } + + int32_t code = 0; + SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); + if (!user) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _return; + } + strncpy(user->user, pTscObj->user, TSDB_USER_LEN); + user->version = htonl(pTscObj->passInfo.ver); + + SKv kv = { + .key = HEARTBEAT_KEY_USER_PASSINFO, + .valueLen = sizeof(SUserPassVersion), + .value = user, + }; + + tscDebug("hb got user basic info, valueLen:%d, user:%s, passVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user, + pTscObj->passInfo.ver, connKey->tscRid); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + + if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { + code = terrno ? terrno : TSDB_CODE_APP_ERROR; + goto _return; + } + +_return: + releaseTscObj(connKey->tscRid); + if (code) { + tscError("hb got user basic info failed since %s", terrstr(code)); + } + + return code; +} + int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SUserAuthVersion *users = NULL; uint32_t userNum = 0; @@ -607,19 +714,23 @@ int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { } int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { - int64_t *clusterId = (int64_t *)param; + SHbParam *hbParam = (SHbParam *)param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(hbParam->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); return code; } - hbGetAppInfo(*clusterId, req); + hbGetAppInfo(hbParam->clusterId, req); hbGetQueryBasicInfo(connKey, req); + if (hbParam->passKeyCnt > 0) { + hbGetUserBasicInfo(connKey, req); + } + code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; @@ -673,7 +784,26 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); + SHbParam param; + switch (pOneReq->connKey.connType) { + case CONN_TYPE__QUERY: { + param.clusterId = pOneReq->clusterId; + param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); + break; + } + default: + break; + } + if (clientHbMgr.reqHandle[pOneReq->connKey.connType]) { + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); + if (code) { + tscWarn("hbGatherAllInfo failed since %s, tscRid:%" PRIi64 ", connType:%" PRIi8, tstrerror(code), + pOneReq->connKey.tscRid, pOneReq->connKey.connType); + } + } + break; + +#if 0 if (code) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; @@ -682,6 +812,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; +#endif } releaseTscObj(rid); @@ -1023,7 +1154,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } } -void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *param) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); @@ -1036,4 +1167,7 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); + if (param) { + atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1); + } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d5247d4b8aaeeddda1768ecc02b565e26da11446..ce174744ef740e13dffb4ab4a44cdf5a601b0f29 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1039,8 +1039,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .sysInfo = pRequest->pTscObj->sysInfo, .allocatorId = pRequest->allocatorRefId}; - SAppInstInfo* pAppInfo = getAppInfo(pRequest); - SQueryPlan* pDag = NULL; + SQueryPlan* pDag = NULL; int64_t st = taosGetTimestampUs(); int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); @@ -1052,7 +1051,6 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } pRequest->metric.execStart = taosGetTimestampUs(); - pRequest->metric.planCostUs = pRequest->metric.execStart - st; if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { @@ -1085,6 +1083,10 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + if (TSDB_CODE_SUCCESS != code) { + pRequest->code = terrno; + } + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } @@ -1132,11 +1134,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.queryFp(pRequest->body.param, pRequest, -1); break; } - - // TODO weired responding code? - if (TSDB_CODE_SUCCESS != code) { - pRequest->code = terrno; - } } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index cfa6f84bd242199a332dbc3da8675dd18b0407cd..b61335475125955b801b5b697992e79c98010431 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -55,7 +55,7 @@ jclass g_tmqClass; jmethodID g_createConsumerErrorCallback; jmethodID g_topicListCallback; -jclass g_consumerClass; +jclass g_consumerClass; // deprecated jmethodID g_commitCallback; @@ -331,13 +331,58 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryImp( int32_t code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, tscon, tstrerror(code), taos_errstr(tres)); + jniError("jobj:%p, conn:%p, code:0x%x, msg:%s", jobj, tscon, code, taos_errstr(tres)); } else { if (taos_is_update_query(tres)) { int32_t affectRows = taos_affected_rows(tres); - jniDebug("jobj:%p, conn:%p, code:%s, affect rows:%d", jobj, tscon, tstrerror(code), affectRows); + jniDebug("jobj:%p, conn:%p, code:0x%x, affect rows:%d", jobj, tscon, code, affectRows); } else { - jniDebug("jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniDebug("jobj:%p, conn:%p, code:0x%x", jobj, tscon, code); + } + } + + taosMemoryFreeClear(str); + return (jlong)tres; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeQueryWithReqId(JNIEnv *env, jobject jobj, + jbyteArray jsql, jlong con, + jlong reqId) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + if (jsql == NULL) { + jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); + return JNI_SQL_NULL; + } + + jsize len = (*env)->GetArrayLength(env, jsql); + + char *str = (char *)taosMemoryCalloc(1, sizeof(char) * (len + 1)); + if (str == NULL) { + jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); + return JNI_OUT_OF_MEMORY; + } + + (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); + if ((*env)->ExceptionCheck(env)) { + // todo handle error + } + + TAOS_RES *tres = taos_query_with_reqid(tscon, str, reqId); + int32_t code = taos_errno(tres); + + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code:0x%x, msg:%s", jobj, tscon, code, taos_errstr(tres)); + } else { + if (taos_is_update_query(tres)) { + int32_t affectRows = taos_affected_rows(tres); + jniDebug("jobj:%p, conn:%p, code:0x%x, affect rows:%d", jobj, tscon, code, affectRows); + } else { + jniDebug("jobj:%p, conn:%p, code:0x%x", jobj, tscon, code); } } @@ -489,7 +534,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn numOfFields); return JNI_FETCH_END; } else { - jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code, + jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: 0x%x, msg:%s", jobj, tscon, code, taos_errstr(result)); return JNI_RESULT_SET_NULL; } @@ -575,21 +620,21 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI TAOS_RES *tres = (TAOS_RES *)res; int32_t numOfFields = taos_num_fields(tres); - if(numOfFields <= 0){ - jniError("jobj:%p, conn:%p, query interrupted. taos_num_fields error code:%d, msg:%s", jobj, tscon, numOfFields, - taos_errstr(tres)); + if (numOfFields <= 0) { + jniError("jobj:%p, conn:%p, query interrupted. taos_num_fields error code: 0x%x, msg:%s", jobj, tscon, + taos_errno(tres), taos_errstr(tres)); return JNI_RESULT_SET_NULL; } - void *data; - int32_t numOfRows; + void *data = NULL; + int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(tres, &numOfRows, &data); if (numOfRows == 0) { if (error_code == JNI_SUCCESS) { jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code, + jniError("jobj:%p, conn:%p, query interrupted. fetch block error code: 0x%x, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); return JNI_RESULT_SET_NULL; } @@ -639,7 +684,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTab } int code = taos_validate_sql(tscon, str); - jniDebug("jobj:%p, conn:%p, code is %d", jobj, tscon, code); + jniDebug("jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); taosMemoryFreeClear(str); return code; @@ -704,7 +749,45 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtImp(J int32_t code = taos_stmt_prepare(pStmt, str, len); taosMemoryFreeClear(str); if (code != TSDB_CODE_SUCCESS) { - jniError("prepareStmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); + jniError("prepareStmt jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); + return JNI_TDENGINE_ERROR; + } + + return (jlong)pStmt; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_prepareStmtWithReqId(JNIEnv *env, jobject jobj, + jbyteArray jsql, jlong con, + jlong reqId) { + TAOS *tscon = (TAOS *)con; + if (tscon == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + if (jsql == NULL) { + jniError("jobj:%p, conn:%p, empty sql string", jobj, tscon); + return JNI_SQL_NULL; + } + + jsize len = (*env)->GetArrayLength(env, jsql); + + char *str = (char *)taosMemoryCalloc(1, sizeof(char) * (len + 1)); + if (str == NULL) { + jniError("jobj:%p, conn:%p, alloc memory failed", jobj, tscon); + return JNI_OUT_OF_MEMORY; + } + + (*env)->GetByteArrayRegion(env, jsql, 0, len, (jbyte *)str); + if ((*env)->ExceptionCheck(env)) { + // todo handle error + } + + TAOS_STMT *pStmt = taos_stmt_init_with_reqid(tscon, reqId); + int32_t code = taos_stmt_prepare(pStmt, str, len); + taosMemoryFreeClear(str); + if (code != TSDB_CODE_SUCCESS) { + jniError("prepareStmtWithReqId jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); return JNI_TDENGINE_ERROR; } @@ -732,8 +815,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI if (code != TSDB_CODE_SUCCESS) { (*env)->ReleaseStringUTFChars(env, jname, name); - jniError("bindTableName jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("bindTableName jobj:%p, conn:%p, code: 0x%x", jobj, tsconn, code); + return code; } jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); @@ -807,8 +890,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI (*env)->ReleaseStringUTFChars(env, tableName, name); if (code != TSDB_CODE_SUCCESS) { - jniError("tableNameTags jobj:%p, conn:%p, code:%s", jobj, tsconn, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("tableNameTags jobj:%p, conn:%p, code: 0x%x", jobj, tsconn, code); + return code; } return JNI_SUCCESS; } @@ -873,8 +956,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( taosMemoryFreeClear(b); if (code != TSDB_CODE_SUCCESS) { - jniError("bindColData jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("bindColData jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); + return code; } return JNI_SUCCESS; @@ -896,8 +979,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_addBatchImp(JNIEn int32_t code = taos_stmt_add_batch(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("add batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("add batch jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); + return code; } jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); @@ -920,8 +1003,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J int32_t code = taos_stmt_execute(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("excute batch jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("excute batch jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); + return code; } jniDebug("jobj:%p, conn:%p, batch execute", jobj, tscon); @@ -944,8 +1027,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv int32_t code = taos_stmt_close(pStmt); if (code != TSDB_CODE_SUCCESS) { - jniError("close stmt jobj:%p, conn:%p, code:%s", jobj, tscon, tstrerror(code)); - return JNI_TDENGINE_ERROR; + jniError("close stmt jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); + return code; } jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); @@ -1006,12 +1089,9 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JN TAOS_RES *tres = schemalessInsert(env, jobj, lines, taos, protocol, precision); - if (tres == NULL) { - return JNI_OUT_OF_MEMORY; - } int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { - jniError("jobj:%p, conn:%p, code:%s, msg:%s", jobj, taos, tstrerror(code), taos_errstr(tres)); + jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); taos_free_result(tres); return JNI_TDENGINE_ERROR; } @@ -1030,12 +1110,251 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsert } TAOS_RES *tres = schemalessInsert(env, jobj, lines, taos, protocol, precision); - if (tres == NULL) { + + return (jlong)tres; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithReqId( + JNIEnv *env, jobject jobj, jlong conn, jobjectArray lines, jint protocol, jint precision, jlong reqId) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + int numLines = (*env)->GetArrayLength(env, lines); + char **c_lines = taosMemoryCalloc(numLines, sizeof(char *)); + if (c_lines == NULL) { + jniError("c_lines:%p, alloc memory failed", c_lines); + return JNI_OUT_OF_MEMORY; + } + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); + } + + TAOS_RES *tres = taos_schemaless_insert_with_reqid(taos, c_lines, numLines, protocol, precision, reqId); + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); + } + + taosMemoryFreeClear(c_lines); + + return (jlong)tres; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithTtl(JNIEnv *env, jobject jobj, + jlong conn, jobjectArray lines, + jint protocol, jint precision, + jint ttl) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + int numLines = (*env)->GetArrayLength(env, lines); + char **c_lines = taosMemoryCalloc(numLines, sizeof(char *)); + if (c_lines == NULL) { + jniError("c_lines:%p, alloc memory failed", c_lines); + return JNI_OUT_OF_MEMORY; + } + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); + } + + TAOS_RES *tres = taos_schemaless_insert_ttl(taos, c_lines, numLines, protocol, precision, ttl); + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); + } + + taosMemoryFreeClear(c_lines); + + return (jlong)tres; +} + +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertWithTtlAndReqId( + JNIEnv *env, jobject jobj, jlong conn, jobjectArray lines, jint protocol, jint precision, jint ttl, jlong reqId) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + return JNI_CONNECTION_NULL; + } + + int numLines = (*env)->GetArrayLength(env, lines); + char **c_lines = taosMemoryCalloc(numLines, sizeof(char *)); + if (c_lines == NULL) { + jniError("c_lines:%p, alloc memory failed", c_lines); return JNI_OUT_OF_MEMORY; } + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + c_lines[i] = (char *)(*env)->GetStringUTFChars(env, line, 0); + } + + TAOS_RES *tres = taos_schemaless_insert_ttl_with_reqid(taos, c_lines, numLines, protocol, precision, ttl, reqId); + + for (int i = 0; i < numLines; ++i) { + jstring line = (jstring)((*env)->GetObjectArrayElement(env, lines, i)); + (*env)->ReleaseStringUTFChars(env, line, c_lines[i]); + } + + taosMemoryFreeClear(c_lines); + return (jlong)tres; } +JNIEXPORT jobject createSchemalessResp(JNIEnv *env, int totalRows, int code, const char *msg) { + // find class + jclass schemaless_clazz = (*env)->FindClass(env, "com/taosdata/jdbc/SchemalessResp"); + // find methods + jmethodID init_method = (*env)->GetMethodID(env, schemaless_clazz, "", "()V"); + jmethodID setCode_method = (*env)->GetMethodID(env, schemaless_clazz, "setCode", "(I)V"); + jmethodID setMsg_method = (*env)->GetMethodID(env, schemaless_clazz, "setMsg", "(Ljava/lang/String;)V"); + jmethodID setTotalRows_method = (*env)->GetMethodID(env, schemaless_clazz, "setTotalRows", "(I)V"); + // new schemaless + jobject schemaless_obj = (*env)->NewObject(env, schemaless_clazz, init_method); + // set code + (*env)->CallVoidMethod(env, schemaless_obj, setCode_method, code); + // set totalRows + (*env)->CallVoidMethod(env, schemaless_obj, setTotalRows_method, totalRows); + // set message + jstring message = (*env)->NewStringUTF(env, msg); + (*env)->CallVoidMethod(env, schemaless_obj, setMsg_method, message); + + return schemaless_obj; +} + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRaw(JNIEnv *env, jobject jobj, + jlong conn, jstring data, + jint protocol, jint precision) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + char *msg = "JNI connection is NULL"; + return createSchemalessResp(env, 0, JNI_CONNECTION_NULL, msg); + } + + char *line = (char *)(*env)->GetStringUTFChars(env, data, NULL); + jint len = (*env)->GetStringUTFLength(env, data); + int32_t totalRows; + TAOS_RES *tres = taos_schemaless_insert_raw(taos, line, len, &totalRows, protocol, precision); + + (*env)->ReleaseStringUTFChars(env, data, line); + + // if (tres == NULL) { + // jniError("jobj:%p, schemaless raw insert failed", jobj); + // char *msg = "JNI schemaless raw insert return null"; + // return createSchemalessResp(env, 0, JNI_TDENGINE_ERROR, msg); + // } + + int code = taos_errno(tres); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); + taos_free_result(tres); + return jobject; + } + taos_free_result(tres); + + return createSchemalessResp(env, totalRows, JNI_SUCCESS, NULL); +} + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithReqId( + JNIEnv *env, jobject jobj, jlong conn, jstring data, jint protocol, jint precision, jlong reqId) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + char *msg = "JNI connection is NULL"; + return createSchemalessResp(env, 0, JNI_CONNECTION_NULL, msg); + } + + char *line = (char *)(*env)->GetStringUTFChars(env, data, NULL); + jint len = (*env)->GetStringUTFLength(env, data); + int32_t totalRows; + TAOS_RES *tres = taos_schemaless_insert_raw_with_reqid(taos, line, len, &totalRows, protocol, precision, reqId); + + (*env)->ReleaseStringUTFChars(env, data, line); + + int code = taos_errno(tres); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); + taos_free_result(tres); + return jobject; + } + taos_free_result(tres); + + return createSchemalessResp(env, totalRows, JNI_SUCCESS, NULL); +} + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithTtl(JNIEnv *env, jobject jobj, + jlong conn, jstring data, + jint protocol, + jint precision, jint ttl) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + char *msg = "JNI connection is NULL"; + return createSchemalessResp(env, 0, JNI_CONNECTION_NULL, msg); + } + + char *line = (char *)(*env)->GetStringUTFChars(env, data, NULL); + jint len = (*env)->GetStringUTFLength(env, data); + int32_t totalRows; + TAOS_RES *tres = taos_schemaless_insert_raw_ttl(taos, line, len, &totalRows, protocol, precision, ttl); + + (*env)->ReleaseStringUTFChars(env, data, line); + + int code = taos_errno(tres); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); + taos_free_result(tres); + return jobject; + } + taos_free_result(tres); + + return createSchemalessResp(env, totalRows, JNI_SUCCESS, NULL); +} + +JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertRawWithTtlAndReqId( + JNIEnv *env, jobject jobj, jlong conn, jstring data, jint protocol, jint precision, jint ttl, jlong reqId) { + TAOS *taos = (TAOS *)conn; + if (taos == NULL) { + jniError("jobj:%p, connection already closed", jobj); + char *msg = "JNI connection is NULL"; + return createSchemalessResp(env, 0, JNI_CONNECTION_NULL, msg); + } + + char *line = (char *)(*env)->GetStringUTFChars(env, data, NULL); + jint len = (*env)->GetStringUTFLength(env, data); + int32_t totalRows; + TAOS_RES *tres = + taos_schemaless_insert_raw_ttl_with_reqid(taos, line, len, &totalRows, protocol, precision, ttl, reqId); + + (*env)->ReleaseStringUTFChars(env, data, line); + + int code = taos_errno(tres); + if (code != TSDB_CODE_SUCCESS) { + jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); + taos_free_result(tres); + return jobject; + } + taos_free_result(tres); + + return createSchemalessResp(env, totalRows, JNI_SUCCESS, NULL); +} + // TABLE_VG_ID_FID_CACHE cache resp object for getTableVgID typedef struct TABLE_VG_ID_FIELD_CACHE { int cached; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 63db9a679cdd0e312596b891a4a399db133aa00d..aed11b4fb153983e02d05d2d2d59f5d4a0b7a94d 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -119,6 +119,39 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } +int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) { + if (taos == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + STscObj *pObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + tscError("invalid parameter for %s", __func__); + return terrno; + } + + switch (type) { + case TAOS_NOTIFY_PASSVER: { + pObj->passInfo.fp = fp; + pObj->passInfo.param = param; + if (fp) { + atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1); + } + break; + } + default: { + terrno = TSDB_CODE_INVALID_PARA; + releaseTscObj(*(int64_t *)taos); + return terrno; + } + } + + releaseTscObj(*(int64_t *)taos); + return 0; +} + void taos_close_internal(void *taos) { if (taos == NULL) { return; @@ -271,8 +304,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SReqResultInfo *pResultInfo; if (msg->resIter == -1) { pResultInfo = tmqGetNextResInfo(res, true); - tscDebug("consumer:0x%" PRIx64 ", vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId, - msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows); } else { pResultInfo = tmqGetCurResInfo(res); } @@ -287,9 +318,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } - tscDebug("consumer:0x%" PRIx64 " vgId:%d, numOfRows:%" PRId64 ", total rows:%" PRId64, msg->rsp.head.consumerId, - msg->vgId, pResultInfo->numOfRows, pResultInfo->totalRows); - doSetOneRowPtr(pResultInfo); pResultInfo->current += 1; return pResultInfo->row; @@ -611,6 +639,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { } int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { + *numOfRows = 0; + *pData = NULL; + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 21590022b13663f20a28f35565c0f69cdccba9f3..52c5fc79408a05cf499d33059c6e7c07d176b615 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { goto End; } - if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 2)) != 0) { + if ((code = taosCheckVersionCompatibleFromStr(version, connectRsp.sVer, 3)) != 0) { setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; @@ -130,6 +130,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { lastClusterId = connectRsp.clusterId; pTscObj->connType = connectRsp.connType; + pTscObj->passInfo.ver = connectRsp.passVer; hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); @@ -506,7 +507,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true); } - if(code != 0){ + if (code != 0) { taosMemoryFree(pRes); } tFreeSShowVariablesRsp(&rsp); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a12f94cf1d88f3f52c958e20dfa130f98813da70..32f28e456319cd67a4808fa9f0ffefaedc1d3dbf 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -25,9 +25,7 @@ #include "tref.h" #include "ttimer.h" -static tb_uid_t processSuid(tb_uid_t suid, char* db){ - return suid + MurmurHash3_32(db, strlen(db)); -} +static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t) { @@ -194,6 +192,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { SDecoder coder; char* string = NULL; + uDebug("processCreateStb called"); // decode and process req void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); @@ -203,7 +202,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); - + uDebug("processCreateStb %s", string); _err: tDecoderClear(&coder); return string; @@ -213,6 +212,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; char* string = NULL; + uDebug("processAlterStb called"); // decode and process req void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -223,6 +223,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); + uDebug("processAlterStb %s", string); _err: tDecoderClear(&coder); @@ -346,6 +347,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { SVCreateTbReq* pCreateReq; char* string = NULL; // decode + uDebug("processCreateTable called"); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); @@ -362,6 +364,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); } + uDebug("processCreateTable :%s", string); } _exit: @@ -377,6 +380,7 @@ _exit: } static char* processAutoCreateTable(STaosxRsp* rsp) { + uDebug("processAutoCreateTable called"); if (rsp->createTableNum <= 0) { uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0"); goto _exit; @@ -402,7 +406,7 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { } } string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); - + uDebug("processAutoCreateTable :%s", string); _exit: for (int i = 0; i < rsp->createTableNum; i++) { tDecoderClear(&decoder[i]); @@ -422,6 +426,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { char* string = NULL; cJSON* json = NULL; + uDebug("processAlterTable called"); // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); @@ -527,6 +532,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { break; } string = cJSON_PrintUnformatted(json); + uDebug("processAlterTable :%s", string); _exit: cJSON_Delete(json); @@ -539,6 +545,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SVDropStbReq req = {0}; char* string = NULL; cJSON* json = NULL; + uDebug("processDropSTable called"); // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -560,7 +567,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "tableName", tableName); string = cJSON_PrintUnformatted(json); - + uDebug("processDropSTable :%s", string); _exit: cJSON_Delete(json); tDecoderClear(&decoder); @@ -573,6 +580,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { cJSON* json = NULL; char* string = NULL; + uDebug("processDeleteTable called"); // decode and process req void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); @@ -599,7 +607,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "sql", sqlJson); string = cJSON_PrintUnformatted(json); - + uDebug("processDeleteTable :%s", string); _exit: cJSON_Delete(json); tDecoderClear(&coder); @@ -612,6 +620,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { char* string = NULL; cJSON* json = NULL; + uDebug("processDropTable called"); // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); @@ -641,7 +650,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "tableNameList", tableNameList); string = cJSON_PrintUnformatted(json); - + uDebug("processDropTable :%s", string); _exit: cJSON_Delete(json); tDecoderClear(&decoder); @@ -655,6 +664,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; + uDebug("taosCreateStb called"); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -698,6 +708,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.source = TD_REQ_FROM_TAOX; pReq.igExists = true; + uDebug("taosCreateStb name:%s suid:%" PRId64 " processSuid:%" PRId64, req.name, req.suid, pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -744,6 +755,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; + uDebug("taosDropStb called"); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -763,11 +775,37 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; + SName pName = {0}; + toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); + STableMeta* pTableMeta = NULL; + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + code = TSDB_CODE_SUCCESS; + taosMemoryFreeClear(pTableMeta); + goto end; + } + if (code != TSDB_CODE_SUCCESS) { + uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", req.name); + goto end; + } + pReq.suid = pTableMeta->uid; + taosMemoryFreeClear(pTableMeta); + // build drop stable pReq.igNotExists = true; pReq.source = TD_REQ_FROM_TAOX; - pReq.suid = processSuid(req.suid, pRequest->pDb); + // pReq.suid = processSuid(req.suid, pRequest->pDb); + uDebug("taosDropStb name:%s suid:%" PRId64 " new suid:%" PRId64, req.name, req.suid, pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -792,7 +830,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { launchQueryImpl(pRequest, &pQuery, true, NULL); if (pRequest->code == TSDB_CODE_SUCCESS) { - SCatalog* pCatalog = NULL; + // SCatalog* pCatalog = NULL; catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); catalogRemoveTableMeta(pCatalog, &tableName); } @@ -825,6 +863,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { SQuery* pQuery = NULL; SHashObj* pVgroupHashmap = NULL; + uDebug("taosCreateTable called"); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -884,13 +923,17 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { if (pCreateReq->type == TSDB_CHILD_TABLE) { STableMeta* pTableMeta = NULL; SName sName = {0}; - pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); + tb_uid_t oldSuid = pCreateReq->ctb.suid; + // pCreateReq->ctb.suid = processSuid(pCreateReq->ctb.suid, pRequest->pDb); toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.stbName, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); if (code != TSDB_CODE_SUCCESS) { uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.stbName); goto end; } + pCreateReq->ctb.suid = pTableMeta->uid; + uDebug("taosCreateTable name:%s sname:%s suid:%" PRId64 " new suid:%" PRId64, pCreateReq->name, + pCreateReq->ctb.stbName, oldSuid, pCreateReq->ctb.suid); for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) { char* tName = taosArrayGet(pCreateReq->ctb.tagName, i); @@ -979,6 +1022,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { SQuery* pQuery = NULL; SHashObj* pVgroupHashmap = NULL; + uDebug("taosDropTable called"); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1022,7 +1066,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; - pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); + // pDropReq->suid = processSuid(pDropReq->suid, pRequest->pDb); SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1032,6 +1076,22 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + STableMeta* pTableMeta = NULL; + code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + code = TSDB_CODE_SUCCESS; + taosMemoryFreeClear(pTableMeta); + continue; + } + if (code != TSDB_CODE_SUCCESS) { + uError("taosDropTable:catalogGetTableMeta failed. table name: %s", pDropReq->name); + goto end; + } + tb_uid_t oldSuid = pDropReq->suid; + pDropReq->suid = pTableMeta->suid; + taosMemoryFreeClear(pTableMeta); + uDebug("taosDropTable name:%s suid:%" PRId64 " new suid:%" PRId64, pDropReq->name, oldSuid, pDropReq->suid); + taosArrayPush(pRequest->tableList, &pName); SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); if (pTableBatch == NULL) { @@ -1046,6 +1106,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } } + if (taosHashGetSize(pVgroupHashmap) == 0) { + goto end; + } SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); if (NULL == pBufArray) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1114,6 +1177,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { SDecoder coder = {0}; int32_t code = TSDB_CODE_SUCCESS; + uDebug("taosDeleteData called"); // decode and process req void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); @@ -1151,6 +1215,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { SArray* pArray = NULL; SVgDataBlocks* pVgData = NULL; + uDebug("taosAlterTable called"); code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { @@ -1196,6 +1261,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } + uDebug("taosAlterTable name:%s", req.tbName); pArray = taosArrayInit(1, sizeof(void*)); if (NULL == pArray) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1261,6 +1327,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; SHashObj* pVgHash = NULL; + uDebug("taos_write_raw_block_with_fields called"); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1280,6 +1347,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); tstrncpy(pName.tname, tbname, sizeof(pName.tname)); + uDebug("taos_write_raw_block_with_fields name:%s", tbname); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1305,6 +1373,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } + // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); + pQuery = smlInitHandle(); if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1312,6 +1382,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + // uError("td23101 1vgId:%d, numEps:%d, name:%s, uid:%"PRIu64, vgData.vgId, vgData.epSet.numOfEps, tbname, + // pTableMeta->uid); code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false); if (code != TSDB_CODE_SUCCESS) { @@ -1342,6 +1414,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SQuery* pQuery = NULL; SHashObj* pVgHash = NULL; + uDebug("taos_write_raw_block called"); SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { uError("WriteRaw:createRequest error request is null"); @@ -1360,6 +1433,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); tstrncpy(pName.tname, tbname, sizeof(pName.tname)); + uDebug("taos_write_raw_block name:%s", tbname); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1423,6 +1497,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { SMqRspObj rspObj = {0}; SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; + uDebug("tmqWriteRawDataImpl called"); terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); @@ -1436,7 +1511,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resType = RES_TYPE__TMQ; tDecoderInit(&decoder, data, dataLen); - code = tDecodeSMqDataRsp(&decoder, &rspObj.rsp); + code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); if (code != 0) { uError("WriteRaw:decode smqDataRsp error"); code = TSDB_CODE_INVALID_MSG; @@ -1468,7 +1543,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); + uDebug("tmqWriteRawDataImpl raw data block num:%d", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); if (!rspObj.rsp.withSchema) { @@ -1483,7 +1558,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - uDebug("raw data tbname:%s\n", tbName); + uDebug("tmqWriteRawDataImpl raw data tbname:%s", tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); @@ -1527,6 +1602,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uError("WriteRaw:rawBlockBindData failed"); goto end; } + taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); @@ -1539,7 +1615,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { code = pRequest->code; end: - tDeleteSMqDataRsp(&rspObj.rsp); + tDeleteMqDataRsp(&rspObj.rsp); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1556,6 +1632,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; SVCreateTbReq* pCreateReqDst = NULL; + uDebug("tmqWriteRawMetaDataImpl called"); terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); @@ -1602,7 +1679,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); + uDebug("tmqWriteRawMetaDataImpl raw data block num:%d", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); if (!rspObj.rsp.withSchema) { @@ -1617,7 +1694,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uDebug("raw data tbname:%s\n", tbName); + uDebug("tmqWriteRawMetaDataImpl raw data tbname:%s\n", tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); @@ -1632,6 +1709,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { tDecoderClear(&decoderTmp); + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); uError("WriteRaw: tDecodeSVCreateTbReq error"); code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -1640,21 +1718,30 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) if (pCreateReq.type != TSDB_CHILD_TABLE) { uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName); code = TSDB_CODE_TSC_INVALID_VALUE; + tDecoderClear(&decoderTmp); + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); goto end; } if (strcmp(tbName, pCreateReq.name) == 0) { cloneSVreateTbReq(&pCreateReq, &pCreateReqDst); - pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); + // pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); tDecoderClear(&decoderTmp); + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); break; } tDecoderClear(&decoderTmp); + tDestroySVCreateTbReq(&pCreateReq, TSDB_MSG_FLG_DECODE); } - if (pCreateReqDst) { + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; + } + + if (pCreateReqDst) { // change stable name to get meta strcpy(pName.tname, pCreateReqDst->ctb.stbName); - } else { - strcpy(pName.tname, tbName); } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { @@ -1667,16 +1754,10 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - SVgroupInfo vg; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - if (pCreateReqDst) { pTableMeta->vgId = vg.vgId; pTableMeta->uid = pCreateReqDst->uid; + pCreateReqDst->ctb.suid = pTableMeta->suid; } void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); if (hData == NULL) { @@ -1700,6 +1781,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } pCreateReqDst = NULL; + taosMemoryFreeClear(pTableMeta); } code = smlBuildOutput(pQuery, pVgHash); @@ -1726,6 +1808,7 @@ end: } char* tmq_get_json_meta(TAOS_RES* res) { + uDebug("tmq_get_json_meta called"); if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { return NULL; } @@ -1760,6 +1843,7 @@ char* tmq_get_json_meta(TAOS_RES* res) { void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { + uDebug("tmq_get_raw called"); if (!raw || !res) { return TSDB_CODE_INVALID_PARA; } @@ -1768,12 +1852,13 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw = pMetaRspObj->metaRsp.metaRsp; raw->raw_len = pMetaRspObj->metaRsp.metaRspLen; raw->raw_type = pMetaRspObj->metaRsp.resMsgType; + uDebug("tmq_get_raw meta"); } else if (TD_RES_TMQ(res)) { SMqRspObj* rspObj = ((SMqRspObj*)res); int32_t len = 0; int32_t code = 0; - tEncodeSize(tEncodeSMqDataRsp, &rspObj->rsp, len, code); + tEncodeSize(tEncodeMqDataRsp, &rspObj->rsp, len, code); if (code < 0) { return -1; } @@ -1781,12 +1866,13 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { void* buf = taosMemoryCalloc(1, len); SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeSMqDataRsp(&encoder, &rspObj->rsp); + tEncodeMqDataRsp(&encoder, &rspObj->rsp); tEncoderClear(&encoder); raw->raw = buf; raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ; + uDebug("tmq_get_raw data"); } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* rspObj = ((SMqTaosxRspObj*)res); @@ -1806,19 +1892,23 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw = buf; raw->raw_len = len; raw->raw_type = RES_TYPE__TMQ_METADATA; + uDebug("tmq_get_raw meta data"); } else { + uError("tmq_get_raw error:%d", *(int8_t*)res); return TSDB_CODE_TMQ_INVALID_MSG; } return TSDB_CODE_SUCCESS; } void tmq_free_raw(tmq_raw_data raw) { + uDebug("tmq_free_raw raw_type:%d", raw.raw_type); if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); } } int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { + uDebug("tmq_write_raw called"); if (!taos) { return TSDB_CODE_INVALID_PARA; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 32f748d4a9008fcf4e3b36a6d1da010c37e60156..17150286e1407e65a38e040877da7e053ca2e946 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -117,7 +117,7 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; - if (unit > INT64_MAX / tsInt64) { + if (tsInt64 != 0 && unit > INT64_MAX / tsInt64) { return -1; } tsInt64 *= unit; @@ -174,6 +174,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { memset(childTableName, 0, TSDB_TABLE_NAME_LEN); strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + taosArrayRemove(tags, i); break; } } @@ -256,20 +257,15 @@ cleanup: kvVal->type = TSDB_DATA_TYPE_FLOAT; \ kvVal->f = (float)result; -#define SET_BIGINT \ - if (smlDoubleToInt64OverFlow(result)) { \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = (int64_t)result; +#define SET_BIGINT \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; #define SET_INT \ if (!IS_VALID_INT(result)) { \ @@ -287,20 +283,15 @@ cleanup: kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ kvVal->i = result; -#define SET_UBIGINT \ - if (result >= (double)UINT64_MAX || result < 0) { \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = result; +#define SET_UBIGINT \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; #define SET_UINT \ if (!IS_VALID_UINT(result)) { \ @@ -543,8 +534,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); + uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -637,7 +627,10 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO for (int j = 0; j < taosArrayGetSize(cols); ++j) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); ESchemaAction action = SCHEMA_ACTION_NULL; - smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + int code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + if (code != 0) { + return code; + } if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { SField field = {0}; field.type = kv->type; @@ -646,7 +639,11 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO taosArrayPush(results, &field); } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; + if (index == NULL) { + uError("smlBuildFieldsList get error, key:%s", kv->key); + return TSDB_CODE_SML_INVALID_DATA; + } + uint16_t newIndex = *index; if (isTag) newIndex -= numOfCols; SField *field = (SField *)taosArrayGet(results, newIndex); field->bytes = getBytes(kv->type, kv->length); @@ -702,7 +699,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = 1; SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; + field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; strcpy(field.name, tsSmlTagName); taosArrayPush(pReq.pTags, &field); } @@ -743,6 +740,8 @@ end: } static int32_t smlModifyDBSchemas(SSmlHandle *info) { + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat, + info->needModifySchema); if (info->dataFormat && !info->needModifySchema) { return TSDB_CODE_SUCCESS; } @@ -772,11 +771,19 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - + code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); + goto end; + } + code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); + goto end; + } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); @@ -804,6 +811,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, + action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); SArray *pTags = @@ -820,8 +829,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { taosArrayPush(pTags, &field); } } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); + code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); + goto end; + } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { @@ -851,6 +864,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } if (action != SCHEMA_ACTION_NULL) { + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, + action); SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); SArray *pTags = @@ -868,8 +883,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); + code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); + goto end; + } code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { @@ -913,15 +932,21 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { } sTableData->tableMeta = pTableMeta; - - tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); + uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid, + pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } + uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat, + info->needModifySchema); + return 0; end: taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); - // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code, + tstrerror(code), info->dataFormat, info->needModifySchema); + return code; } @@ -997,8 +1022,9 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols } else { size_t tmp = taosArrayGetSize(metaArray); if (tmp > INT16_MAX) { + smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key); uError("too many cols or tags"); - return -1; + return TSDB_CODE_SML_INVALID_DATA; } int16_t size = tmp; int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); @@ -1097,6 +1123,9 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) { } if (taos != NULL) { info->taos = acquireTscObj(*(int64_t *)taos); + if (info->taos == NULL) { + goto cleanup; + } code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); @@ -1137,7 +1166,9 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { for (size_t i = 0; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - if(terrno == TSDB_CODE_DUP_KEY){return terrno;} + if (terrno == TSDB_CODE_DUP_KEY) { + return terrno; + } } taosArrayPush(colsArray, &kvHash); @@ -1145,22 +1176,25 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) { } static int32_t smlParseLineBottom(SSmlHandle *info) { + uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat, + info->lineNum); if (info->dataFormat) return TSDB_CODE_SUCCESS; for (int32_t i = 0; i < info->lineNum; i++) { SSmlLineInfo *elements = info->lines + i; SSmlTableInfo *tinfo = NULL; if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); - if(tmp) tinfo = *tmp; + SSmlTableInfo **tmp = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); + if (tmp) tinfo = *tmp; } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, - elements->measureLen + elements->tagsLen); - if(tmp) tinfo = *tmp; + SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, + elements->measureLen + elements->tagsLen); + if (tmp) tinfo = *tmp; } else { - SSmlTableInfo** tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, - elements->measureLen + elements->tagsLen); - if(tmp) tinfo = *tmp; + SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, + elements->measureLen + elements->tagsLen); + if (tmp) tinfo = *tmp; } if (tinfo == NULL) { @@ -1187,6 +1221,8 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); if (tableMeta) { // update meta + uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat, + info->lineNum); ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf); if (ret == TSDB_CODE_SUCCESS) { ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf); @@ -1201,20 +1237,32 @@ static int32_t smlParseLineBottom(SSmlHandle *info) { // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); // return ret; // } - + uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, + info->lineNum); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); - if(terrno == TSDB_CODE_DUP_KEY){return terrno;} + if (terrno == TSDB_CODE_DUP_KEY) { + return terrno; + } smlInsertMeta(meta->colHash, meta->cols, elements->colArray); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); } } + uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum); return TSDB_CODE_SUCCESS; } static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; + uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat); + + if (info->pRequest->dbList == NULL) { + info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN); + } + void *data = taosArrayReserve(info->pRequest->dbList, 1); + memcpy(data, info->pRequest->pDb, + TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN); SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); while (oneTable) { @@ -1224,6 +1272,11 @@ static int32_t smlInsertData(SSmlHandle *info) { tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); + if (info->pRequest->tableList == NULL) { + info->pRequest->tableList = taosArrayInit(1, sizeof(SName)); + } + taosArrayPush(info->pRequest->tableList, &pName); + SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; conn.requestId = info->pRequest->requestId; @@ -1248,6 +1301,8 @@ static int32_t smlInsertData(SSmlHandle *info) { // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname, + tableData->uid, info->dataFormat); code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, @@ -1270,20 +1325,23 @@ static int32_t smlInsertData(SSmlHandle *info) { atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code, + tstrerror(info->pRequest->code)); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { uDebug( "SML:0x%" PRIx64 - " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ + " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 "", - info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, - info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, - info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, - info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime, - info->cost.endTime - info->cost.parseTime); + info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables, + info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, + info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, + info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, + info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } int32_t smlClearForRerun(SSmlHandle *info) { @@ -1324,6 +1382,7 @@ int32_t smlClearForRerun(SSmlHandle *info) { } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { + uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id); int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { if (lines) { @@ -1359,8 +1418,17 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } - uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, - (info->isRawLine ? "rawdata" : tmp)); + char cTmp = 0; // for print tmp if is raw + if (info->isRawLine) { + cTmp = tmp[len - 1]; + tmp[len - 1] = '\0'; + } + + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id, + info->isRawLine, numLines, info->protocol, len, tmp); + if (info->isRawLine) { + tmp[len - 1] = cTmp; + } if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->dataFormat) { @@ -1385,6 +1453,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } if (info->reRun) { + uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id); i = 0; rawLine = oldRaw; code = smlClearForRerun(info); @@ -1395,6 +1464,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } i++; } + uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id); return code; } @@ -1425,6 +1495,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); if (code == 0) break; + taosMsleep(500); + uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { @@ -1449,62 +1521,83 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } + SRequestObj *request = NULL; + SSmlHandle *info = NULL; + int cnt = 0; + while (1) { + request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (request == NULL) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } + info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; + + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + if (request->pDb == NULL) { + request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); + goto end; + } - SSmlHandle *info = smlBuildSmlInfo(taos); - if (info == NULL) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - return (TAOS_RES *)request; - } - info->pRequest = request; - info->isRawLine = rawLine != NULL; - info->ttl = ttl; - info->precision = precision; - info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - info->lineNum = numLines; - - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - if (request->pDb == NULL) { - request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); - goto end; - } + if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { + request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; + smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); + goto end; + } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { - request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; - smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); - goto end; - } + if (protocol == TSDB_SML_LINE_PROTOCOL && + (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) { + request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; + smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); + goto end; + } - if (protocol == TSDB_SML_LINE_PROTOCOL && - (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) { - request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE; - smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL); - goto end; - } + if (protocol == TSDB_SML_JSON_PROTOCOL) { + numLines = 1; + } else if (numLines <= 0) { + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); + goto end; + } - if (protocol == TSDB_SML_JSON_PROTOCOL) { - numLines = 1; - } else if (numLines <= 0) { - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); - goto end; + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; + if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING || + code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) { + if (cnt++ >= 10) { + uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); + break; + } + taosMsleep(100); + refreshMeta(request->pTscObj, request); + uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code, + tstrerror(code)); + smlDestroyInfo(info); + info = NULL; + taos_free_result(request); + request = NULL; + continue; + } + smlPrintStatisticInfo(info); + break; } - code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); - request->code = code; - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - end: smlDestroyInfo(info); return (TAOS_RES *)request; diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 4609dd68b04460e27fb5ff63bd30731ac3dd4227..9fd98e33b7ce8768f530764f0e3b926771f8cea5 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -1238,7 +1238,8 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) { payloadNum = payloadNum << 1; void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + ret = TSDB_CODE_OUT_OF_MEMORY; + return ret; } info->lines = (SSmlLineInfo *)tmp; memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c index 66f1316cd5c0dce18a9e89277a010ba00777719d..335e3a1dc76a0b1cfd231f43e3e43632cd4e44ef 100644 --- a/source/client/src/clientSmlLine.c +++ b/source/client/src/clientSmlLine.c @@ -292,6 +292,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin info->currSTableMeta->uid = tinfo->uid; tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); if (tinfo->tableDataCtx == NULL) { + smlDestroyTableInfo(info, tinfo); smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); return TSDB_CODE_SML_INVALID_DATA; } @@ -436,7 +437,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin // bind data ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); if (unlikely(ret != TSDB_CODE_SUCCESS)) { - uError("smlBuildCol error, retry"); + uDebug("smlBuildCol error, retry"); info->dataFormat = false; info->reRun = true; return TSDB_CODE_SUCCESS; @@ -582,10 +583,14 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; if (info->dataFormat) { - smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); - smlBuildRow(info->currTableDataCtx); + uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts); + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + if(ret != TSDB_CODE_SUCCESS){return ret;} + ret = smlBuildRow(info->currTableDataCtx); + if(ret != TSDB_CODE_SUCCESS){return ret;} clearColValArray(info->currTableDataCtx->pValues); } else { + uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts); taosArraySet(elements->colArray, 0, &kv); } info->preLine = *elements; diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index ccf79cfc64f462e1818f58cb96216a6b73ea49f2..036442573dbb7faa2f1989b03162c84ada5390fc 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -292,7 +292,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine return TSDB_CODE_SUCCESS; } - if (info->dataFormat) { + if (info->dataFormat && info->currSTableMeta != NULL) { if (needConverTime) { kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3ed157efef1fc84b58cfe4d566965123108047cd..6e529f1a0b44556d5f07e750f050f83711fdc0b2 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -32,8 +32,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { STMT_LOG_SEQ(newStatus); } + if (pStmt->errCode && newStatus != STMT_PREPARE) { + STMT_DLOG("stmt already failed with err: %s", tstrerror(pStmt->errCode)); + return pStmt->errCode; + } + switch (newStatus) { case STMT_PREPARE: + pStmt->errCode = 0; break; case STMT_SETTBNAME: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) { @@ -197,7 +203,10 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa STscStmt* pStmt = (STscStmt*)stmt; *pVgHash = pStmt->sql.pVgHash; + pStmt->sql.pVgHash = NULL; + *pBlockHash = pStmt->exec.pBlockHash; + pStmt->exec.pBlockHash = NULL; return TSDB_CODE_SUCCESS; } @@ -325,6 +334,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { } int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + STMT_DLOG_E("start to free SQL info"); + taosMemoryFree(pStmt->sql.queryRes.fields); taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); @@ -351,6 +362,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + STMT_DLOG_E("end to free SQL info"); + return TSDB_CODE_SUCCESS; } @@ -441,11 +454,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { .mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)}; int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { - STMT_ERR_RET(stmtCleanBindInfo(pStmt)); - tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + stmtCleanBindInfo(pStmt); - return TSDB_CODE_SUCCESS; + STMT_ERR_RET(code); } STMT_ERR_RET(code); @@ -922,9 +934,13 @@ _return: int stmtClose(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to free stmt"); + stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); + STMT_DLOG_E("stmt freed"); + return TSDB_CODE_SUCCESS; } @@ -959,15 +975,17 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { } int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get tag fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -979,27 +997,33 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchTagFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchTagFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + pStmt->errCode = preCode; + + return code; } int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get col fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -1011,15 +1035,19 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchColFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchColFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + pStmt->errCode = preCode; + + return code; } int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 9f24deff94adf5da052d70987e8b9bc926d3faf0..76fd1d84d0bec43ddea33172aecdcbdfa3ab68a3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -20,17 +20,14 @@ #include "tdatablock.h" #include "tdef.h" #include "tglobal.h" -#include "tmsgtype.h" #include "tqueue.h" #include "tref.h" #include "ttimer.h" -#if 0 -#undef tsem_post -#define tsem_post(x) \ - tscInfo("call sem post at %s %d", __FUNCTION__, __LINE__); \ - sem_post(x) -#endif +#define EMPTY_BLOCK_POLL_IDLE_DURATION 10 +#define DEFAULT_AUTO_COMMIT_INTERVAL 5000 + +typedef void (*__tmq_askep_fn_t)(tmq_t* pTmq, int32_t code, SDataBuf* pBuf, void* pParam); struct SMqMgmt { int8_t inited; @@ -76,18 +73,16 @@ struct tmq_conf_t { }; struct tmq_t { - int64_t refId; - // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t withTbName; - int8_t useSnapshot; - int8_t autoCommit; - int32_t autoCommitInterval; - int32_t resetOffsetCfg; - uint64_t consumerId; - bool hbBgEnable; - + int64_t refId; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; + uint64_t consumerId; + bool hbBgEnable; tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -98,27 +93,28 @@ struct tmq_t { int8_t epStatus; int32_t epSkipCnt; #endif + // poll info int64_t pollCnt; + int64_t totalRows; // timer - tmr_h hbLiveTimer; - tmr_h epTimer; - tmr_h reportTimer; - tmr_h commitTimer; - - // connection - STscObj* pTscObj; - - // container + tmr_h hbLiveTimer; + tmr_h epTimer; + tmr_h reportTimer; + tmr_h commitTimer; + STscObj* pTscObj; // connection SArray* clientTopics; // SArray STaosQueue* mqueue; // queue of rsp STaosQall* qall; STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit - - // ctl - tsem_t rspSem; + tsem_t rspSem; }; +typedef struct SAskEpInfo { + int32_t code; + tsem_t sem; +} SAskEpInfo; + enum { TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__WAIT, @@ -137,21 +133,26 @@ enum { TMQ_DELAYED_TASK__COMMIT, }; -typedef struct { - // statistics - int64_t pollCnt; - // offset +typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; STqOffsetVal currentOffset; - // connection info - int32_t vgId; - int32_t vgStatus; - int32_t vgSkipCnt; - SEpSet epSet; + int64_t walVerBegin; + int64_t walVerEnd; +} SVgOffsetInfo; + +typedef struct { + int64_t pollCnt; + int64_t numOfRows; + SVgOffsetInfo offsetInfo; + int32_t vgId; + int32_t vgStatus; + int32_t vgSkipCnt; // here used to mark the slow vgroups + bool receiveInfo; + int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + SEpSet epSet; } SMqClientVg; typedef struct { - // subscribe info char topicName[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; SArray* vgs; // SArray @@ -160,9 +161,12 @@ typedef struct { typedef struct { int8_t tmqRspType; - int32_t epoch; + int32_t epoch; // epoch can be used to guard the vgHandle + int32_t vgId; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; + uint64_t reqId; + SEpSet* pEpset; union { SMqDataRsp dataRsp; SMqMetaRsp metaRsp; @@ -178,11 +182,10 @@ typedef struct { } SMqSubscribeCbParam; typedef struct { - int64_t refId; - int32_t epoch; - int32_t code; - int32_t async; - tsem_t rspSem; + int64_t refId; + int32_t epoch; + void* pParam; + __tmq_askep_fn_t pUserFn; } SMqAskEpCbParam; typedef struct { @@ -191,34 +194,59 @@ typedef struct { SMqClientVg* pVg; SMqClientTopic* pTopic; int32_t vgId; - tsem_t rspSem; - uint64_t requestId; // request id for debug purpose + uint64_t requestId; // request id for debug purpose } SMqPollCbParam; +typedef struct SMqVgCommon { + tsem_t rsp; + int32_t numOfRsp; + SArray* pList; + TdThreadMutex mutex; + int64_t consumerId; + char* pTopicName; + int32_t code; +} SMqVgCommon; + +typedef struct SMqVgWalInfoParam { + int32_t vgId; + int32_t epoch; + int32_t totalReq; + SMqVgCommon* pCommon; +} SMqVgWalInfoParam; + typedef struct { int64_t refId; int32_t epoch; - int8_t automatic; - int8_t async; int32_t waitingRspNum; int32_t totalRspNum; - int32_t rspErr; - tmq_commit_cb* userCb; + int32_t code; + tmq_commit_cb* callbackFn; /*SArray* successfulOffsets;*/ /*SArray* failedOffsets;*/ - void* userParam; - tsem_t rspSem; + void* userParam; } SMqCommitCbParamSet; typedef struct { SMqCommitCbParamSet* params; - STqOffset* pOffset; - SMqClientVg* pMqVg; - /*char topicName[TSDB_TOPIC_FNAME_LEN];*/ - /*int32_t vgId;*/ + SMqVgOffset* pOffset; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + tmq_t* pTmq; } SMqCommitCbParam; -static int32_t tmqAskEp(tmq_t* tmq, bool async); +typedef struct SSyncCommitInfo { + tsem_t sem; + int32_t code; +} SSyncCommitInfo; + +static int32_t doAskEp(tmq_t* tmq); +static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); +static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); +static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, + int32_t index, int32_t totalVgroups, int32_t type); +static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); +static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param); +static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param); tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -229,8 +257,8 @@ tmq_conf_t* tmq_conf_new() { conf->withTbName = false; conf->autoCommit = true; - conf->autoCommitInterval = 5000; - conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL; + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; conf->hbBgEnable = true; return conf; @@ -238,29 +266,35 @@ tmq_conf_t* tmq_conf_new() { void tmq_conf_destroy(tmq_conf_t* conf) { if (conf) { - if (conf->ip) taosMemoryFree(conf->ip); - if (conf->user) taosMemoryFree(conf->user); - if (conf->pass) taosMemoryFree(conf->pass); + if (conf->ip) { + taosMemoryFree(conf->ip); + } + if (conf->user) { + taosMemoryFree(conf->user); + } + if (conf->pass) { + taosMemoryFree(conf->pass); + } taosMemoryFree(conf); } } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id") == 0) { + if (strcasecmp(key, "group.id") == 0) { tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; } - if (strcmp(key, "client.id") == 0) { + if (strcasecmp(key, "client.id") == 0) { tstrncpy(conf->clientId, value, 256); return TMQ_CONF_OK; } - if (strcmp(key, "enable.auto.commit") == 0) { - if (strcmp(value, "true") == 0) { + if (strcasecmp(key, "enable.auto.commit") == 0) { + if (strcasecmp(value, "true") == 0) { conf->autoCommit = true; return TMQ_CONF_OK; - } else if (strcmp(value, "false") == 0) { + } else if (strcasecmp(value, "false") == 0) { conf->autoCommit = false; return TMQ_CONF_OK; } else { @@ -268,31 +302,31 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcmp(key, "auto.commit.interval.ms") == 0) { - conf->autoCommitInterval = atoi(value); + if (strcasecmp(key, "auto.commit.interval.ms") == 0) { + conf->autoCommitInterval = taosStr2int64(value); return TMQ_CONF_OK; } - if (strcmp(key, "auto.offset.reset") == 0) { - if (strcmp(value, "none") == 0) { - conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE; + if (strcasecmp(key, "auto.offset.reset") == 0) { + if (strcasecmp(value, "none") == 0) { + conf->resetOffset = TMQ_OFFSET__RESET_NONE; return TMQ_CONF_OK; - } else if (strcmp(value, "earliest") == 0) { - conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + } else if (strcasecmp(value, "earliest") == 0) { + conf->resetOffset = TMQ_OFFSET__RESET_EARLIEAST; return TMQ_CONF_OK; - } else if (strcmp(value, "latest") == 0) { - conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST; + } else if (strcasecmp(value, "latest") == 0) { + conf->resetOffset = TMQ_OFFSET__RESET_LATEST; return TMQ_CONF_OK; } else { return TMQ_CONF_INVALID; } } - if (strcmp(key, "msg.with.table.name") == 0) { - if (strcmp(value, "true") == 0) { + if (strcasecmp(key, "msg.with.table.name") == 0) { + if (strcasecmp(value, "true") == 0) { conf->withTbName = true; return TMQ_CONF_OK; - } else if (strcmp(value, "false") == 0) { + } else if (strcasecmp(value, "false") == 0) { conf->withTbName = false; return TMQ_CONF_OK; } else { @@ -300,11 +334,11 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcmp(key, "experimental.snapshot.enable") == 0) { - if (strcmp(value, "true") == 0) { + if (strcasecmp(key, "experimental.snapshot.enable") == 0) { + if (strcasecmp(value, "true") == 0) { conf->snapEnable = true; return TMQ_CONF_OK; - } else if (strcmp(value, "false") == 0) { + } else if (strcasecmp(value, "false") == 0) { conf->snapEnable = false; return TMQ_CONF_OK; } else { @@ -312,52 +346,52 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } - if (strcmp(key, "experimental.snapshot.batch.size") == 0) { - conf->snapBatchSize = atoi(value); + if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) { + conf->snapBatchSize = taosStr2int64(value); return TMQ_CONF_OK; } - if (strcmp(key, "enable.heartbeat.background") == 0) { - if (strcmp(value, "true") == 0) { - conf->hbBgEnable = true; - return TMQ_CONF_OK; - } else if (strcmp(value, "false") == 0) { - conf->hbBgEnable = false; - return TMQ_CONF_OK; - } else { - return TMQ_CONF_INVALID; - } - return TMQ_CONF_OK; + if (strcasecmp(key, "enable.heartbeat.background") == 0) { + // if (strcasecmp(value, "true") == 0) { + // conf->hbBgEnable = true; + // return TMQ_CONF_OK; + // } else if (strcasecmp(value, "false") == 0) { + // conf->hbBgEnable = false; + // return TMQ_CONF_OK; + // } else { + tscError("the default value of enable.heartbeat.background is true, can not be seted"); + return TMQ_CONF_INVALID; + // } } - if (strcmp(key, "td.connect.ip") == 0) { + if (strcasecmp(key, "td.connect.ip") == 0) { conf->ip = taosStrdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "td.connect.user") == 0) { + + if (strcasecmp(key, "td.connect.user") == 0) { conf->user = taosStrdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "td.connect.pass") == 0) { + + if (strcasecmp(key, "td.connect.pass") == 0) { conf->pass = taosStrdup(value); return TMQ_CONF_OK; } - if (strcmp(key, "td.connect.port") == 0) { - conf->port = atoi(value); + + if (strcasecmp(key, "td.connect.port") == 0) { + conf->port = taosStr2int64(value); return TMQ_CONF_OK; } - if (strcmp(key, "td.connect.db") == 0) { - /*conf->db = taosStrdup(value);*/ + + if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; } return TMQ_CONF_UNKNOWN; } -tmq_list_t* tmq_list_new() { - // - return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); -} +tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; @@ -385,108 +419,103 @@ char** tmq_list_to_c_array(const tmq_list_t* list) { return container->pData; } -static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { - return sprintf(dst, "%s:%d", topicName, vg); -} +static SMqClientVg* foundClientVg(SArray* pTopicList, const char* pName, int32_t vgId, int32_t* index, + int32_t* numOfVgroups) { + int32_t numOfTopics = taosArrayGetSize(pTopicList); + *index = -1; + *numOfVgroups = 0; -int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParamSet->refId); - if (tmq == NULL) { - if (!pParamSet->async) { - tsem_destroy(&pParamSet->rspSem); + for (int32_t i = 0; i < numOfTopics; ++i) { + SMqClientTopic* pTopic = taosArrayGet(pTopicList, i); + if (strcmp(pTopic->topicName, pName) != 0) { + continue; } - taosMemoryFree(pParamSet); - terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - return -1; - } - // if no more waiting rsp - if (pParamSet->async) { - // call async cb func - if (pParamSet->automatic && tmq->commitCb) { - tmq->commitCb(tmq, pParamSet->rspErr, tmq->commitCbUserParam); - } else if (!pParamSet->automatic && pParamSet->userCb) { - // sem post - pParamSet->userCb(tmq, pParamSet->rspErr, pParamSet->userParam); + *numOfVgroups = taosArrayGetSize(pTopic->vgs); + for (int32_t j = 0; j < (*numOfVgroups); ++j) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + if (pClientVg->vgId == vgId) { + *index = j; + return pClientVg; + } } - taosMemoryFree(pParamSet); - } else { - tsem_post(&pParamSet->rspSem); } -#if 0 - taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); - taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif - return 0; -} - -static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } + return NULL; } -int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { +// Two problems do not need to be addressed here +// 1. update to of epset. the response of poll request will automatically handle this problem +// 2. commit failure. This one needs to be resolved. +static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; - // push into array -#if 0 - if (code == 0) { - taosArrayPush(pParamSet->failedOffsets, &pParam->pOffset); - } else { - taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); - } -#endif - - // there may be race condition. fix it - if (pBuf->pEpSet != NULL && pParam->pMqVg != NULL) { - SMqClientVg* pMqVg = pParam->pMqVg; - SEp* pEp = GET_ACTIVE_EP(pBuf->pEpSet); - SEp* pOld = GET_ACTIVE_EP(&(pMqVg->epSet)); - uDebug("subKey:%s update the epset vgId:%d, ep:%s:%d, old ep:%s:%d", pParam->pOffset->subKey, pMqVg->vgId, - pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); - pParam->pMqVg->epSet = *pBuf->pEpSet; - } + // if (code != TSDB_CODE_SUCCESS) { // if commit offset failed, let's try again + // taosThreadMutexLock(&pParam->pTmq->lock); + // int32_t numOfVgroups, index; + // SMqClientVg* pVg = foundClientVg(pParam->pTmq->clientTopics, pParam->topicName, pParam->vgId, &index, + // &numOfVgroups); if (pVg == NULL) { + // tscDebug("consumer:0x%" PRIx64 + // " subKey:%s vgId:%d commit failed, code:%s has been transferred to other consumer, no need retry + // ordinal:%d/%d", pParam->pTmq->consumerId, pParam->pOffset->subKey, pParam->vgId, tstrerror(code), + // index + 1, numOfVgroups); + // } else { // let's retry the commit + // int32_t code1 = doSendCommitMsg(pParam->pTmq, pVg, pParam->topicName, pParamSet, index, numOfVgroups); + // if (code1 != TSDB_CODE_SUCCESS) { // retry failed. + // tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 + // " retry failed, ignore this commit. code:%s ordinal:%d/%d", + // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, + // tstrerror(terrno), index + 1, numOfVgroups); + // } + // } + // + // taosThreadMutexUnlock(&pParam->pTmq->lock); + // + // taosMemoryFree(pParam->pOffset); + // taosMemoryFree(pBuf->pData); + // taosMemoryFree(pBuf->pEpSet); + // + // commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); + // return 0; + // } + // + // // todo replace the pTmq with refId taosMemoryFree(pParam->pOffset); taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pEpSet); - /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, - * pOffset->version);*/ - - tmqCommitRspCountDown(pParamSet); + commitRspCountDown(pParamSet, pParam->pTmq->consumerId, pParam->topicName, pParam->vgId); return 0; } -static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pTopic, SMqCommitCbParamSet* pParamSet) { - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); +static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, + int32_t index, int32_t totalVgroups, int32_t type) { + SMqVgOffset* pOffset = taosMemoryCalloc(1, sizeof(SMqVgOffset)); if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - pOffset->val = pVg->currentOffset; + pOffset->consumerId = tmq->consumerId; + pOffset->offset.val = pVg->offsetInfo.currentOffset; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); + memcpy(pOffset->offset.subKey, tmq->groupId, groupLen); + pOffset->offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->offset.subKey + groupLen + 1, pTopicName); - int32_t len; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeMqVgOffset, pOffset, len, code); if (code < 0) { - return -1; + return TSDB_CODE_INVALID_PARA; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); if (buf == NULL) { taosMemoryFree(pOffset); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); @@ -495,7 +524,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); + tEncodeMqVgOffset(&encoder, pOffset); tEncoderClear(&encoder); // build param @@ -503,12 +532,15 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT if (pParam == NULL) { taosMemoryFree(pOffset); taosMemoryFree(buf); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } pParam->params = pParamSet; pParam->pOffset = pOffset; - pParam->pMqVg = pVg; // there may be an race condition + pParam->vgId = pVg->vgId; + pParam->pTmq = tmq; + + tstrncpy(pParam->topicName, pTopicName, tListLen(pParam->topicName)); // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -516,139 +548,146 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT taosMemoryFree(pOffset); taosMemoryFree(buf); taosMemoryFree(pParam); - return -1; + return TSDB_CODE_OUT_OF_MEMORY; } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; - - SEp* pEp = &pVg->epSet.eps[pVg->epSet.inUse]; - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64 " prev:%" PRId64 ", ep:%s:%d", tmq->consumerId, - pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, pEp->port); - - // TODO: put into cb, the commit offset should be move to the callback function - pVg->committedOffset = pVg->currentOffset; + pMsgSendInfo->msgInfo = (SDataBuf) { .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL }; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; - // send msg + pMsgSendInfo->msgType = type; atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); atomic_add_fetch_32(&pParamSet->totalRspNum, 1); + SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); + char offsetBuf[80] = {0}; + tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->offset.val); + + char commitBuf[80] = {0}; + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); + tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, + tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, + totalVgroups, pMsgSendInfo->requestId); + int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - return 0; + + return TSDB_CODE_SUCCESS; +} + +static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { + int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < numOfTopics; ++i) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(pTopic->topicName, pTopicName) != 0) { + continue; + } + + return pTopic; + } + + tscError("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); + return NULL; } -int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { - char* topic; - int32_t vgId; - if (TD_RES_TMQ(msg)) { - SMqRspObj* pRspObj = (SMqRspObj*)msg; - topic = pRspObj->topic; +static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tmq_commit_cb* pCommitFp, void* userParam) { + char* pTopicName = NULL; + int32_t vgId = 0; + int32_t code = 0; + + if (pRes == NULL || tmq == NULL) { + pCommitFp(tmq, TSDB_CODE_INVALID_PARA, userParam); + return; + } + + if (TD_RES_TMQ(pRes)) { + SMqRspObj* pRspObj = (SMqRspObj*)pRes; + pTopicName = pRspObj->topic; vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(msg)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; - topic = pMetaRspObj->topic; + } else if (TD_RES_TMQ_META(pRes)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)pRes; + pTopicName = pMetaRspObj->topic; vgId = pMetaRspObj->vgId; - } else if (TD_RES_TMQ_METADATA(msg)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)msg; - topic = pRspObj->topic; + } else if (TD_RES_TMQ_METADATA(pRes)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)pRes; + pTopicName = pRspObj->topic; vgId = pRspObj->vgId; } else { - return TSDB_CODE_TMQ_INVALID_MSG; + pCommitFp(tmq, TSDB_CODE_TMQ_INVALID_MSG, userParam); + return; } SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); + return; } + pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - pParamSet->automatic = 0; - pParamSet->async = async; - pParamSet->userCb = userCb; + pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); - int32_t code = -1; + int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, topic) != 0) continue; - for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId != vgId) continue; + tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); - if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); - goto FAIL; - } - goto HANDLE_RSP; - } - } + SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); + if (pTopic == NULL) { + tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, + pTopicName, numOfTopics); + taosMemoryFree(pParamSet); + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } -HANDLE_RSP: - if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); - return 0; + int32_t j = 0; + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + for (j = 0; j < numOfVgroups; j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->vgId == vgId) { + break; + } } - if (!async) { - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); + if (j == numOfVgroups) { + tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, + vgId, numOfVgroups, pTopicName); taosMemoryFree(pParamSet); - return code; - } else { - code = 0; + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } -FAIL: - if (code != 0 && async) { - userCb(tmq, code, userParam); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { + code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, type); + + // failed to commit, callback user function directly. + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pParamSet); + pCommitFp(tmq, code, userParam); + } + } else { // do not perform commit, callback user function directly. + taosMemoryFree(pParamSet); + pCommitFp(tmq, code, userParam); } - return 0; } -static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { - int32_t code = -1; - +static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - if (async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } - } - return -1; + pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); + return; } pParamSet->refId = tmq->refId; pParamSet->epoch = tmq->epoch; - - pParamSet->automatic = automatic; - pParamSet->async = async; - pParamSet->userCb = userCb; + pParamSet->callbackFn = pCommitFp; pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); // init as 1 to prevent concurrency issue pParamSet->waitingRspNum = 1; @@ -658,75 +697,65 @@ static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, + numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + + if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { + int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, TDMT_VND_TMQ_COMMIT_OFFSET); + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, tstrerror(terrno), + j + 1, numOfVgroups); continue; } + + // update the offset value. + pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset; } else { - tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, not commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset.version, j + 1, numOfVgroups); + tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.currentOffset.version, j + 1, numOfVgroups); } } } + tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, + numOfTopics); + // no request is sent if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - return 0; + pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); + return; } // count down since waiting rsp num init as 1 - tmqCommitRspCountDown(pParamSet); - - if (!async) { - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); -#if 0 - taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); - taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); -#endif - } - - return code; -} - -int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { - if (msg) { - return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); - } else { - return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam); - } + commitRspCountDown(pParamSet, tmq->consumerId, "", 0); } -void tmqAssignAskEpTask(void* param, void* tmrId) { - int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); +static void generateTimedTask(int64_t refId, int32_t type) { + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = TMQ_DELAYED_TASK__ASK_EP; + *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } + taosReleaseRef(tmqMgmt.rsetId, refId); +} + +void tmqAssignAskEpTask(void* param, void* tmrId) { + int64_t refId = *(int64_t*)param; + generateTimedTask(refId, TMQ_DELAYED_TASK__ASK_EP); taosMemoryFree(param); } void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - *pTaskType = TMQ_DELAYED_TASK__COMMIT; - taosWriteQitem(tmq->delayedTask, pTaskType); - tsem_post(&tmq->rspSem); - } + generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT); taosMemoryFree(param); } @@ -739,6 +768,8 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } + + taosReleaseRef(tmqMgmt.rsetId, refId); taosMemoryFree(param); } @@ -752,7 +783,8 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { taosMemoryFree(param); return; @@ -765,17 +797,19 @@ void tmqSendHbReq(void* param, void* tmrId) { int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req); if (tlen < 0) { tscError("tSerializeSMqHbReq failed"); - return; + goto OVER; } + void* pReq = taosMemoryCalloc(1, tlen); if (tlen < 0) { tscError("failed to malloc MqHbReq msg, size:%d", tlen); - return; + goto OVER; } + if (tSerializeSMqHbReq(pReq, tlen, &req) < 0) { tscError("tSerializeSMqHbReq %d failed", tlen); taosMemoryFree(pReq); - return; + goto OVER; } SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -783,11 +817,8 @@ void tmqSendHbReq(void* param, void* tmrId) { taosMemoryFree(pReq); goto OVER; } - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = tlen, - .handle = NULL, - }; + + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL }; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; @@ -802,6 +833,13 @@ void tmqSendHbReq(void* param, void* tmrId) { OVER: taosTmrReset(tmqSendHbReq, 1000, param, tmqMgmt.timer, &tmq->hbLiveTimer); + taosReleaseRef(tmqMgmt.rsetId, refId); +} + +static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { + if (code != 0) { + tscDebug("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); + } } int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { @@ -819,7 +857,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { while (pTaskType != NULL) { if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { - tmqAskEp(pTmq, true); + asyncAskEp(pTmq, addToQueueCallbackFn, NULL); int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; @@ -827,12 +865,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner(pTmq, NULL, 1, 1, pTmq->commitCb, pTmq->commitCbUserParam); + tmq_commit_cb* pCallbackFn = pTmq->commitCb ? pTmq->commitCb : defaultCommitCbFn; + asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; - tscDebug("consumer:0x%" PRIx64 " commit to vnode(s) in %.2fs", pTmq->consumerId, + tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval / 1000.0); taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { @@ -846,7 +885,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { return 0; } -static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { +static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { // do nothing } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { @@ -854,15 +893,21 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->dataRsp.blockDataLen); taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree); taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosMemoryFree(pRsp->metaRsp.metaRsp); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->taosxRsp.blockDataLen); taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree); @@ -871,6 +916,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { taosArrayDestroy(pRsp->taosxRsp.createTableLen); taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree); } + + return NULL; } void tmqClearUnhandleMsg(tmq_t* tmq) { @@ -936,28 +983,34 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { return rsp; } +static void freeClientVgImpl(void* param) { + SMqClientTopic* pTopic = param; + taosMemoryFreeClear(pTopic->schema.pSchema); + taosArrayDestroy(pTopic->vgs); +} + void tmqFreeImpl(void* handle) { - tmq_t* tmq = (tmq_t*)handle; + tmq_t* tmq = (tmq_t*)handle; + int64_t id = tmq->consumerId; // TODO stop timer if (tmq->mqueue) { tmqClearUnhandleMsg(tmq); taosCloseQueue(tmq->mqueue); } - if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); - taosFreeQall(tmq->qall); + if (tmq->delayedTask) { + taosCloseQueue(tmq->delayedTask); + } + + taosFreeQall(tmq->qall); tsem_destroy(&tmq->rspSem); - int32_t sz = taosArrayGetSize(tmq->clientTopics); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroy(pTopic->vgs); - } - taosArrayDestroy(tmq->clientTopics); + taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taos_close_internal(tmq->pTscObj); taosMemoryFree(tmq); + + tscDebug("consumer:0x%" PRIx64 " closed", id); } static void tmqMgmtInit(void) { @@ -984,7 +1037,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("failed to create consumer, consumer group %s, code:%s", conf->groupId, terrstr()); + tscError("failed to create consumer, groupId:%s, code:%s", conf->groupId, terrstr()); return NULL; } @@ -993,23 +1046,20 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->mqueue = taosOpenQueue(); - pTmq->qall = taosAllocateQall(); pTmq->delayedTask = taosOpenQueue(); + pTmq->qall = taosAllocateQall(); if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer:0x%" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); - goto FAIL; + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); + goto _failed; } // init status pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; - /*pTmq->epStatus = 0;*/ - /*pTmq->epSkipCnt = 0;*/ // set conf strcpy(pTmq->clientId, conf->clientId); @@ -1031,22 +1081,20 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); - goto FAIL; + goto _failed; } // init connection pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) { - tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tsem_destroy(&pTmq->rspSem); - goto FAIL; + goto _failed; } pTmq->refId = taosAddRef(tmqMgmt.rsetId, pTmq); if (pTmq->refId < 0) { - tmqFreeImpl(pTmq); - return NULL; + goto _failed; } if (pTmq->hbBgEnable) { @@ -1055,20 +1103,23 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } - tscInfo("consumer:0x%" PRIx64 " is setup, groupId:%s", pTmq->consumerId, pTmq->groupId); - return pTmq; + char buf[80] = {0}; + STqOffsetVal offset = {.type = pTmq->resetOffsetCfg}; + tFormatOffset(buf, tListLen(buf), &offset); + tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 + ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s, backgroudHB:%d", + pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, + buf, pTmq->hbBgEnable); -FAIL: - if (pTmq->clientTopics) taosArrayDestroy(pTmq->clientTopics); - if (pTmq->mqueue) taosCloseQueue(pTmq->mqueue); - if (pTmq->delayedTask) taosCloseQueue(pTmq->delayedTask); - if (pTmq->qall) taosFreeQall(pTmq->qall); - taosMemoryFree(pTmq); + return pTmq; +_failed: + tmqFreeImpl(pTmq); return NULL; } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { + const int32_t MAX_RETRY_COUNT = 120 * 4; // let's wait for 4 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1121,21 +1172,13 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - SMqSubscribeCbParam param = { - .rspErr = 0, - .refId = tmq->refId, - .epoch = tmq->epoch, - }; - + SMqSubscribeCbParam param = { .rspErr = 0, .refId = tmq->refId, .epoch = tmq->epoch }; if (tsem_init(¶m.rspSem, 0, 0) != 0) { + code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; } - sendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; @@ -1161,8 +1204,10 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } int32_t retryCnt = 0; - while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { - if (retryCnt++ > 40) { + while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { + if (retryCnt++ > MAX_RETRY_COUNT) { + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, max retry reached:%d", tmq->consumerId, retryCnt); + code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; } @@ -1193,19 +1238,19 @@ FAIL: } void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { - // conf->commitCb = cb; conf->commitCbUserParam = param; } -int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { +static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; + + int64_t refId = pParam->refId; SMqClientVg* pVg = pParam->pVg; SMqClientTopic* pTopic = pParam->pTopic; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1220,8 +1265,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { - tscWarn("consumer:0x%"PRIx64" msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%"PRIx64, tmq->consumerId, vgId, - epoch, tstrerror(code), requestId); + tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId, + vgId, epoch, tstrerror(code), requestId); if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet); @@ -1230,7 +1275,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { taosMsleep(500); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); - tscDebug("consumer:0x%" PRIx64" wait for the re-balance, wait for 500ms and set status to be RECOVER", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, wait for 500ms and set status to be RECOVER", + tmq->consumerId); } else if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0); if (pRspWrapper == NULL) { @@ -1240,31 +1286,33 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; - /*pRspWrapper->vgHandle = pVg;*/ - /*pRspWrapper->topicHandle = pTopic;*/ taosWriteQitem(tmq->mqueue, pRspWrapper); - tsem_post(&tmq->rspSem); + } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert + taosMsleep(500); } goto CREATE_MSG_FAIL; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; - int32_t tmqEpoch = atomic_load_32(&tmq->epoch); - if (msgEpoch < tmqEpoch) { + int32_t clientEpoch = atomic_load_32(&tmq->epoch); + if (msgEpoch < clientEpoch) { // do not write into queue since updating epoch reset - tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%"PRIx64, - tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId); + tscWarn("consumer:0x%" PRIx64 + " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); + taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); return 0; } - if (msgEpoch != tmqEpoch) { - tscWarn("consumer:0x%"PRIx64" mismatch rsp from vgId:%d, epoch %d, current epoch %d, reqId:0x%"PRIx64, tmq->consumerId, vgId, - msgEpoch, tmqEpoch, requestId); + if (msgEpoch != clientEpoch) { + tscWarn("consumer:0x%" PRIx64 " mismatch rsp from vgId:%d, epoch %d, current epoch %d, reqId:0x%" PRIx64, + tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); } // handle meta rsp @@ -1274,28 +1322,34 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (pRspWrapper == NULL) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); - tscWarn("consumer:0x%"PRIx64" msg discard from vgId:%d, epoch %d since out of memory", tmq->consumerId, vgId, epoch); + tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, epoch %d since out of memory", tmq->consumerId, vgId, + epoch); goto CREATE_MSG_FAIL; } pRspWrapper->tmqRspType = rspType; pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; + pRspWrapper->reqId = requestId; + pRspWrapper->pEpset = pMsg->pEpSet; + pRspWrapper->vgId = pVg->vgId; + pMsg->pEpSet = NULL; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req offset:%" PRId64 ", rsp offset:%" PRId64 " type %d, reqId:0x%"PRIx64, - tmq->consumerId, pVg->vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, - rspType, requestId); + char buf[80]; + tFormatOffset(buf, 80, &pRspWrapper->dataRsp.rspOffset); + tscDebug("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, req ver:%" PRId64 ", rsp:%s type %d, reqId:0x%" PRIx64, + tmq->consumerId, vgId, pRspWrapper->dataRsp.reqOffset.version, buf, rspType, requestId); } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeSMqMetaRsp(&decoder, &pRspWrapper->metaRsp); + tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); } else if (rspType == TMQ_MSG_TYPE__TAOSX_RSP) { @@ -1304,16 +1358,19 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); + } else { // invalid rspType + tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); } taosMemoryFree(pMsg->pData); - taosMemoryFree(pMsg->pEpSet); + taosWriteQitem(tmq->mqueue, pRspWrapper); - tscDebug("consumer:0x%" PRIx64 ", put poll res into mqueue, total in queue:%d, reqId:0x%" PRIx64, tmq->consumerId, - tmq->mqueue->numOfItems, requestId); + int32_t total = taosQueueItemSize(tmq->mqueue); + tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, + tmq->consumerId, rspType, vgId, total, requestId); - taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); return 0; @@ -1323,10 +1380,72 @@ CREATE_MSG_FAIL: } tsem_post(&tmq->rspSem); + taosReleaseRef(tmqMgmt.rsetId, refId); + return -1; } -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { +typedef struct SVgroupSaveInfo { + STqOffsetVal offset; + int64_t numOfRows; +} SVgroupSaveInfo; + +static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, + tmq_t* tmq) { + pTopic->schema = pTopicEp->schema; + pTopicEp->schema.nCols = 0; + pTopicEp->schema.pSchema = NULL; + + char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); + + tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); + + tscDebug("consumer:0x%" PRIx64 ", update topic:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); + + for (int32_t j = 0; j < vgNumGet; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + + makeTopicVgroupKey(vgKey, pTopic->topicName, pVgEp->vgId); + SVgroupSaveInfo* pInfo = taosHashGet(pVgOffsetHashMap, vgKey, strlen(vgKey)); + + int64_t numOfRows = 0; + STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; + if (pInfo != NULL) { + offsetNew = pInfo->offset; + numOfRows = pInfo->numOfRows; + } + + SMqClientVg clientVg = { + .pollCnt = 0, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet, + .vgStatus = TMQ_VG_STATUS__IDLE, + .vgSkipCnt = 0, + .emptyBlockReceiveTs = 0, + .numOfRows = numOfRows, + }; + + clientVg.offsetInfo.currentOffset = offsetNew; + clientVg.offsetInfo.committedOffset = offsetNew; + clientVg.offsetInfo.walVerBegin = -1; + clientVg.offsetInfo.walVerEnd = -1; + taosArrayPush(pTopic->vgs, &clientVg); + } +} + +static void freeClientVgInfo(void* param) { + SMqClientTopic* pTopic = param; + if (pTopic->schema.nCols) { + taosMemoryFreeClear(pTopic->schema.pSchema); + } + + taosArrayDestroy(pTopic->vgs); +} + +static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); @@ -1335,18 +1454,22 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); + if (epoch <= tmq->epoch) { + return false; + } SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { return false; } - SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pHash == NULL) { + SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + if (pVgOffsetHashMap == NULL) { taosArrayDestroy(newTopics); return false; } + // todo extract method for (int32_t i = 0; i < topicNumCur; i++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); @@ -1355,12 +1478,15 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { tscDebug("consumer:0x%" PRIx64 ", new vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); + makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); + char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffset); - tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, - pVgCur->vgId, vgKey, buf); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal)); + tFormatOffset(buf, 80, &pVgCur->offsetInfo.currentOffset); + tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, + vgKey, buf); + + SVgroupSaveInfo info = {.offset = pVgCur->offsetInfo.currentOffset, .numOfRows = pVgCur->numOfRows}; + taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } } @@ -1368,87 +1494,50 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { for (int32_t i = 0; i < topicNumGet; i++) { SMqClientTopic topic = {0}; SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - topic.schema = pTopicEp->schema; - pTopicEp->schema.nCols = 0; - pTopicEp->schema.pSchema = NULL; - tstrncpy(topic.topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); - tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscDebug("consumer:0x%" PRIx64 ", update topic: %s", tmq->consumerId, topic.topicName); - - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - STqOffsetVal* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; - if (pOffset != NULL) { - offsetNew = *pOffset; - } - - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = offsetNew, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } + initClientTopicFromRsp(&topic, pTopicEp, pVgOffsetHashMap, tmq); taosArrayPush(newTopics, &topic); } + taosHashCleanup(pVgOffsetHashMap); + // destroy current buffered existed topics info if (tmq->clientTopics) { - int32_t sz = taosArrayGetSize(tmq->clientTopics); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - taosArrayDestroy(pTopic->vgs); - } - - taosArrayDestroy(tmq->clientTopics); + taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } - - taosHashCleanup(pHash); tmq->clientTopics = newTopics; - if (taosArrayGetSize(tmq->clientTopics) == 0) { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - } else { - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - } - + int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; + atomic_store_8(&tmq->status, flag); atomic_store_32(&tmq->epoch, epoch); + tscDebug("consumer:0x%" PRIx64 " update topic info completed", tmq->consumerId); return set; } -int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { +int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; - int8_t async = pParam->async; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); if (tmq == NULL) { - if (!async) { - tsem_destroy(&pParam->rspSem); - } else { - taosMemoryFree(pParam); - } + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + pParam->pUserFn(tmq, terrno, NULL, pParam->pParam); + taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); - terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - return -1; + taosMemoryFree(pParam); + return terrno; } - pParam->code = code; - if (code != 0) { - tscError("consumer:0x%" PRIx64 ", get topic endpoint error, async:%d, code:%s", tmq->consumerId, pParam->async, - tstrerror(code)); - goto END; + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); + pParam->pUserFn(tmq, code, NULL, pParam->pParam); + + taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pParam); + return code; } // tmq's epoch is monotonically increase, @@ -1459,132 +1548,26 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (head->epoch <= epoch) { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, no need to update local ep", tmq->consumerId, head->epoch, epoch); - goto END; - } - - tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, - head->epoch, epoch); - if (!async) { - SMqAskEpRsp rsp; - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); - /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ - /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp(tmq, head->epoch, &rsp); - tDeleteSMqAskEpRsp(&rsp); - } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); - if (pWrapper == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto END; + if (tmq->status == TMQ_CONSUMER_STATUS__RECOVER) { + SMqAskEpRsp rsp; + tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); + int8_t flag = (taosArrayGetSize(rsp.topics) == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; + atomic_store_8(&tmq->status, flag); + tDeleteSMqAskEpRsp(&rsp); } - pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; - pWrapper->epoch = head->epoch; - memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); - - taosWriteQitem(tmq->mqueue, pWrapper); - tsem_post(&tmq->rspSem); - } - -END: - /*atomic_store_8(&tmq->epStatus, 0);*/ - if (!async) { - tsem_post(&pParam->rspSem); } else { - taosMemoryFree(pParam); + tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, + head->epoch, epoch); } + pParam->pUserFn(tmq, code, pMsg, pParam->pParam); + taosReleaseRef(tmqMgmt.rsetId, pParam->refId); + taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pData); - return code; -} - -int32_t tmqAskEp(tmq_t* tmq, bool async) { - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); - if (epStatus == 1) { - int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); - tscTrace("consumer:0x%" PRIx64 ", skip ask ep cnt %d", tmq->consumerId, epSkipCnt); - if (epSkipCnt < 5000) return 0; - } - atomic_store_32(&tmq->epSkipCnt, 0); -#endif - - SMqAskEpReq req = {0}; - req.consumerId = tmq->consumerId; - req.epoch = tmq->epoch; - strcpy(req.cgroup, tmq->groupId); - - int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); - if (tlen < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", tmq->consumerId); - return -1; - } - - void* pReq = taosMemoryCalloc(1, tlen); - if (pReq == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", tmq->consumerId, tlen); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { - tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", tmq->consumerId, tlen); - taosMemoryFree(pReq); - return -1; - } - - SMqAskEpCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); - if (pParam == NULL) { - tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", tmq->consumerId); - taosMemoryFree(pReq); - /*atomic_store_8(&tmq->epStatus, 0);*/ - return -1; - } - - pParam->refId = tmq->refId; - pParam->epoch = tmq->epoch; - pParam->async = async; - tsem_init(&pParam->rspSem, 0, 0); - - SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (sendInfo == NULL) { - tsem_destroy(&pParam->rspSem); - taosMemoryFree(pParam); - taosMemoryFree(pReq); - /*atomic_store_8(&tmq->epStatus, 0);*/ - return -1; - } - - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = tlen, - .handle = NULL, - }; - - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqAskEpCb; - sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; - - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async, - sendInfo->requestId); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - if (!async) { - tsem_wait(&pParam->rspSem); - code = pParam->code; - taosMemoryFree(pParam); - } - + taosMemoryFree(pParam); return code; } @@ -1598,8 +1581,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->consumerId = tmq->consumerId; pReq->timeout = timeout; pReq->epoch = tmq->epoch; - /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffset; + pReq->reqOffset = pVg->offsetInfo.currentOffset; pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); @@ -1616,21 +1598,33 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } -SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { +SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; + + (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; + if (!pWrapper->dataRsp.withSchema) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } + // extract the rows in this data packet + for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + int64_t rows = htobe64(pRetrieve->numOfRows); + pVg->numOfRows += rows; + (*numOfRows) += rows; + } + return pRspObj; } @@ -1685,7 +1679,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it + pParam->pVg = pVg; pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1697,12 +1691,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p return handleErrorBeforePoll(pVg, pTmq); } - sendInfo->msgInfo = (SDataBuf){ - .pData = msg, - .len = msgSize, - .handle = NULL, - }; - + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; @@ -1711,10 +1700,10 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p int64_t transporterId = 0; char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->currentOffset); + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, - pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId); + tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, + pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId); asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1724,7 +1713,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p } // broadcast the poll request to all related vnodes -int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { +static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); @@ -1734,19 +1723,18 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); + if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms + tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, + tmq->epoch, pVg->vgId); + continue; + } + + int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; -#if 0 - if (skipCnt < 30000) { - continue; - } else { - tscDebug("consumer:0x%" PRIx64 ",skip vgId:%d skip too much reset", tmq->consumerId, pVg->vgId); - } -#endif } atomic_store_32(&pVg->vgSkipCnt, 0); @@ -1757,16 +1745,17 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } } + tscDebug("consumer:0x%" PRIx64 " end to poll data", tmq->consumerId); return 0; } -int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) { +static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); + doUpdateLocalEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ tDeleteSMqAskEpRsp(rspMsg); *pReset = true; @@ -1780,141 +1769,171 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) return 0; } -void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { +static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems); while (1) { - SMqRspWrapper* rspWrapper = NULL; - taosGetQitem(tmq->qall, (void**)&rspWrapper); + SMqRspWrapper* pRspWrapper = NULL; + taosGetQitem(tmq->qall, (void**)&pRspWrapper); - if (rspWrapper == NULL) { + if (pRspWrapper == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); - taosGetQitem(tmq->qall, (void**)&rspWrapper); - - if (rspWrapper == NULL) { + taosGetQitem(tmq->qall, (void**)&pRspWrapper); + if (pRspWrapper == NULL) { return NULL; } } - if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { - taosFreeQitem(rspWrapper); + tscDebug("consumer:0x%" PRIx64 " handle rsp, type:%d", tmq->consumerId, pRspWrapper->tmqRspType); + + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { + taosFreeQitem(pRspWrapper); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(terrno)); return NULL; - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; + + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + SMqDataRsp* pDataRsp = &pollRspWrapper->dataRsp; - /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { + if (pDataRsp->head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; - atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (pollRspWrapper->dataRsp.blockNum == 0) { - tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d", tmq->consumerId, pVg->vgId); - taosFreeQitem(pollRspWrapper); - rspWrapper = NULL; - continue; + // update the epset + if (pollRspWrapper->pEpset != NULL) { + SEp* pEp = GET_ACTIVE_EP(pollRspWrapper->pEpset); + SEp* pOld = GET_ACTIVE_EP(&(pVg->epSet)); + tscDebug("consumer:0x%" PRIx64 " update epset vgId:%d, ep:%s:%d, old ep:%s:%d", tmq->consumerId, pVg->vgId, + pEp->fqdn, pEp->port, pOld->fqdn, pOld->port); + pVg->epSet = *pollRspWrapper->pEpset; } - // build rsp - char buf[80]; - tFormatOffset(buf, 80, &pVg->currentOffset); - SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper); - tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d", tmq->consumerId, - pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum); + // update the local offset value only for the returned values. + pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; - taosFreeQitem(pollRspWrapper); - return pRsp; + // update the status + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + + // update the valid wal version range + pVg->offsetInfo.walVerBegin = pDataRsp->head.walsver; + pVg->offsetInfo.walVerEnd = pDataRsp->head.walever; + pVg->receiveInfo = true; + + char buf[80]; + tFormatOffset(buf, 80, &pDataRsp->rspOffset); + if (pDataRsp->blockNum == 0) { + tscDebug("consumer:0x%" PRIx64 " empty block received, vgId:%d, offset:%s, vg total:%" PRId64 + " total:%" PRId64 " reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); + pRspWrapper = tmqFreeRspWrapper(pRspWrapper); + taosFreeQitem(pollRspWrapper); + } else { // build rsp + int64_t numOfRows = 0; + SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + tmq->totalRows += numOfRows; + + tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 + " vg total:%" PRId64 " total:%" PRId64 ", reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, + pollRspWrapper->reqId); + taosFreeQitem(pollRspWrapper); + return pRsp; + } } else { - tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->dataRsp.head.epoch, consumerEpoch); - tmqFreeRspWrapper(rspWrapper); + tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); + pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); } - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + // todo handle the wal range and epset for each vgroup + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, - * rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); - tmqFreeRspWrapper(rspWrapper); + tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); + pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); } - } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { - SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; - /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, - * rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + if (pollRspWrapper->taosxRsp.blockNum == 0) { + tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 " reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, pVg->numOfRows, pollRspWrapper->reqId); + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); - rspWrapper = NULL; continue; + } else { + pVg->emptyBlockReceiveTs = 0; // reset the ts } // build rsp - void* pRsp = NULL; + void* pRsp = NULL; + int64_t numOfRows = 0; if (pollRspWrapper->taosxRsp.createTableNum == 0) { - pRsp = tmqBuildRspFromWrapper(pollRspWrapper); + pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); } else { pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper); } + + tmq->totalRows += numOfRows; + + char buf[80]; + tFormatOffset(buf, 80, &pVg->offsetInfo.currentOffset); + tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 + ", vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64, + tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, + tmq->totalRows, pollRspWrapper->reqId); + taosFreeQitem(pollRspWrapper); return pRsp; + } else { - tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); - tmqFreeRspWrapper(rspWrapper); + tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); + pRspWrapper = tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pollRspWrapper); } } else { - /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ + tscDebug("consumer:0x%" PRIx64 " not data msg received", tmq->consumerId); + bool reset = false; - tmqHandleNoPollRsp(tmq, rspWrapper, &reset); - taosFreeQitem(rspWrapper); + tmqHandleNoPollRsp(tmq, pRspWrapper, &reset); + taosFreeQitem(pRspWrapper); if (pollIfReset && reset) { tscDebug("consumer:0x%" PRIx64 ", reset and repoll", tmq->consumerId); tmqPollImpl(tmq, timeout); } } } - - tscDebug("consumer:0x%" PRIx64 " handle the rsp completed", tmq->consumerId); } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { void* rspObj; int64_t startTime = taosGetTimestampMs(); - tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64, tmq->consumerId, startTime); - -#if 0 - tmqHandleAllDelayedTask(tmq); - tmqPollImpl(tmq, timeout); - rspObj = tmqHandleAllRsp(tmq, timeout, false); - if (rspObj) { - return (TAOS_RES*)rspObj; - } -#endif + tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, + timeout); // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { @@ -1923,9 +1942,9 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { + while (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { int32_t retryCnt = 0; - while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > 40) { return NULL; } @@ -1940,7 +1959,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { if (tmqPollImpl(tmq, timeout) < 0) { tscDebug("consumer:0x%" PRIx64 " return due to poll error", tmq->consumerId); - /*return NULL;*/ } rspObj = tmqHandleAllRsp(tmq, timeout, false); @@ -1952,7 +1970,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - if (timeout != -1) { + if (timeout >= 0) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; if (elapsedTime > timeout) { @@ -1960,9 +1978,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; } - /*tscInfo("consumer:0x%" PRIx64 ", (epoch %d) wait, start time %" PRId64 ", current time %" PRId64*/ - /*", left time %" PRId64,*/ - /*tmq->consumerId, tmq->epoch, startTime, currentTime, (timeout - elapsedTime));*/ tsem_timewait(&tmq->rspSem, (timeout - elapsedTime)); } else { // use tsem_timewait instead of tsem_wait to avoid unexpected stuck @@ -1971,17 +1986,43 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } } +static void displayConsumeStatistics(const tmq_t* pTmq) { + int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); + tscDebug("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", + pTmq->consumerId, pTmq->pollCnt, pTmq->totalRows, numOfTopics, pTmq->epoch); + + tscDebug("consumer:0x%" PRIx64 " rows dist begin: ", pTmq->consumerId); + for (int32_t i = 0; i < numOfTopics; ++i) { + SMqClientTopic* pTopics = taosArrayGet(pTmq->clientTopics, i); + + tscDebug("consumer:0x%" PRIx64 " topic:%d", pTmq->consumerId, i); + int32_t numOfVgs = taosArrayGetSize(pTopics->vgs); + for (int32_t j = 0; j < numOfVgs; ++j) { + SMqClientVg* pVg = taosArrayGet(pTopics->vgs, j); + tscDebug("topic:%s, %d. vgId:%d rows:%" PRId64, pTopics->topicName, j, pVg->vgId, pVg->numOfRows); + } + } + + tscDebug("consumer:0x%" PRIx64 " rows dist end", pTmq->consumerId); +} + int32_t tmq_consumer_close(tmq_t* tmq) { + tscDebug("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); + displayConsumeStatistics(tmq); + if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - int32_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != 0) { - return rsp; + // if auto commit is set, commit before close consumer. Otherwise, do nothing. + if (tmq->autoCommit) { + int32_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp != 0) { + return rsp; + } } int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); while (1) { - rsp = tmq_subscribe(tmq, lst); + int32_t rsp = tmq_subscribe(tmq, lst); if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { break; } else { @@ -1991,7 +2032,10 @@ int32_t tmq_consumer_close(tmq_t* tmq) { } tmq_list_destroy(lst); + } else { + tscWarn("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId); } + taosRemoveRef(tmqMgmt.rsetId, tmq->refId); return 0; } @@ -2082,12 +2126,480 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { - // - tmqCommitInner(tmq, msg, 0, 1, cb, param); +void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { + if (pRes == NULL) { // here needs to commit all offsets. + asyncCommitAllOffsets(tmq, cb, param); + } else { // only commit one offset + asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, cb, param); + } } -int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { - // - return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); +static void commitCallBackFn(tmq_t *UNUSED_PARAM(tmq), int32_t code, void* param) { + SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param; + pInfo->code = code; + tsem_post(&pInfo->sem); +} + +int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { + int32_t code = 0; + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + if (pRes == NULL) { + asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); + } else { + asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, commitCallBackFn, pInfo); + } + + tsem_wait(&pInfo->sem); + code = pInfo->code; + + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + + tscDebug("consumer:0x%" PRIx64 " sync commit done, code:%s", tmq->consumerId, tstrerror(code)); + return code; +} + +void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { + SAskEpInfo* pInfo = param; + pInfo->code = code; + + if (code == TSDB_CODE_SUCCESS) { + SMqRspHead* head = pDataBuf->pData; + + SMqAskEpRsp rsp; + tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp); + doUpdateLocalEp(pTmq, head->epoch, &rsp); + tDeleteSMqAskEpRsp(&rsp); + } + + tsem_post(&pInfo->sem); +} + +void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param) { + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return; + } + + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); + if (pWrapper == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + SMqRspHead* head = pDataBuf->pData; + + pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; + pWrapper->epoch = head->epoch; + memcpy(&pWrapper->msg, pDataBuf->pData, sizeof(SMqRspHead)); + tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &pWrapper->msg); + + taosWriteQitem(pTmq->mqueue, pWrapper); +} + +int32_t doAskEp(tmq_t* pTmq) { + SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); + tsem_init(&pInfo->sem, 0, 0); + + asyncAskEp(pTmq, updateEpCallbackFn, pInfo); + tsem_wait(&pInfo->sem); + + int32_t code = pInfo->code; + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + return code; +} + +void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param) { + SMqAskEpReq req = {0}; + req.consumerId = pTmq->consumerId; + req.epoch = pTmq->epoch; + strcpy(req.cgroup, pTmq->groupId); + + int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); + if (tlen < 0) { + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", pTmq->consumerId); + askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param); + return; + } + + void* pReq = taosMemoryCalloc(1, tlen); + if (pReq == NULL) { + tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", pTmq->consumerId, tlen); + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; + } + + if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); + taosMemoryFree(pReq); + + askEpFn(pTmq, TSDB_CODE_INVALID_PARA, NULL, param); + return; + } + + SMqAskEpCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); + taosMemoryFree(pReq); + + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; + } + + pParam->refId = pTmq->refId; + pParam->epoch = pTmq->epoch; + pParam->pUserFn = askEpFn; + pParam->pParam = param; + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pParam); + taosMemoryFree(pReq); + askEpFn(pTmq, TSDB_CODE_OUT_OF_MEMORY, NULL, param); + return; + } + + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; + + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = askEpCallbackFn; + sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; + + SEpSet epSet = getEpSet_s(&pTmq->pTscObj->pAppInfo->mgmtEp); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, reqId:0x%" PRIx64, pTmq->consumerId, sendInfo->requestId); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); +} + +int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg) { + return sprintf(dst, "%s:%d", topicName, vg); +} + +int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { + int64_t refId = pParamSet->refId; + + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq == NULL) { + taosMemoryFree(pParamSet); + terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; + return -1; + } + + // if no more waiting rsp + pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); + taosMemoryFree(pParamSet); + + taosReleaseRef(tmqMgmt.rsetId, refId); + return 0; +} + +void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + if (waitingRspNum == 0) { + tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, + vgId); + tmqCommitDone(pParamSet); + } else { + tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, + waitingRspNum); + } +} + +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + pRspObj->resIter++; + + if (pRspObj->resIter < pRspObj->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + if (pRspObj->rsp.withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + taosMemoryFreeClear(pRspObj->resInfo.row); + taosMemoryFreeClear(pRspObj->resInfo.pCol); + taosMemoryFreeClear(pRspObj->resInfo.length); + taosMemoryFreeClear(pRspObj->resInfo.convertBuf); + taosMemoryFreeClear(pRspObj->resInfo.convertJson); + } + + setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); + return &pRspObj->resInfo; + } + + return NULL; +} + +static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqVgWalInfoParam* pParam = param; + SMqVgCommon* pCommon = pParam->pCommon; + + int32_t total = atomic_add_fetch_32(&pCommon->numOfRsp, 1); + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, + pParam->vgId, pCommon->pTopicName); + pCommon->code = code; + } else { + SMqDataRsp rsp; + SDecoder decoder; + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); + tDecodeMqDataRsp(&decoder, &rsp); + tDecoderClear(&decoder); + + SMqRspHead* pHead = pMsg->pData; + + tmq_topic_assignment assignment = {.begin = pHead->walsver, + .end = pHead->walever, + .currentOffset = rsp.rspOffset.version, + .vgId = pParam->vgId}; + + taosThreadMutexLock(&pCommon->mutex); + taosArrayPush(pCommon->pList, &assignment); + taosThreadMutexUnlock(&pCommon->mutex); + } + + if (total == pParam->totalReq) { + tsem_post(&pCommon->rsp); + } + + taosMemoryFree(pParam); + return 0; +} + +static void destroyCommonInfo(SMqVgCommon* pCommon) { + taosArrayDestroy(pCommon->pList); + tsem_destroy(&pCommon->rsp); + taosThreadMutexDestroy(&pCommon->mutex); + taosMemoryFree(pCommon->pTopicName); + taosMemoryFree(pCommon); +} + +int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, + int32_t* numOfAssignment) { + *numOfAssignment = 0; + *assignment = NULL; + + int32_t accId = tmq->pTscObj->acctId; + char tname[128] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + // in case of snapshot is opened, no valid offset will return + *numOfAssignment = taosArrayGetSize(pTopic->vgs); + + *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); + if (*assignment == NULL) { + tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, + (*numOfAssignment) * sizeof(tmq_topic_assignment)); + return TSDB_CODE_OUT_OF_MEMORY; + } + + bool needFetch = false; + + for (int32_t j = 0; j < (*numOfAssignment); ++j) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + if (!pClientVg->receiveInfo) { + needFetch = true; + break; + } + + tmq_topic_assignment* pAssignment = &(*assignment)[j]; + if (pClientVg->offsetInfo.currentOffset.type == TMQ_OFFSET__LOG) { + pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version; + } else { + pAssignment->currentOffset = 0; + } + + pAssignment->begin = pClientVg->offsetInfo.walVerBegin; + pAssignment->end = pClientVg->offsetInfo.walVerEnd; + pAssignment->vgId = pClientVg->vgId; + } + + if (needFetch) { + SMqVgCommon* pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); + if (pCommon == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment)); + tsem_init(&pCommon->rsp, 0, 0); + taosThreadMutexInit(&pCommon->mutex, 0); + pCommon->pTopicName = taosStrdup(pTopic->topicName); + pCommon->consumerId = tmq->consumerId; + + terrno = TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < (*numOfAssignment); ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + + SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); + if (pParam == NULL) { + destroyCommonInfo(pCommon); + return terrno; + } + + pParam->epoch = tmq->epoch; + pParam->vgId = pClientVg->vgId; + pParam->totalReq = *numOfAssignment; + pParam->pCommon = pCommon; + + SMqPollReq req = {0}; + tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg); + + int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); + if (msgSize < 0) { + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + char* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pParam); + taosMemoryFree(msg); + destroyCommonInfo(pCommon); + return terrno; + } + + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; + sendInfo->requestId = req.reqId; + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqGetWalInfoCb; + sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; + + int64_t transporterId = 0; + char offsetFormatBuf[80]; + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); + + tscDebug("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); + } + + tsem_wait(&pCommon->rsp); + int32_t code = pCommon->code; + + terrno = code; + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(*assignment); + *assignment = NULL; + *numOfAssignment = 0; + } else { + int32_t num = taosArrayGetSize(pCommon->pList); + for(int32_t i = 0; i < num; ++i) { + (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i); + } + *numOfAssignment = num; + } + + destroyCommonInfo(pCommon); + return code; + } else { + return TSDB_CODE_SUCCESS; + } } + +int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[128] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + return TSDB_CODE_INVALID_PARA; + } + + SMqClientVg* pVg = NULL; + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + pVg = pClientVg; + break; + } + } + + if (pVg == NULL) { + tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + return TSDB_CODE_INVALID_PARA; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + + int32_t type = pOffsetInfo->currentOffset.type; + if (type != TMQ_OFFSET__LOG) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); + return TSDB_CODE_INVALID_PARA; + } + + if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { + tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64, tmq->consumerId, offset); + return TSDB_CODE_INVALID_PARA; + } + + // update the offset, and then commit to vnode + if (pOffsetInfo->currentOffset.type == TMQ_OFFSET__LOG) { + pOffsetInfo->currentOffset.version = offset; + pOffsetInfo->committedOffset.version = INT64_MIN; + } + + SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId}; + tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic)); + + tscDebug("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + if (pInfo == NULL) { + tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo); + + tsem_wait(&pInfo->sem); + int32_t code = pInfo->code; + + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId, + tstrerror(code)); + } + + return code; +} \ No newline at end of file diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index a8c9f2279d73f2cedf1c698d703c5591885a431e..894c51d13c9dd8e4205763893206de7ef86cdf7c 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( TAOS_RES *tres = (TAOS_RES *)res; - void *data; - int32_t numOfRows; + void *data = NULL; + int32_t numOfRows = 0; int error_code = taos_fetch_raw_block(tres, &numOfRows, &data); if (numOfRows == 0) { if (error_code == JNI_SUCCESS) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2f3d60001944365451e312baf48097169f4587bf..f15a93cb2c300587bfffe880b8a45bf0aef60175 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -30,6 +30,27 @@ #include "taos.h" namespace { + +void printSubResults(void* pRes, int32_t* totalRows) { + char buf[1024]; + + while (1) { + TAOS_ROW row = taos_fetch_row(pRes); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_field_count(pRes); + int32_t precision = taos_result_precision(pRes); + taos_print_row(buf, row, fields, numOfFields); + *totalRows += 1; + printf("precision: %d, row content: %s\n", precision, buf); + } + +// taos_free_result(pRes); +} + void showDB(TAOS* pConn) { TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -52,15 +73,15 @@ void printResult(TAOS_RES* pRes) { int32_t n = 0; char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t* length = taos_fetch_lengths(pRes); -// for(int32_t i = 0; i < numOfFields; ++i) { -// printf("(%d):%d " , i, length[i]); -// } -// printf("\n"); -// -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// memset(str, 0, sizeof(str)); + // int32_t* length = taos_fetch_lengths(pRes); + // for(int32_t i = 0; i < numOfFields; ++i) { + // printf("(%d):%d " , i, length[i]); + // } + // printf("\n"); + // + // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + // printf("%s\n", str); + // memset(str, 0, sizeof(str)); } } @@ -112,7 +133,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for(int32_t i = 0; i < 2000; i += 20) { + for (int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -131,7 +152,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } } -void *queryThread(void *arg) { +void* queryThread(void* arg) { TAOS* pConn = taos_connect("192.168.0.209", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to db, reason:%s", taos_errstr(pConn)); @@ -141,9 +162,10 @@ void *queryThread(void *arg) { int64_t el = 0; for (int32_t i = 0; i < 5000000; ++i) { - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); TAOS_RES* pRes = taos_query(pConn, - "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= 1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); + "SELECT _wstart as ts,max(usage_user) FROM benchmarkcpu.host_49 WHERE ts >= " + "1451618560000 AND ts < 1451622160000 INTERVAL(1m) ;"); if (taos_errno(pRes) != 0) { printf("failed, reason:%s\n", taos_errstr(pRes)); } else { @@ -153,7 +175,7 @@ void *queryThread(void *arg) { taos_free_result(pRes); el += (taosGetTimestampUs() - st); if (i % 1000 == 0 && i != 0) { - printf("total:%d, avg time:%.2fms\n", i, el/(double)(i*1000)); + printf("total:%d, avg time:%.2fms\n", i, el / (double)(i * 1000)); } } @@ -161,10 +183,84 @@ void *queryThread(void *arg) { return NULL; } -static int32_t numOfThreads = 1; +int32_t numOfThreads = 1; + +void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { + printf("auto commit success, code:%d\n\n\n\n", code); +} + +void* doConsumeData(void* param) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + tmq_conf_set(conf, "group.id", "cgrpName41"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, "topic_t2"); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + + tmq_list_destroy(topicList); + + TAOS_FIELD* fields = NULL; + int32_t numOfFields = 0; + int32_t precision = 0; + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 25000; + + int32_t count = 0; + + while (1) { + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes) { + char buf[1024]; + + const char* topicName = tmq_get_topic_name(pRes); + const char* dbName = tmq_get_db_name(pRes); + int32_t vgroupId = tmq_get_vgroup_id(pRes); + + printf("topic: %s\n", topicName); + printf("db: %s\n", dbName); + printf("vgroup id: %d\n", vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(pRes); + if (row == NULL) { + break; + } + + fields = taos_fetch_fields(pRes); + numOfFields = taos_field_count(pRes); + precision = taos_result_precision(pRes); + taos_print_row(buf, row, fields, numOfFields); + totalRows += 1; + // printf("precision: %d, row content: %s\n", precision, buf); + } + + taos_free_result(pRes); + } else { + break; + } + } -void tmq_commit_cb_print(tmq_t *pTmq, int32_t code, void *param) { - printf("success, code:%d\n", code); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return NULL; } } // namespace @@ -188,7 +284,6 @@ TEST(clientCase, driverInit_Test) { TEST(clientCase, connect_Test) { taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); @@ -418,17 +513,17 @@ TEST(clientCase, create_stable_Test) { ASSERT_EQ(numOfFields, 0); taos_free_result(pRes); -// pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } -// -// pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// pRes = taos_query(pConn, "drop stable `123_$^)`"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); -// } + // pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + // if (taos_errno(pRes) != 0) { + // printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + // } + // + // pRes = taos_query(pConn, "use abc1"); + // taos_free_result(pRes); + // pRes = taos_query(pConn, "drop stable `123_$^)`"); + // if (taos_errno(pRes) != 0) { + // printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + // } taos_close(pConn); } @@ -593,8 +688,7 @@ TEST(clientCase, create_multiple_tables) { for (int32_t i = 0; i < 500; i += 2) { char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1); + snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5)", i, i + 1); TAOS_RES* pres = taos_query(pConn, sql); if (taos_errno(pres) != 0) { printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); @@ -628,7 +722,7 @@ TEST(clientCase, show_table_Test) { int32_t numOfFields = taos_num_fields(pRes); int32_t count = 0; - char str[512] = {0}; + char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { int32_t code = taos_print_row(str, pRow, pFields, numOfFields); @@ -639,30 +733,30 @@ TEST(clientCase, show_table_Test) { taos_close(pConn); } -//TEST(clientCase, drop_stable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != nullptr); +// TEST(clientCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != nullptr); // -// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); // -// pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("error in using db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); // -// pRes = taos_query(pConn, "drop stable st1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); -// } +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } // -// taos_free_result(pRes); -// taos_close(pConn); -//} +// taos_free_result(pRes); +// taos_close(pConn); +// } TEST(clientCase, generated_request_id_test) { SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); @@ -671,7 +765,7 @@ TEST(clientCase, generated_request_id_test) { uint64_t v = generateRequestId(); void* result = taosHashGet(phash, &v, sizeof(v)); if (result != nullptr) { -// printf("0x%llx, index:%d\n", v, i); + // printf("0x%llx, index:%d\n", v, i); } assert(result == nullptr); taosHashPut(phash, &v, sizeof(v), NULL, 0); @@ -708,7 +802,7 @@ TEST(clientCase, projection_query_tables) { // } // taos_free_result(pRes); - TAOS_RES* pRes = taos_query(pConn, "use abc2"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); @@ -730,7 +824,7 @@ TEST(clientCase, projection_query_tables) { } taos_free_result(pRes); - for (int32_t i = 0; i < 2; ++i) { + for (int32_t i = 0; i < 1; ++i) { printf("create table :%d\n", i); createNewTable(pConn, i); } @@ -759,7 +853,7 @@ TEST(clientCase, projection_query_tables) { TEST(clientCase, tsbs_perf_test) { TdThread qid[20] = {0}; - for(int32_t i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { taosThreadCreate(&qid[i], NULL, queryThread, NULL); } getchar(); @@ -785,8 +879,8 @@ TEST(clientCase, projection_query_stables) { char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); + // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + // printf("%s\n", str); } taos_free_result(pRes); @@ -825,7 +919,8 @@ use test; create table m1(ts timestamp, k int) tags(a int); create table tm0 using m1 tags(1); create table tm1 using m1 tags(2); -insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22); +insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 +1:1:2.99', 22); */ TEST(clientCase, async_api_test) { @@ -839,7 +934,7 @@ TEST(clientCase, async_api_test) { printf("failed, reason:%s\n", taos_errstr(pRes)); } - int32_t n = 0; + int32_t n = 0; TAOS_ROW pRow = NULL; TAOS_FIELD* pFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes); @@ -847,8 +942,8 @@ TEST(clientCase, async_api_test) { char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { int32_t* length = taos_fetch_lengths(pRes); - for(int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d " , i, length[i]); + for (int32_t i = 0; i < numOfFields; ++i) { + printf("(%d):%d ", i, length[i]); } printf("\n"); @@ -890,15 +985,15 @@ TEST(clientCase, update_test) { taos_free_result(pRes); - char s[256] = {0}; - for(int32_t i = 0; i < 17000; ++i) { + char s[256] = {0}; + for (int32_t i = 0; i < 17000; ++i) { sprintf(s, "insert into tup values(now+%da, %d)", i, i); pRes = taos_query(pConn, s); taos_free_result(pRes); } } -TEST(clientCase, subscription_test) { +TEST(clientCase, sub_db_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -912,11 +1007,11 @@ TEST(clientCase, subscription_test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName"); + tmq_conf_set(conf, "group.id", "cgrpNamedb"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "experimental.snapshot.enable", "false"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -925,7 +1020,8 @@ TEST(clientCase, subscription_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); -// tmq_list_append(topicList, "topic_t1"); + tmq_list_append(topicList, "topic_t1"); +// tmq_list_append(topicList, "topic_s2"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -954,7 +1050,7 @@ TEST(clientCase, subscription_test) { printf("db: %s\n", dbName); printf("vgroup id: %d\n", vgroupId); - if (count ++ > 20) { + if (count++ > 200) { tmq_unsubscribe(tmq); break; } @@ -970,11 +1066,103 @@ TEST(clientCase, subscription_test) { taos_print_row(buf, row, fields, numOfFields); printf("precision: %d, row content: %s\n", precision, buf); } + taos_free_result(pRes); } -// return rows; } fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +TEST(clientCase, sub_tb_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + tmq_conf_set(conf, "group.id", "cgrpName1024"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "experimental.snapshot.enable", "false"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + + // 创建订阅 topics 列表 + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, "topic_t1"); + + // 启动订阅 + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + TAOS_FIELD* fields = NULL; + int32_t numOfFields = 0; + int32_t precision = 0; + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 25000; + + int32_t count = 0; + + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + + while (1) { + TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); + if (pRes != NULL) { + const char* topicName = tmq_get_topic_name(pRes); + const char* dbName = tmq_get_db_name(pRes); + int32_t vgroupId = tmq_get_vgroup_id(pRes); + + printf("topic: %s\n", topicName); + printf("db: %s\n", dbName); + printf("vgroup id: %d\n", vgroupId); + + printSubResults(pRes, &totalRows); + } else { +// tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgroupHandle, pAssign[0].begin); +// break; + } + + tmq_commit_sync(tmq, pRes); + if (pRes != NULL) { + taos_free_result(pRes); + } + + tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, pAssign[0].begin); + } + + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + +TEST(clientCase, sub_tb_mt_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + TdThread qid[20] = {0}; + + for (int32_t i = 0; i < 1; ++i) { + taosThreadCreate(&qid[i], NULL, doConsumeData, NULL); + } + + for (int32_t i = 0; i < 4; ++i) { + taosThreadJoin(qid[i], NULL); + } +} + #pragma GCC diagnostic pop diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 4aa47a381966c225da16a2f48a555833558522de..cd3dd63ef04b56ca1ea933bbb8c3053da8917f4f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -114,6 +114,9 @@ static const SSysDbTableSchema userFuncSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "func_language", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "func_body", .bytes = TSDB_MAX_BINARY_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "func_version", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, }; static const SSysDbTableSchema userIdxSchema[] = { @@ -233,6 +236,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "cacheelements", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, // {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, }; @@ -295,7 +299,9 @@ static const SSysDbTableSchema vnodesSchema[] = { static const SSysDbTableSchema userUserPrivilegesSchema[] = { {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 453e4a8d8e5c13a7bd503bd491c2b998b4d7d286..f4d2ed01b079b11ff00c7d021a918f5dbe543240 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -147,9 +147,17 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { return TSDB_CODE_SUCCESS; } -static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, - int32_t itemLen, int32_t numOfRows) { - ASSERT(pColumnInfoData->info.bytes >= itemLen); +static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, + int32_t itemLen, int32_t numOfRows, bool trimValue) { + if (pColumnInfoData->info.bytes < itemLen) { + uWarn("column/tag actual data len %d is bigger than schema len %d, trim it:%d", itemLen, pColumnInfoData->info.bytes, trimValue); + if (trimValue) { + itemLen = pColumnInfoData->info.bytes; + } else { + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } + } + size_t start = 1; // the first item @@ -178,10 +186,12 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren pColumnInfoData->varmeta.length += numOfRows * itemLen; } + + return TSDB_CODE_SUCCESS; } int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, - uint32_t numOfRows) { + uint32_t numOfRows, bool trimValue) { int32_t len = pColumnInfoData->info.bytes; if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { len = varDataTLen(pData); @@ -193,8 +203,7 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, } } - doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows); - return TSDB_CODE_SUCCESS; + return doCopyNItems(pColumnInfoData, currentRow, pData, len, numOfRows, trimValue); } static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, @@ -212,6 +221,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } uint8_t* p = (uint8_t*)pSource->nullbitmap; + pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] &= (0B11111111 << shiftBits); // clear remind bits pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits if (BitmapLen(numOfRow1) == BitmapLen(total)) { @@ -223,6 +233,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + memset(start, 0, overCount); while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { start[i - 1] |= (p[i] >> remindBits); // copy remind bits @@ -300,9 +311,11 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->pData = tmp; if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + if (btmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); memset(btmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = btmp; } @@ -973,7 +986,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); - uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); + uDebug("blockDataSort easy cost:%" PRId64 ", rows:%" PRId64 "\n", p1 - p0, pDataBlock->info.rows); return TSDB_CODE_SUCCESS; } else { // var data type @@ -1026,6 +1039,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } +#if 0 typedef struct SHelper { int32_t index; union { @@ -1074,59 +1088,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; - // SSDataBlock* pDataBlock = pHelper->pDataBlock; - SHelper* left = (SHelper*)p1; SHelper* right = (SHelper*)p2; SArray* pInfo = pHelper->orderInfo; int32_t offset = 0; - // for(int32_t i = 0; i < pInfo->size; ++i) { - // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); - // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); - - // if (pColInfoData->hasNull) { - // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); - // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); - // if (leftNull && rightNull) { - // continue; // continue to next slot - // } - // - // if (rightNull) { - // return pHelper->nullFirst? 1:-1; - // } - // - // if (leftNull) { - // return pHelper->nullFirst? -1:1; - // } - // } - - // void* left1 = colDataGetData(pColInfoData, left); - // void* right1 = colDataGetData(pColInfoData, right); - - // switch(pColInfoData->info.type) { - // case TSDB_DATA_TYPE_INT: { int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); - // offset += pColInfoData->info.bytes; if (leftx == rightx) { - // break; return 0; } else { - // if (pOrder->order == TSDB_ORDER_ASC) { return (leftx < rightx) ? -1 : 1; - // } else { - // return (leftx < rightx)? 1:-1; - // } - } - // } - // default: - // assert(0); - // } - // } - + } return 0; } @@ -1170,6 +1145,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF // destroyTupleIndex(index); return 0; } +#endif void blockDataCleanup(SSDataBlock* pDataBlock) { blockDataEmpty(pDataBlock); @@ -1180,7 +1156,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { + if (pInfo->capacity == 0) { return; } @@ -1618,12 +1594,13 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { i += 1; } } else if (n > 8) { - int32_t gap = len - newLen; + int32_t remain = (total % 8 != 0 && total % 8 <= tail) ? 1 : 0; + int32_t gap = len - newLen - remain; while (i < newLen) { uint8_t v = p[i + gap]; p[i] = (v << tail); - if (i < newLen - 1) { + if (i < newLen - 1 + remain) { uint8_t next = p[i + gap + 1]; p[i] |= (next >> (8 - tail)); } @@ -1739,14 +1716,14 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.id.uid; int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; - int32_t rows = pBlock->info.rows; + int64_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, tbUid); tlen += taosEncodeFixedI16(buf, numOfCols); tlen += taosEncodeFixedI16(buf, hasVarCol); - tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI64(buf, rows); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); @@ -1777,7 +1754,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid); buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); - buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI64(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); for (int32_t i = 0; i < sz; i++) { @@ -1864,7 +1841,9 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } } struct tm ptm = {0}; - taosLocalTime(&tt, &ptm); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { @@ -1878,6 +1857,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } +#if 0 void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) { SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock*)); taosArrayPush(dataBlocks, &pBlock); @@ -1970,6 +1950,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { } } +#endif + // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { int32_t size = 2048; @@ -1981,7 +1963,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", + "|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); @@ -2044,8 +2026,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_DOUBLE: - // len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); - // if (len >= size - 1) return dumpBuf; + len += snprintf(dumpBuf + len, size - len, " %15f |", *(double*)var); + if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_BOOL: len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 2bb870837272a0422de500d56e89c7ca245a7beb..f379084cf558a6e9d4225f52163beb1cfea8a8ce 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2439,6 +2439,12 @@ _exit: int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, char *data) { int32_t code = 0; + if(data == NULL){ + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + } + goto _exit; + } if (IS_VAR_DATA_TYPE(type)) { // var-length data type for (int32_t i = 0; i < nRows; ++i) { @@ -2447,6 +2453,11 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { + if(ASSERT(varDataTLen(data + offset) <= bytes)){ + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + code = TSDB_CODE_INVALID_PARA; + goto _exit; + } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), varDataLen(data + offset)); } @@ -2492,9 +2503,11 @@ _exit: int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { int32_t code = 0; - ASSERT(pColData->type == pBind->buffer_type); - - if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { + ASSERT(pColData->type == pBind->buffer_type); + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8058f9fddd6ee29e2b217f059255190493093378..da4a91223815af34c5dfb679046a9e81bbe46edc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -20,6 +20,10 @@ #include "tlog.h" #include "tmisce.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; @@ -99,7 +103,9 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table // query int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; +int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT bool tsEnableQueryHb = false; +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -112,6 +118,7 @@ int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -154,6 +161,7 @@ char tsTagFilterCache = 0; // positive value (in MB) int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; +int32_t tsCacheLazyLoadThreshold = 500; int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; @@ -227,7 +235,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); if (taosIsDir(cfgDir)) { +#ifdef CUS_PROMPT + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT); +#else snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); +#endif } else { tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } @@ -318,6 +330,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; @@ -331,6 +344,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -497,6 +511,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + GRANT_CFG_ADD; return 0; } @@ -717,6 +733,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; @@ -724,6 +741,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; + tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; @@ -824,6 +842,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } + tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32; + tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; GRANT_CFG_GET; @@ -1233,13 +1253,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi } if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { - uError("failed to load cfg since %s", terrstr()); + printf("failed to load cfg since %s", terrstr()); cfgCleanup(pCfg); return -1; } if (cfgLoadFromArray(pCfg, pArgs) != 0) { - uError("failed to load cfg from array since %s", terrstr()); + printf("failed to load cfg from array since %s", terrstr()); cfgCleanup(pCfg); return -1; } @@ -1255,13 +1275,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (taosMulModeMkDir(tsLogDir, 0777) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create dir:%s since %s", tsLogDir, terrstr()); + printf("failed to create dir:%s since %s", tsLogDir, terrstr()); cfgCleanup(pCfg); return -1; } if (taosInitLog(logname, logFileNum) != 0) { - uError("failed to init log file since %s", terrstr()); + printf("failed to init log file since %s", terrstr()); cfgCleanup(pCfg); return -1; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6dd28094cb4227470942d992b1279838658841bf..4139d6c7d449ec4b5c529cd2b0b6814258a9e380 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1070,7 +1070,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1; if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1; - if (tEncodeI64(&encoder, reserved) < 0) return -1; + if (tEncodeI32(&encoder, pload->numOfCachedTables) < 0) return -1; + if (tEncodeI32(&encoder, reserved) < 0) return -1; if (tEncodeI64(&encoder, reserved) < 0) return -1; if (tEncodeI64(&encoder, reserved) < 0) return -1; } @@ -1148,7 +1149,8 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; - if (tDecodeI64(&decoder, &reserved) < 0) return -1; + if (tDecodeI32(&decoder, &vload.numOfCachedTables) < 0) return -1; + if (tDecodeI32(&decoder, (int32_t *)&reserved) < 0) return -1; if (tDecodeI64(&decoder, &reserved) < 0) return -1; if (tDecodeI64(&decoder, &reserved) < 0) return -1; if (taosArrayPush(pReq->pVloads, &vload) == NULL) { @@ -1366,6 +1368,12 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1; + int32_t len = strlen(pReq->tabName); + if (tEncodeI32(&encoder, len) < 0) return -1; + if (len > 0) { + if (tEncodeCStr(&encoder, pReq->tabName) < 0) return -1; + } + if (tEncodeBinary(&encoder, pReq->tagCond, pReq->tagCondLen) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1385,6 +1393,16 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1; + if (!tDecodeIsEnd(&decoder)) { + int32_t len = 0; + if (tDecodeI32(&decoder, &len) < 0) return -1; + if (len > 0) { + if (tDecodeCStrTo(&decoder, pReq->tabName) < 0) return -1; + } + uint64_t tagCondLen = 0; + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->tagCond, &tagCondLen) < 0) return -1; + pReq->tagCondLen = tagCondLen; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -1427,6 +1445,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; @@ -1449,6 +1468,54 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) db = taosHashIterate(pRsp->writeDbs, db); } + int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs); + int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs); + int32_t numOfUseTbs = taosHashGetSize(pRsp->useDbs); + if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfUseTbs) < 0) return -1; + + char *tb = taosHashIterate(pRsp->readTbs, NULL); + while (tb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(tb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + size_t valueLen = 0; + valueLen = strlen(tb); + if (tEncodeI32(pEncoder, valueLen) < 0) return -1; + if (tEncodeCStr(pEncoder, tb) < 0) return -1; + + tb = taosHashIterate(pRsp->readTbs, tb); + } + + tb = taosHashIterate(pRsp->writeTbs, NULL); + while (tb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(tb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + size_t valueLen = 0; + valueLen = strlen(tb); + if (tEncodeI32(pEncoder, valueLen) < 0) return -1; + if (tEncodeCStr(pEncoder, tb) < 0) return -1; + + tb = taosHashIterate(pRsp->writeTbs, tb); + } + + int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + if (tEncodeI32(pEncoder, *useDb) < 0) return -1; + useDb = taosHashIterate(pRsp->useDbs, useDb); + } + return 0; } @@ -1471,7 +1538,11 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { + pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || + pRsp->writeTbs == NULL || pRsp->useDbs == NULL) { return -1; } @@ -1510,6 +1581,63 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs taosHashPut(pRsp->writeDbs, db, len, db, len); } + if (!tDecodeIsEnd(pDecoder)) { + int32_t numOfReadTbs = 0; + int32_t numOfWriteTbs = 0; + int32_t numOfUseDbs = 0; + if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfReadTbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t valuelen = 0; + if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; + char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + + taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfWriteTbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t valuelen = 0; + if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; + char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + + taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfUseDbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t ref = 0; + if (tDecodeI32(pDecoder, &ref) < 0) return -1; + taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); + } + } + return 0; } @@ -1531,6 +1659,9 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); + taosHashCleanup(pRsp->writeTbs); + taosHashCleanup(pRsp->readTbs); + taosHashCleanup(pRsp->useDbs); } int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { @@ -1700,6 +1831,9 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; } + + if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1742,6 +1876,13 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; } + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1; + } else { + pReq->orReplace = false; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -1853,6 +1994,13 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * } } + if (pRsp->numOfFuncs != (int32_t)taosArrayGetSize(pRsp->pFuncExtraInfos)) return -1; + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo *extraInfo = taosArrayGet(pRsp->pFuncExtraInfos, i); + if (tEncodeI32(&encoder, extraInfo->funcVersion) < 0) return -1; + if (tEncodeI64(&encoder, extraInfo->funcCreatedTime) < 0) return -1; + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1900,6 +2048,22 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp taosArrayPush(pRsp->pFuncInfos, &fInfo); } + + pRsp->pFuncExtraInfos = taosArrayInit(pRsp->numOfFuncs, sizeof(SFuncExtraInfo)); + if (pRsp->pFuncExtraInfos == NULL) return -1; + if (tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo extraInfo = { 0 }; + taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); + } + } else { + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo extraInfo = { 0 }; + if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1; + if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1; + taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); + } + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -1922,6 +2086,7 @@ void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { tFreeSFuncInfo(pInfo); } taosArrayDestroy(pRsp->pFuncInfos); + taosArrayDestroy(pRsp->pFuncExtraInfos); } int32_t tSerializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq) { @@ -2219,6 +2384,12 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; if (tEncodeI32(&encoder, pReq->sstTrigger) < 0) return -1; + + // 1st modification + if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; + // 2nd modification + if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2246,6 +2417,22 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; if (tDecodeI32(&decoder, &pReq->sstTrigger) < 0) return -1; + + // 1st modification + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; + } else { + pReq->minRows = -1; + } + + // 2nd modification + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pReq->walRetentionPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walRetentionSize) < 0) return -1; + } else { + pReq->walRetentionPeriod = -1; + pReq->walRetentionSize = -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2749,6 +2936,59 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp *pUserPassRsp = taosArrayGet(pRsp->pArray, i); + if (tEncodeCStr(&encoder, pUserPassRsp->user) < 0) return -1; + if (tEncodeI32(&encoder, pUserPassRsp->version) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserPassRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp rsp = {0}; + if (tDecodeCStrTo(&decoder, rsp.user) < 0) return -1; + if (tDecodeI32(&decoder, &rsp.version) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSUserPassBatchRsp(SUserPassBatchRsp *pRsp) { + if(pRsp) { + taosArrayDestroy(pRsp->pArray); + } +} + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3789,6 +4029,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->svrTimestamp) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3812,6 +4053,13 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->svrTimestamp) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->passVer) < 0) return -1; + } else { + pRsp->passVer = 0; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3942,6 +4190,12 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); @@ -4020,6 +4274,14 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4172,6 +4434,13 @@ int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeCon for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + + // 1st modification + if (tEncodeI16(&encoder, pReq->sttTrigger) < 0) return -1; + if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; + // 2nd modification + if (tEncodeI32(&encoder, pReq->walRetentionPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pReq->walRetentionSize) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4201,6 +4470,23 @@ int32_t tDeserializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeC if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } + // 1st modification + if (tDecodeIsEnd(&decoder)) { + pReq->sttTrigger = -1; + pReq->minRows = -1; + } else { + if (tDecodeI16(&decoder, &pReq->sttTrigger) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; + } + + // 2n modification + if (tDecodeIsEnd(&decoder)) { + pReq->walRetentionPeriod = -1; + pReq->walRetentionSize = -1; + } else { + if (tDecodeI32(&decoder, &pReq->walRetentionPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->walRetentionSize) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -4222,6 +4508,12 @@ int32_t tSerializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnodeRe for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4245,7 +4537,15 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -4413,6 +4713,31 @@ int32_t tDeserializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupR return 0; } +int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->useless) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4496,6 +4821,31 @@ int32_t tDeserializeSSplitVgroupReq(void *buf, int32_t bufLen, SSplitVgroupReq * return 0; } +int32_t tSerializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSForceBecomeFollowerReq(void *buf, int32_t bufLen, SForceBecomeFollowerReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4506,6 +4856,12 @@ int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq * SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + if (tEncodeI64(&encoder, pReq->lastIndex) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4523,6 +4879,14 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + if (tDecodeI64(&decoder, &pReq->lastIndex) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -6612,8 +6976,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) { snprintf(buf, maxLen, "offset(snapshot) uid:%" PRId64 " ts:%" PRId64, pVal->uid, pVal->ts); } else { - ASSERT(0); + return TSDB_CODE_INVALID_PARA; } + return 0; } @@ -6647,6 +7012,18 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { return 0; } +int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset) { + if (tEncodeSTqOffset(pEncoder, &pOffset->offset) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->consumerId) < 0) return -1; + return 0; +} + +int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset) { + if (tDecodeSTqOffset(pDecoder, &pOffset->offset) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->consumerId) < 0) return -1; + return 0; +} + int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->topic) < 0) return -1; if (tEncodeI64(pEncoder, pInfo->ntbUid) < 0) return -1; @@ -6711,21 +7088,21 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { return 0; } -int32_t tEncodeSMqMetaRsp(SEncoder *pEncoder, const SMqMetaRsp *pRsp) { +int32_t tEncodeMqMetaRsp(SEncoder *pEncoder, const SMqMetaRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI16(pEncoder, pRsp->resMsgType)) return -1; if (tEncodeBinary(pEncoder, pRsp->metaRsp, pRsp->metaRspLen)) return -1; return 0; } -int32_t tDecodeSMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { +int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI16(pDecoder, &pRsp->resMsgType) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &pRsp->metaRsp, (uint64_t *)&pRsp->metaRspLen) < 0) return -1; return 0; } -int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -6750,7 +7127,7 @@ int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { return 0; } -int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -6795,9 +7172,8 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } -void tDeleteSMqDataRsp(SMqDataRsp *pRsp) { - taosArrayDestroy(pRsp->blockDataLen); - pRsp->blockDataLen = NULL; +void tDeleteMqDataRsp(SMqDataRsp *pRsp) { + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); @@ -6897,8 +7273,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { } void tDeleteSTaosxRsp(STaosxRsp *pRsp) { - taosArrayDestroy(pRsp->blockDataLen); - pRsp->blockDataLen = NULL; + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); @@ -6906,8 +7281,7 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) { taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); pRsp->blockTbName = NULL; - taosArrayDestroy(pRsp->createTableLen); - pRsp->createTableLen = NULL; + pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; } @@ -7185,6 +7559,7 @@ void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { tDestroySSubmitTbData(&aSubmitTbData[i], flag); } taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; } int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { @@ -7279,3 +7654,41 @@ void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { } } } + +int32_t tEncodeMqSubTopicEp(void **buf, const SMqSubTopicEp *pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeString(buf, pTopicEp->db); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp *pVgEp = (SMqSubVgEp *)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; +} + +void *tDecodeMqSubTopicEp(void *buf, SMqSubTopicEp *pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeStringTo(buf, pTopicEp->db); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; +} + +void tDeleteMqSubTopicEp(SMqSubTopicEp *pSubTopicEp) { + taosMemoryFreeClear(pSubTopicEp->schema.pSchema); + pSubTopicEp->schema.nCols = 0; + taosArrayDestroy(pSubTopicEp->vgs); +} \ No newline at end of file diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 7996498d451f95df794fab75c644eed044d37c4b..dcd539bd91e11c570a9f779f9c1c9a3811765c41 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -727,7 +727,7 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { struct tm tm; time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); - taosLocalTime(&tt, &tm); + taosLocalTime(&tt, &tm, NULL); int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)numOfMonth; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -750,11 +750,11 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char struct tm tm; time_t t = (time_t)skey; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { @@ -774,7 +774,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; time_t tt = (time_t)start; - taosLocalTime(&tt, &tm); + taosLocalTime(&tt, &tm, NULL); tm.tm_sec = 0; tm.tm_min = 0; tm.tm_hour = 0; @@ -867,13 +867,17 @@ const char* fmtts(int64_t ts) { if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm); } if (ts > -62135625943000 && ts < 32503651200000) { time_t t = (time_t)(ts / 1000); - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } if (pos > 0) { buf[pos++] = ' '; buf[pos++] = '|'; @@ -885,7 +889,9 @@ const char* fmtts(int64_t ts) { { time_t t = (time_t)(ts / 1000000); - taosLocalTime(&t, &tm); + if (taosLocalTime(&t, &tm, buf) == NULL) { + return buf; + } if (pos > 0) { buf[pos++] = ' '; buf[pos++] = '|'; @@ -937,7 +943,9 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision) ASSERT(false); } - taosLocalTime(", &ptm); + if (taosLocalTime(", &ptm, buf) == NULL) { + return; + } int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", &ptm); length += snprintf(ts + length, fractionLen, format, mod); length += (int32_t)strftime(ts + length, 40 - length, "%z", &ptm); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index d3cffaa185382af4ddfa7db681a1b9357582fb96..f0e020edfed8136e6b7fbfc3a023c1ef2822fd01 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -18,7 +18,23 @@ #include "mnode.h" #include "tconfig.h" #include "tglobal.h" +#include "version.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#else +#ifndef CUS_NAME + #define CUS_NAME "TDengine" +#endif + +#ifndef CUS_PROMPT + #define CUS_PROMPT "taos" +#endif + +#ifndef CUS_EMAIL + #define CUS_EMAIL "" +#endif +#endif // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." #define DM_CFG_DIR "Configuration directory." @@ -76,28 +92,28 @@ void dmLogCrash(int signum, void *sigInfo, void *context) { taosIgnSignal(SIGINT); taosIgnSignal(SIGBREAK); -#ifndef WINDOWS +#ifndef WINDOWS taosIgnSignal(SIGBUS); #endif taosIgnSignal(SIGABRT); taosIgnSignal(SIGFPE); taosIgnSignal(SIGSEGV); - char *pMsg = NULL; + char *pMsg = NULL; const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; int32_t dflag = 255; - int64_t msgLen= -1; - + int64_t msgLen = -1; + if (tsEnableCrashReport) { if (taosGenCrashJsonMsg(signum, &pMsg, dmGetClusterId(), global.startTime)) { taosPrintLog(flags, level, dflag, "failed to generate crash json msg"); goto _return; } else { - msgLen = strlen(pMsg); + msgLen = strlen(pMsg); } } - + _return: taosLogCrashInfo("taosd", pMsg, msgLen, signum, sigInfo); @@ -123,7 +139,7 @@ static void dmSetSignalHandle() { #ifndef WINDOWS taosSetSignal(SIGBUS, dmLogCrash); -#endif +#endif taosSetSignal(SIGABRT, dmLogCrash); taosSetSignal(SIGFPE, dmLogCrash); taosSetSignal(SIGSEGV, dmLogCrash); @@ -134,7 +150,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; if (argc < 2) return 0; - + global.envCmd = taosMemoryMalloc((argc - 1) * sizeof(char *)); memset(global.envCmd, 0, (argc - 1) * sizeof(char *)); for (int32_t i = 1; i < argc; ++i) { @@ -203,6 +219,9 @@ static void dmPrintVersion() { #endif printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); printf("gitinfo: %s\n", gitinfo); +#ifdef TD_ENTERPRISE + printf("gitinfoOfInternal: %s\n", gitinfoOfInternal); +#endif printf("buildInfo: %s\n", buildinfo); } @@ -228,7 +247,7 @@ static void dmDumpCfg() { } static int32_t dmInitLog() { - return taosCreateLog("taosdlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void taosCleanupArgs() { @@ -284,7 +303,7 @@ int mainWindows(int argc, char **argv) { printf("failed to init memory dbg, error:%s\n", tstrerror(code)); return code; } - tsAsyncLog = false; + tsAsyncLog = false; printf("memory dbg enabled\n"); } #endif diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 600a1f6829e11a1d38ffa262e7d360687c67b90d..6c40fa808fcaf008b3d4614594eb7bab262525f6 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -32,6 +32,7 @@ typedef struct SDnodeMgmt { TdThread crashReportThread; SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 228f301aec7712fce516ddad2d22f4b2823a36b0..19982698968c8c91bc3f53edd3868f173b070005 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -352,6 +352,7 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 51df293ba70bc37be14c763ecb4a5ea296077bd4..09783a5ea9495506392ec68f9ec5395fa4281d6e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -48,6 +48,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; + pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 0d3b423771267cc567a4cfc95f2b550c508342d3..06b6221940a4f0991fe520a58e2918409139380b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -227,6 +227,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_DROP_SNODE: code = (*pMgmt->processDropNodeFp)(SNODE, pMsg); break; + case TDMT_DND_ALTER_MNODE_TYPE: + code = (*pMgmt->processAlterNodeTypeFp)(MNODE, pMsg); + break; case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index b47742b4ed8db0cf55df46118b20a07bf00bd6bd..24f75e3c8b01dd41f958ec5913352b9baba991b7 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -34,7 +34,7 @@ typedef struct SMnodeMgmt { SSingleWorker readWorker; SSingleWorker writeWorker; SSingleWorker syncWorker; - SSingleWorker syncCtrlWorker; + SSingleWorker syncRdWorker; bool stopped; int32_t refCount; TdThreadRwlock lock; @@ -54,7 +54,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index f06669a610142a38b6b2937c95bd150c324df568..cb0849f4b96f945a41a2e27a0abcba2244484bba 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -24,12 +24,16 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); if (code < 0) return 0; + tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code); + if (code < 0) return 0; SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); if (replicas == NULL) return 0; - pOption->numOfReplicas = tjsonGetArraySize(replicas); + pOption->numOfTotalReplicas = tjsonGetArraySize(replicas); + + pOption->numOfReplicas = 0; - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonGetArrayItem(replicas, i); if (replica == NULL) return -1; @@ -40,6 +44,14 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); + if (code < 0) return -1; + if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){ + pOption->numOfReplicas++; + } + } + + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { } return 0; @@ -112,14 +124,14 @@ _OVER: } static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { - if (pOption->deploy && pOption->numOfReplicas > 0) { + if (pOption->deploy && pOption->numOfTotalReplicas > 0) { if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; SJson *replicas = tjsonCreateArray(); if (replicas == NULL) return -1; if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonCreateObject(); if (replica == NULL) return -1; @@ -127,10 +139,13 @@ static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; + if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1; if (tjsonAddItemToArray(replicas, replica) < 0) return -1; } } + if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; return 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index fe65c3dde96a0e81ab8211fa87aebc7e92ee0294..76fe0ec2decb3f17f9c5c9c82bcafd1b47af8f49 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -33,12 +33,24 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, + .numOfTotalReplicas = createReq.replica + createReq.learnerReplica, + .selfIndex = -1, .lastIndex = createReq.lastIndex}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); - for (int32_t i = 0; i < option.numOfReplicas; ++i) { + for (int32_t i = 0; i < createReq.replica; ++i) { if (createReq.replicas[i].id == pInput->pData->dnodeId) { option.selfIndex = i; } + option.nodeRoles[i] = TAOS_SYNC_ROLE_VOTER; + } + + memcpy(&(option.replicas[createReq.replica]), createReq.learnerReplicas, sizeof(createReq.learnerReplicas)); + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + if (createReq.learnerReplicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = createReq.replica + i; + } + option.nodeRoles[createReq.replica + i] = TAOS_SYNC_ROLE_LEARNER; } if (option.selfIndex == -1) { @@ -92,6 +104,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -126,6 +140,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SPLIT_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP_LEADER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -188,7 +203,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; @@ -198,11 +213,15 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncCtrlQueue, 1) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index e7f71ad4200bd2b1c3c5fcf10e82aa9aa0ea417f..7840528db9329f7b5c10449bee0d8a213e87ebe6 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,9 +45,11 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->dnodeId = pMgmt->pData->dnodeId; pOption->selfIndex = 0; pOption->numOfReplicas = 1; + pOption->numOfTotalReplicas = 1; pOption->replicas[0].id = 1; pOption->replicas[0].port = tsServerPort; tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->lastIndex = SYNC_INDEX_INVALID; } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { @@ -123,9 +125,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (option.numOfReplicas > 0) { + if (option.numOfTotalReplicas > 0) { option.deploy = true; option.numOfReplicas = 0; + option.numOfTotalReplicas = 0; if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; @@ -152,6 +155,14 @@ static void mmStop(SMnodeMgmt *pMgmt) { mndStop(pMgmt->pMnode); } +static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { + return mndIsCatchUp(pMgmt->pMnode); +} + +static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { + return mndGetRole(pMgmt->pMnode); +} + SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = mmOpen; @@ -162,6 +173,8 @@ SMgmtFunc mmGetMgmtFunc() { mgmtFunc.dropFp = (NodeDropFp)mmProcessDropReq; mgmtFunc.requiredFp = mmRequire; mgmtFunc.getHandlesFp = mmGetMsgHandles; + mgmtFunc.isCatchUpFp = (NodeIsCatchUpFp)mmSyncIsCatchUp; + mgmtFunc.nodeRoleFp = (NodeRole)mmSyncGetRole; return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index b0810d528f761d6362b27a818636d52e8421429c..0152e5d0b1a02041b2d70961a7bf85e7a516dd82 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -111,8 +111,8 @@ int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -int32_t mmPutMsgToSyncCtrlQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutMsgToWorker(pMgmt, &pMgmt->syncCtrlWorker, pMsg); +int32_t mmPutMsgToSyncRdQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncRdWorker, pMsg); } int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { @@ -151,8 +151,8 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { case SYNC_QUEUE: pWorker = &pMgmt->syncWorker; break; - case SYNC_CTRL_QUEUE: - pWorker = &pMgmt->syncCtrlWorker; + case SYNC_RD_QUEUE: + pWorker = &pMgmt->syncRdWorker; break; default: terrno = TSDB_CODE_INVALID_PARA; @@ -238,12 +238,12 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { SSingleWorkerCfg scCfg = { .min = 1, .max = 1, - .name = "mnode-sync-ctrl", + .name = "mnode-sync-rd", .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; - if (tSingleWorkerInit(&pMgmt->syncCtrlWorker, &scCfg) != 0) { - dError("failed to start mnode mnode-sync-ctrl worker since %s", terrstr()); + if (tSingleWorkerInit(&pMgmt->syncRdWorker, &scCfg) != 0) { + dError("failed to start mnode mnode-sync-rd worker since %s", terrstr()); return -1; } @@ -259,6 +259,6 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); - tSingleWorkerCleanup(&pMgmt->syncCtrlWorker); + tSingleWorkerCleanup(&pMgmt->syncRdWorker); dDebug("mnode workers are closed"); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 28097311ac3530788d83c965ed4918d4621effc4..e2223497671362e1c8af19e3481c797e8a2b3972 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -55,6 +55,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { smClose(pMgmt); return -1; } + tmsgReportStartup("snode-impl", "initialized"); if (smStartWorker(pMgmt) != 0) { diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index e3fa2964b74b697cd507a9fddcb27a848010e2e1..83fb331dbd5fc368f39c8ab866426200c99ab6bc 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -59,7 +59,7 @@ typedef struct { SVnode *pImpl; SMultiWorker pWriteW; SMultiWorker pSyncW; - SMultiWorker pSyncCtrlW; + SMultiWorker pSyncRdW; SMultiWorker pApplyW; STaosQueue *pQueryQ; STaosQueue *pStreamQ; @@ -90,6 +90,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); // vmFile.c int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); @@ -107,7 +108,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmPutMsgToStreamQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d7f91b74a8d1827f512b85b2a7e91221306172a3..9dbc12cf622af70028629cc152c9f60593c21004 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -21,7 +21,7 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) { if (pInfo->pVloads == NULL) return; tfsUpdateSize(pMgmt->pTfs); - + taosThreadRwlockRdlock(&pMgmt->lock); void *pIter = taosHashIterate(pMgmt->hash, NULL); @@ -131,15 +131,34 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->standby = 0; - pCfg->syncCfg.myIndex = pCreate->selfIndex; - pCfg->syncCfg.replicaNum = pCreate->replica; + pCfg->syncCfg.replicaNum = 0; + pCfg->syncCfg.totalReplicaNum = 0; + memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); for (int32_t i = 0; i < pCreate->replica; ++i) { SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; - pNode->nodeId = pCreate->replicas[i].id; - pNode->nodePort = pCreate->replicas[i].port; - tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + pNode->nodeId = pCreate->replicas[pCfg->syncCfg.replicaNum].id; + pNode->nodePort = pCreate->replicas[pCfg->syncCfg.replicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + tstrncpy(pNode->nodeFqdn, pCreate->replicas[pCfg->syncCfg.replicaNum].fqdn, TSDB_FQDN_LEN); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.replicaNum++; + } + if(pCreate->selfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->selfIndex; + } + for (int32_t i = pCfg->syncCfg.replicaNum; i < pCreate->replica + pCreate->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + pNode->nodeId = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].id; + pNode->nodePort = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].fqdn, TSDB_FQDN_LEN); tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.totalReplicaNum++; + } + pCfg->syncCfg.totalReplicaNum += pCfg->syncCfg.replicaNum; + if(pCreate->learnerSelfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->replica + pCreate->learnerSelfIndex; } } @@ -182,23 +201,40 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 + if(req.learnerReplica == 0) + { + req.learnerSelfIndex = -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 ", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 - ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d", - req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, + ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " + "learnerReplica:%d learnerSelfIndex:%d strict:%d", + req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, + (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, - req.hashSuffix, req.replica, req.selfIndex, req.strict); + req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict); for (int32_t i = 0; i < req.replica; ++i) { dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, req.replicas[i].id); } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + dInfo("vgId:%d, learnerReplica:%d ep:%s:%u dnode:%d", req.vgId, i, req.learnerReplicas[i].fqdn, + req.learnerReplicas[i].port, req.replicas[i].id); + } - SReplica *pReplica = &req.replicas[req.selfIndex]; + SReplica *pReplica = NULL; + if(req.selfIndex != -1){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -275,7 +311,8 @@ _OVER: vnodeClose(pImpl); vnodeDestroy(path, pMgmt->pTfs); } else { - dInfo("vgId:%d, vnode is created", req.vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", + req.vgId, TMSG_INFO(pMsg->msgType)); } tFreeSCreateVnodeReq(&req); @@ -283,6 +320,122 @@ _OVER: return code; } +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + SAlterVnodeTypeReq req = {0}; + if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if(req.learnerReplicas == 0){ + req.learnerSelfIndex = -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request", + req.vgId, TMSG_INFO(pMsg->msgType)); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); + terrno = TSDB_CODE_VND_NOT_EXIST; + return -1; + } + + ESyncRole role = vnodeGetRole(pVnode->pImpl); + dInfo("vgId:%d, checking node role:%d", req.vgId, role); + if(role == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("vgId:%d, checking node catch up", req.vgId); + if(vnodeIsCatchUp(pVnode->pImpl) != 1){ + terrno = TSDB_CODE_VND_NOT_CATCH_UP; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); + + int32_t vgId = req.vgId; + dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d", vgId, req.replica, req.selfIndex, + req.strict); + for (int32_t i = 0; i < req.replica; ++i) { + SReplica *pReplica = &req.replicas[i]; + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + SReplica *pReplica = &req.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + + if (req.replica <= 0 || + (req.selfIndex < 0 && req.learnerSelfIndex <0)|| + req.selfIndex >= req.replica || req.learnerSelfIndex >= req.learnerReplica) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, failed to alter replica since invalid msg", vgId); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + SReplica *pReplica = NULL; + if(req.selfIndex > 0){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } + + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("vgId:%d, start to close vnode", vgId); + SWrapperCfg wrapperCfg = { + .dropped = pVnode->dropped, + .vgId = pVnode->vgId, + .vgVersion = pVnode->vgVersion, + }; + tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); + vmCloseVnode(pMgmt, pVnode, false); + + char path[TSDB_FILENAME_LEN] = {0}; + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); + + dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); + if (vnodeAlterReplica(path, &req, pMgmt->pTfs) < 0) { + dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + dInfo("vgId:%d, begin to open vnode", vgId); + SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + if (vmOpenVnode(pMgmt, &wrapperCfg, pImpl) != 0) { + dError("vgId:%d, failed to open vnode mgmt since %s", vgId, terrstr()); + return -1; + } + + if (vnodeStart(pImpl) != 0) { + dError("vgId:%d, failed to start sync since %s", vgId, terrstr()); + return -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, end to process alter-node-type-request, vnode config is altered", + req.vgId, TMSG_INFO(pMsg->msgType)); + return 0; +} + int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SDisableVnodeWriteReq req = {0}; if (tDeserializeSDisableVnodeWriteReq(pMsg->pCont, pMsg->contLen, &req) != 0) { @@ -377,21 +530,40 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if(alterReq.learnerReplica == 0){ + alterReq.learnerSelfIndex = -1; + } + int32_t vgId = alterReq.vgId; - dInfo("vgId:%d, start to alter vnode replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex, - alterReq.strict); + dInfo("vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { SReplica *pReplica = &alterReq.replicas[i]; dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } - - if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + SReplica *pReplica = &alterReq.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); + } + + if (alterReq.replica <= 0 || + (alterReq.selfIndex < 0 && alterReq.learnerSelfIndex <0)|| + alterReq.selfIndex >= alterReq.replica || alterReq.learnerSelfIndex >= alterReq.learnerReplica) { terrno = TSDB_CODE_INVALID_MSG; dError("vgId:%d, failed to alter replica since invalid msg", vgId); return -1; } - SReplica *pReplica = &alterReq.replicas[alterReq.selfIndex]; + SReplica *pReplica = NULL; + if(alterReq.selfIndex != -1){ + pReplica = &alterReq.replicas[alterReq.selfIndex]; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerSelfIndex]; + } + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -425,7 +597,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, close vnode", vgId); + dInfo("vgId:%d, begin to open vnode", vgId); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); @@ -442,7 +614,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, vnode config is altered", vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); return 0; } @@ -517,9 +692,11 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK_TO_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_WALINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -548,8 +725,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; @@ -559,12 +737,14 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncCtrlQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 8008e5f81031af617a169f1b4414338f71de897e..0244a4fd6e34c8f362047a9829ecc56285ff31fa 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "vnd.h" SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -78,6 +79,11 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) { char path[TSDB_FILENAME_LEN] = {0}; + bool atExit = true; + + if (vnodeIsLeader(pVnode->pImpl)) { + vnodeProposeCommitOnNeed(pVnode->pImpl, atExit); + } taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); @@ -98,9 +104,9 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) pVnode->pSyncW.queue->threadId); tMultiWorkerCleanup(&pVnode->pSyncW); - dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId, - pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId); - tMultiWorkerCleanup(&pVnode->pSyncCtrlW); + dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, + pVnode->pSyncRdW.queue->threadId); + tMultiWorkerCleanup(&pVnode->pSyncRdW); dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, pVnode->pApplyW.queue->threadId); @@ -124,7 +130,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) vmFreeQueue(pMgmt, pVnode); if (commitAndRemoveWal) { - dInfo("vgId:%d, commit data", pVnode->vgId); + dInfo("vgId:%d, commit data for vnode split", pVnode->vgId); vnodeSyncCommit(pVnode->pImpl); vnodeBegin(pVnode->pImpl); dInfo("vgId:%d, commit data finished", pVnode->vgId); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 7aa1c9f56acc9e1cec882e6a317e0bc0ed2a5eb1..a7c4b2e70e13155d955fb2fd8d4862fc6725b2ea 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -49,6 +49,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_VND_ALTER_HASHRANGE: code = vmProcessAlterHashRangeReq(pMgmt, pMsg); break; + case TDMT_DND_ALTER_VNODE_TYPE: + code = vmProcessAlterVnodeTypeReq(pMgmt, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; dGError("msg:%p, not processed in vnode-mgmt queue", pMsg); @@ -114,11 +117,11 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vnodeProcessFetchMsg vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -163,8 +166,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, - terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; return terrno; } @@ -216,9 +219,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pSyncW.queue, pMsg); break; - case SYNC_CTRL_QUEUE: - dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg); + case SYNC_RD_QUEUE: + dGTrace("vgId:%d, msg:%p put into vnode-sync-rd queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pSyncRdW.queue, pMsg); break; case APPLY_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); @@ -234,9 +237,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp return code; } -int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); -} +int32_t vmPutMsgToSyncRdQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_RD_QUEUE); } int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } @@ -327,18 +328,18 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl}; SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; - SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; + SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-rd", .fp = (FItems)vmProcessSyncQueue, .param = pVnode}; SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl}; (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg); (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg); - (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg); + (void)tMultiWorkerInit(&pVnode->pSyncRdW, &sccfg); (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pStreamQ = tAutoQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue); - if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL || + if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncRdW.queue == NULL || pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -348,8 +349,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteW.queue->threadId); dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue, pVnode->pSyncW.queue->threadId); - dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue, - pVnode->pSyncCtrlW.queue->threadId); + dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue, + pVnode->pSyncRdW.queue->threadId); dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue, pVnode->pApplyW.queue->threadId); dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index acf96ad397c405c35fe7089aa4d618dcd8eb13e5..3459af1a3a99693f6667728ffe4ab6f2b3923c91 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -169,6 +169,8 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return -1; } + dInfo("start to process create-node-request"); + pWrapper = &pDnode->wrappers[ntype]; if (taosMkDir(pWrapper->path) != 0) { dmReleaseWrapper(pWrapper); @@ -198,6 +200,74 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return code; } +static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { + SDnode *pDnode = dmInstance(); + + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + dError("fail to process alter node type since node not exist"); + return -1; + } + dmReleaseWrapper(pWrapper); + + dInfo("node:%s, start to process alter-node-type-request", pWrapper->name); + + pWrapper = &pDnode->wrappers[ntype]; + + if(pWrapper->func.nodeRoleFp != NULL){ + ESyncRole role = (*pWrapper->func.nodeRoleFp)(pWrapper->pMgmt); + dInfo("node:%s, checking node role:%d", pWrapper->name, role); + if(role == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; + return -1; + } + } + + if(pWrapper->func.isCatchUpFp != NULL){ + dInfo("node:%s, checking node catch up", pWrapper->name); + if((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1){ + terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; + return -1; + } + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pWrapper->name); + + taosThreadMutexLock(&pDnode->mutex); + + dInfo("node:%s, stopping node", pWrapper->name); + dmStopNode(pWrapper); + dInfo("node:%s, closing node", pWrapper->name); + dmCloseNode(pWrapper); + + pWrapper = &pDnode->wrappers[ntype]; + if (taosMkDir(pWrapper->path) != 0) { + dmReleaseWrapper(pWrapper); + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); + + dInfo("node:%s, start to create", pWrapper->name); + int32_t code = (*pWrapper->func.createFp)(&input, pMsg); + if (code != 0) { + dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); + } else { + dInfo("node:%s, has been created", pWrapper->name); + code = dmOpenNode(pWrapper); + if (code == 0) { + code = dmStartNode(pWrapper); + } + pWrapper->deployed = true; + pWrapper->required = true; + } + + taosThreadMutexUnlock(&pDnode->mutex); + return code; +} + static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); @@ -251,6 +321,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .name = pWrapper->name, .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, + .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .getVnodeLoadsFp = dmGetVnodeLoads, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index cfdea40477b1dae1cabc35e8566c1ff7ad86a148..98ef8cd95ba9c201ec23ff9fd8515948016eec4f 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -89,6 +89,7 @@ typedef void (*SendMonitorReportFp)(); typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo); typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); +typedef int32_t (*ProcessAlterNodeTypeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef struct { int32_t dnodeId; @@ -112,6 +113,7 @@ typedef struct { SDnodeData *pData; SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; @@ -132,6 +134,8 @@ typedef int32_t (*NodeCreateFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeDropFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeRequireFp)(const SMgmtInputOpt *pInput, bool *required); typedef SArray *(*NodeGetHandlesFp)(); // array of SMgmtHandle +typedef bool (*NodeIsCatchUpFp)(void *pMgmt); +typedef bool (*NodeRole)(void *pMgmt); typedef struct { NodeOpenFp openFp; @@ -142,6 +146,8 @@ typedef struct { NodeDropFp dropFp; NodeRequireFp requiredFp; NodeGetHandlesFp getHandlesFp; + NodeIsCatchUpFp isCatchUpFp; + NodeRole nodeRoleFp; } SMgmtFunc; typedef struct { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 493ba48601382bdd25ce95b1639708c9f25c202d..b9aa8eb674b9b6f760f0c782f2a9e7aabe55eeb0 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -5,6 +5,7 @@ ENDIF () IF (TD_ENTERPRISE) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/privilege/src/privilege.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c) + LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c) ENDIF () add_library(mnode STATIC ${MNODE_SRC}) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 1176e1af0b8fa09328669f11066235870f39b06a..aa38b94fd7d6a6b0a5118d6aac2a760e7717cd94 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -24,10 +24,10 @@ extern "C" { enum { MQ_CONSUMER_STATUS__MODIFY = 1, - MQ_CONSUMER_STATUS__MODIFY_IN_REB, +// MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, - MQ_CONSUMER_STATUS__LOST_IN_REB, +// MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__LOST_REBD, MQ_CONSUMER_STATUS__REMOVED, }; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index fe2c58e29e1251e084d97f95e39efdf511f67e66..f547ce025d67d5e04441699b2e6b48eb3efd52e2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -176,6 +176,7 @@ typedef struct { char opername[TSDB_TRANS_OPER_LEN]; SArray* pRpcArray; SRWLatch lockRpcArray; + int64_t mTraceId; } STrans; typedef struct { @@ -214,6 +215,8 @@ typedef struct { bool syncRestore; int64_t stateStartTime; SDnodeObj* pDnode; + int32_t role; + SyncIndex lastIndex; } SMnodeObj; typedef struct { @@ -277,9 +280,13 @@ typedef struct { int8_t reserve; int32_t acctId; int32_t authVersion; + int32_t passVersion; SHashObj* readDbs; SHashObj* writeDbs; SHashObj* topics; + SHashObj* readTbs; + SHashObj* writeTbs; + SHashObj* useDbs; SRWLatch lock; } SUserObj; @@ -337,6 +344,7 @@ typedef struct { ESyncState syncState; bool syncRestore; bool syncCanRead; + ESyncRole nodeRole; } SVnodeGid; typedef struct { @@ -357,8 +365,9 @@ typedef struct { int8_t compact; int8_t isTsma; int8_t replica; - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; void* pTsma; + int32_t numOfCachedTables; } SVgObj; typedef struct { @@ -391,7 +400,7 @@ typedef struct { } SSmaObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; + char name[TSDB_INDEX_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; char dstTbName[TSDB_TABLE_FNAME_LEN]; @@ -445,6 +454,8 @@ typedef struct { int32_t codeSize; char* pComment; char* pCode; + int32_t funcVersion; + SRWLatch lock; } SFuncObj; typedef struct { @@ -456,6 +467,7 @@ typedef struct { void* pIter; SMnode* pMnode; STableMetaRsp* pMeta; + bool restore; bool sysDbRsp; char db[TSDB_DB_FNAME_LEN]; char filterTb[TSDB_TABLE_NAME_LEN]; @@ -543,7 +555,7 @@ void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer typedef struct { int32_t vgId; - char* qmsg; //SubPlanToString + char* qmsg; // SubPlanToString SEpSet epSet; } SMqVgEp; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 006c74ef3de223a34aaa483346b28102bfdfe54a..ec83f8a7e6daed8e7a0685087d35c5aad90e390e 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -92,8 +92,11 @@ typedef struct { int64_t transSeq; TdThreadMutex lock; int8_t selfIndex; + int8_t numOfTotalReplicas; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + ESyncRole nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncMgmt; typedef struct { @@ -113,6 +116,7 @@ typedef struct SMnode { bool deploy; char *path; int64_t checkTime; + SyncIndex applied; SSdb *pSdb; SArray *pSteps; SQHandle *pQuery; diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index ac0924aab9965de8ccf25d135e5f4b2082121cbe..66d0ed1d12160b5ea6e9c30f145e0db6faa22b31 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -38,6 +38,7 @@ void mndFreeStb(SStbObj *pStb); int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen); void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); +void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index 8ed7fc6a11043bddda295759d8ba59e9d26be1f8..b2b86775aba1e59ca7caecc935669eb9c53c4380 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -32,6 +32,7 @@ bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); +int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index d6e5d322baf28dbad41d0ea13124e1ec21588f0b..057e3efbbcabe42d43c6030973707e371ccf4fd1 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -52,6 +52,8 @@ typedef struct { int32_t contLen; void *pCont; SSdbRaw *pRaw; + + int64_t mTraceId; } STransAction; typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 8943ba703ee47442a49fe25ab44ed7fcfb7867cd..aa7f97f0870dfd83ba4c3296ba1293e091b6fdba 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -31,9 +31,12 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); SHashObj *mndDupDbHash(SHashObj *pOld); +SHashObj *mndDupTableHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 94584dfe58f3bd40dc4df5b67cde6c415b071693..4d05637a2b3ab872d5a0ba99c022e21f79ca9b05 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -67,7 +67,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { return 0; } -static SClusterObj *mndAcquireCluster(SMnode *pMnode) { +static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -76,23 +76,27 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); if (pIter == NULL) break; + *ppIter = pIter; + return pCluster; } return NULL; } -static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster) { +static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster, void *pIter) { SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pCluster); } int64_t mndGetClusterId(SMnode *pMnode) { int64_t clusterId = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { clusterId = pCluster->id; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return clusterId; @@ -100,10 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) { int64_t mndGetClusterCreateTime(SMnode *pMnode) { int64_t createTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { createTime = pCluster->createdTime; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return createTime; @@ -121,10 +126,11 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { float mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { upTime = mndGetClusterUpTimeImp(pCluster); - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return upTime / 86400.0f; @@ -321,11 +327,12 @@ static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SClusterObj clusterObj = {0}; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); clusterObj.upTime += tsUptimeInterval; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } if (clusterObj.id <= 0) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index f1ef83aca553d682092fe3a7551c8b40aba48f49..0d75b5fd68242b2eed202e2ec432847d5874a211 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -77,13 +77,11 @@ void mndCleanupConsumer(SMnode *pMnode) {} bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - mInfo("tq timer, rebalance counter old val:%d", old); + mDebug("tq timer, rebalance counter old val:%d", old); return old == 0; } -void mndRebEnd() { - mndRebCntDec(); -} +void mndRebEnd() { mndRebCntDec(); } void mndRebCntInc() { int32_t val = atomic_add_fetch_32(&mqRebInExecCnt, 1); @@ -101,7 +99,7 @@ void mndRebCntDec() { int32_t newVal = val - 1; int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal); if (oldVal == val) { - mInfo("rebalance trans end, rebalance counter:%d", newVal); + mDebug("rebalance trans end, rebalance counter:%d", newVal); break; } } @@ -247,23 +245,43 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebInfo; } +static void freeRebalanceItem(void *param) { + SMqRebInfo *pInfo = param; + taosArrayDestroy(pInfo->lostConsumers); + taosArrayDestroy(pInfo->newConsumers); + taosArrayDestroy(pInfo->removedConsumers); +} + static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer; void *pIter = NULL; - mTrace("start to process mq timer"); + mDebug("start to process mq timer"); // rebalance cannot be parallel if (!mndRebTryStart()) { - mInfo("mq rebalance already in progress, do nothing"); + mDebug("mq rebalance already in progress, do nothing"); return 0; } SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); + if (pRebMsg == NULL) { + mError("failed to create the rebalance msg, size:%d, quit mq timer", (int32_t)sizeof(SMqDoRebalanceMsg)); + mndRebEnd(); + return TSDB_CODE_OUT_OF_MEMORY; + } + pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - // TODO set cleanfp + if (pRebMsg->rebSubHash == NULL) { + mError("failed to create rebalance hashmap"); + rpcFreeCont(pRebMsg); + mndRebEnd(); + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosHashSetFreeFp(pRebMsg->rebSubHash, freeRebalanceItem); // iterate all consumers, find all modification while (1) { @@ -356,7 +374,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); - mInfo("mq rebalance finished, no modification"); + mDebug("mq timer finished, no need to re-balance"); mndRebEnd(); } return 0; @@ -538,9 +556,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { return -1; } - ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = serverEpoch; - ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; + SMqRspHead* pHead = buf; + + pHead->mqMsgType = TMQ_MSG_TYPE__EP_RSP; + pHead->epoch = serverEpoch; + pHead->consumerId = pConsumer->consumerId; + pHead->walsver = 0; + pHead->walever = 0; + void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqAskEpRsp(&abuf, &rsp); @@ -576,7 +599,7 @@ int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj return 0; } -static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const char* pUser) { +static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser) { int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { @@ -597,8 +620,13 @@ static int32_t validateTopics(const SArray* pTopicList, SMnode* pMnode, const ch return 0; } -static void* topicNameDup(void* p){ - return taosStrdup((char*) p); +static void *topicNameDup(void *p) { return taosStrdup((char *)p); } + +static void freeItem(void *param) { + void *pItem = *(void **)param; + if (pItem != NULL) { + taosMemoryFree(pItem); + } } int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { @@ -616,7 +644,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *pTopicList = subscribe.topicNames; taosArraySort(pTopicList, taosArrayCompareString); - taosArrayRemoveDuplicateP(pTopicList, taosArrayCompareString, taosMemoryFree); + taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); int32_t newTopicNum = taosArrayGetSize(pTopicList); @@ -633,8 +661,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pExistedConsumer = mndAcquireConsumer(pMnode, consumerId); if (pExistedConsumer == NULL) { - mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s, numOfTopics:%d", consumerId, - subscribe.cgroup, (int32_t) taosArrayGetSize(pTopicList)); + mInfo("receive subscribe request from new consumer:0x%" PRIx64 " cgroup:%s, numOfTopics:%d", consumerId, + subscribe.cgroup, (int32_t)taosArrayGetSize(pTopicList)); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId)); @@ -656,8 +684,9 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { /*taosRLockLatch(&pExistedConsumer->lock);*/ int32_t status = atomic_load_32(&pExistedConsumer->status); - mInfo("receive subscribe request from existed consumer:0x%" PRIx64 " cgroup:%s, current status:%d(%s), subscribe topic num: %d", - consumerId, subscribe.cgroup, status,mndConsumerStatusName(status), newTopicNum); + mInfo("receive subscribe request from existed consumer:0x%" PRIx64 + " cgroup:%s, current status:%d(%s), subscribe topic num: %d", + consumerId, subscribe.cgroup, status, mndConsumerStatusName(status), newTopicNum); if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; @@ -674,7 +703,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); - int32_t oldTopicNum = (pExistedConsumer->currentTopics)? taosArrayGetSize(pExistedConsumer->currentTopics):0; + int32_t oldTopicNum = (pExistedConsumer->currentTopics) ? taosArrayGetSize(pExistedConsumer->currentTopics) : 0; int32_t i = 0, j = 0; while (i < oldTopicNum || j < newTopicNum) { @@ -839,12 +868,53 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, - pConsumer->status, mndConsumerStatusName(pConsumer->status)); + mDebug("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status, + mndConsumerStatusName(pConsumer->status)); tDeleteSMqConsumerObj(pConsumer); return 0; } +static void updateConsumerStatus(SMqConsumerObj *pConsumer) { + int32_t status = pConsumer->status; + + if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { + if (status == MQ_CONSUMER_STATUS__MODIFY) { + pConsumer->status = MQ_CONSUMER_STATUS__READY; + } else if (status == MQ_CONSUMER_STATUS__LOST) { + pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; + } + } +} + +// remove from new topic +static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { + int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < taosArrayGetSize(pConsumer->rebNewTopics); i++) { + char *p = taosArrayGetP(pConsumer->rebNewTopics, i); + if (strcmp(pTopic, p) == 0) { + taosArrayRemove(pConsumer->rebNewTopics, i); + taosMemoryFree(p); + + mDebug("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId, + pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics)); + break; + } + } +} + +// remove from removed topic +static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { + int32_t size = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < size; i++) { + char *p = taosArrayGetP(pConsumer->rebRemovedTopics, i); + if (strcmp(pTopic, p) == 0) { + taosArrayRemove(pConsumer->rebRemovedTopics, i); + taosMemoryFree(p); + break; + } + } +} + static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); @@ -855,6 +925,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ + // this new consumer has identical topics with one existed consumers. if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; } else { @@ -871,7 +942,6 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pNewConsumer->assignedTopics = tmp; pOldConsumer->subscribeTime = pNewConsumer->upTime; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { @@ -911,71 +981,48 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { - /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/ - /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/ + ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); - char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // not exist in current topic - - bool existing = false; -#if 1 + bool existing = false; int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics); for (int32_t i = 0; i < numOfExistedTopics; i++) { char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); - if (strcmp(topic, addedTopic) == 0) { + if (strcmp(topic, pNewTopic) == 0) { existing = true; } } -#endif - // remove from new topic - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); - if (strcmp(addedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->rebNewTopics, i); - taosMemoryFree(topic); - break; - } - } + removeFromNewTopicList(pOldConsumer, pNewTopic); // add to current topic if (!existing) { - taosArrayPush(pOldConsumer->currentTopics, &addedTopic); + taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); } else { - taosMemoryFree(addedTopic); + taosMemoryFree(pNewTopic); } // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; - } - } else { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; - } - } + updateConsumerStatus(pOldConsumer); // the re-balance is triggered when the new consumer is launched. pOldConsumer->rebalanceTime = pNewConsumer->upTime; atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), - pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); - - // not exist in new topic #if 0 for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); @@ -984,14 +1031,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, #endif // remove from removed topic - for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebRemovedTopics); i++) { - char *topic = taosArrayGetP(pOldConsumer->rebRemovedTopics, i); - if (strcmp(removedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->rebRemovedTopics, i); - taosMemoryFree(topic); - break; - } - } + removeFromRemoveTopicList(pOldConsumer, removedTopic); // remove from current topic int32_t i = 0; @@ -1004,32 +1044,20 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, break; } } - // must find the topic - /*A(i < sz);*/ // set status int32_t status = pOldConsumer->status; - if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; - } - } else { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY_IN_REB; - } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { - pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; - } - } + updateConsumerStatus(pOldConsumer); pOldConsumer->rebalanceTime = pNewConsumer->upTime; atomic_add_fetch_32(&pOldConsumer->epoch, 1); - mDebug("consumer:0x%" PRIx64 " state %d(%s) -> %d(%s), new epoch:%d, reb-time:%" PRId64 ", current topics:%d", + mDebug("consumer:0x%" PRIx64 " state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 + ", current topics:%d, newTopics:%d, removeTopics:%d", pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), - pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } taosWUnLockLatch(&pOldConsumer->lock); @@ -1105,8 +1133,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataSetVal(pColInfo, numOfRows, (const char *)clientId, false); // status - char status[20 + VARSTR_HEADER_SIZE] = {0}; - const char* pStatusName = mndConsumerStatusName(pConsumer->status); + char status[20 + VARSTR_HEADER_SIZE] = {0}; + const char *pStatusName = mndConsumerStatusName(pConsumer->status); STR_TO_VARSTR(status, pStatusName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1163,10 +1191,8 @@ static const char *mndConsumerStatusName(int status) { return "ready"; case MQ_CONSUMER_STATUS__LOST: case MQ_CONSUMER_STATUS__LOST_REBD: - case MQ_CONSUMER_STATUS__LOST_IN_REB: return "lost"; case MQ_CONSUMER_STATUS__MODIFY: - case MQ_CONSUMER_STATUS__MODIFY_IN_REB: return "rebalancing"; default: return "unknown"; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index af769713045a2e0f4154d1136534dfc8e12d7be9..df220340688ae7b0230dae3a3242e0fc9ff6d2b9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -275,10 +275,14 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; pOld->cfg.walFsyncPeriod = pNew->cfg.walFsyncPeriod; pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.walRetentionPeriod = pNew->cfg.walRetentionPeriod; + pOld->cfg.walRetentionSize = pNew->cfg.walRetentionSize; pOld->cfg.strict = pNew->cfg.strict; pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.replications = pNew->cfg.replications; pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; + pOld->cfg.minRows = pNew->cfg.minRows; + pOld->cfg.maxRows = pNew->cfg.maxRows; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; pOld->compactStartTime = pNew->compactStartTime; taosWUnLockLatch(&pOld->lock); @@ -616,13 +620,8 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } } else { if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, createReq.db) == 0) { - mInfo("db:%s, is creating and createdb response after trans finished", createReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; - goto _OVER; - } else { - goto _OVER; - } + code = terrno; + goto _OVER; } else if (terrno == TSDB_CODE_MND_DB_IN_DROPPING) { goto _OVER; } else if (terrno == TSDB_CODE_MND_DB_NOT_EXIST) { @@ -725,6 +724,32 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = 0; } + if (pAlter->sstTrigger > 0 && pAlter->sstTrigger != pDb->cfg.sstTrigger) { + pDb->cfg.sstTrigger = pAlter->sstTrigger; + pDb->vgVersion++; + terrno = 0; + } + + if (pAlter->minRows > 0 && pAlter->minRows != pDb->cfg.minRows) { + pDb->cfg.minRows = pAlter->minRows; + pDb->vgVersion++; + terrno = 0; + } + + if (pAlter->walRetentionPeriod > TSDB_DB_MIN_WAL_RETENTION_PERIOD && + pAlter->walRetentionPeriod != pDb->cfg.walRetentionPeriod) { + pDb->cfg.walRetentionPeriod = pAlter->walRetentionPeriod; + pDb->vgVersion++; + terrno = 0; + } + + if (pAlter->walRetentionSize > TSDB_DB_MIN_WAL_RETENTION_SIZE && + pAlter->walRetentionSize != pDb->cfg.walRetentionSize) { + pDb->cfg.walRetentionSize = pAlter->walRetentionSize; + pDb->vgVersion++; + terrno = 0; + } + return terrno; } @@ -821,6 +846,18 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } + int32_t numOfTopics = 0; + if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) { + goto _OVER; + } + + if (numOfTopics != 0 && alterReq.walRetentionPeriod == 0) { + terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; + mError("db:%s, not allowed to set WAL_RETENTION_PERIOD 0 when there are topics defined. numOfTopics:%d", pDb->name, + numOfTopics); + goto _OVER; + } + memcpy(&dbObj, pDb, sizeof(SDbObj)); if (dbObj.cfg.pRetensions != NULL) { dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); @@ -1258,14 +1295,9 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { usedbRsp.errCode = terrno; if (terrno == TSDB_CODE_MND_DB_IN_CREATING) { - if (mndSetRpcInfoForDbTrans(pMnode, pReq, MND_OPER_CREATE_DB, usedbReq.db) == 0) { - mInfo("db:%s, is creating and usedb response after trans finished", usedbReq.db); - code = TSDB_CODE_ACTION_IN_PROGRESS; - goto _OVER; - } + code = terrno; + goto _OVER; } - - mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; @@ -1405,7 +1437,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { +static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb, SRpcMsg *pReq) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; @@ -1427,7 +1459,7 @@ static int32_t mndTrimDb(SMnode *pMnode, SDbObj *pDb) { pHead->vgId = htonl(pVgroup->vgId); tSerializeSVTrimDbReq((char *)pHead + sizeof(SMsgHead), contLen, &trimReq); - SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen}; + SRpcMsg rpcMsg = {.msgType = TDMT_VND_TRIM, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { @@ -1463,7 +1495,7 @@ static int32_t mndProcessTrimDbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndTrimDb(pMnode, pDb); + code = mndTrimDb(pMnode, pDb, pReq); _OVER: if (code != 0) { diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index fb81a764f133f53f786b1d227e6c8d80f25d9c49..c69f08eb6b244b358fa73312d5a86e7ff4818097 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, innerSz) < 0) return -1; for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); - if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; + if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; } } @@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { taosArrayDestroy(pArray); return -1; } - if (tDecodeSStreamTask(pDecoder, pTask) < 0) { + if (tDecodeStreamTask(pDecoder, pTask) < 0) { taosMemoryFree(pTask); taosArrayDestroy(pArray); return -1; @@ -158,7 +158,10 @@ void tFreeStreamObj(SStreamObj *pStream) { taosMemoryFree(pStream->sql); taosMemoryFree(pStream->ast); taosMemoryFree(pStream->physicalPlan); - if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema); + + if (pStream->outputSchema.nCols) { + taosMemoryFree(pStream->outputSchema.pSchema); + } int32_t sz = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < sz; i++) { @@ -166,11 +169,14 @@ void tFreeStreamObj(SStreamObj *pStream) { int32_t taskSz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < taskSz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - tFreeSStreamTask(pTask); + tFreeStreamTask(pTask); } + taosArrayDestroy(pLevel); } + taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema if (pStream->tagSchema.nCols > 0) { taosMemoryFree(pStream->tagSchema.pSchema); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 1d9db37a7d68403781b72b1d8c9d8b43951d0261..1f58ae97a3c7034eda9ef8385e1e057cf0c510d2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -25,6 +25,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "tmisce.h" +#include "mndCluster.h" #define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -366,6 +367,14 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { goto _OVER; } + int64_t clusterid = mndGetClusterId(pMnode); + if (statusReq.clusterId != 0 && statusReq.clusterId != clusterid) { + code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER; + mWarn("dnode:%d, %s, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 ", code:0x%x", + statusReq.dnodeId, statusReq.dnodeEp, statusReq.clusterId, clusterid, code); + goto _OVER; + } + if (statusReq.dnodeId == 0) { pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); if (pDnode == NULL) { @@ -412,6 +421,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (pVgroup != NULL) { if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { pVgroup->cacheUsage = pVload->cacheUsage; + pVgroup->numOfCachedTables = pVload->numOfCachedTables; pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->totalStorage = pVload->totalStorage; @@ -440,7 +450,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { if (roleChanged) { SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb != NULL && pDb->stateTs != curMs) { - mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, pDb->stateTs, curMs); + mInfo("db:%s, stateTs changed by status msg, old stateTs:%" PRId64 " new stateTs:%" PRId64, pDb->name, + pDb->stateTs, curMs); pDb->stateTs = curMs; } mndReleaseDb(pMnode, pDb); @@ -954,7 +965,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; tmsgSendReq(&epSet, &rpcMsg); code = 0; } diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 44a7d49fff672e33af43bb892350ee0b5e02debd..a991bddda8a997c6074ee21e811f3829e0370fe8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -421,6 +421,7 @@ void dumpUser(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); tjsonAddStringToObject(item, "superUser", i642str(pObj->superUser)); tjsonAddStringToObject(item, "authVersion", i642str(pObj->authVersion)); + tjsonAddStringToObject(item, "passVersion", i642str(pObj->passVersion)); tjsonAddStringToObject(item, "numOfReadDbs", i642str(taosHashGetSize(pObj->readDbs))); tjsonAddStringToObject(item, "numOfWriteDbs", i642str(taosHashGetSize(pObj->writeDbs))); sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 8d006f10296eaf1fc1e90613eebc54cfd7e983c7..51a231daf463c357eb494c87fad4b0187a9c43f6 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -21,7 +21,7 @@ #include "mndTrans.h" #include "mndUser.h" -#define SDB_FUNC_VER 1 +#define SDB_FUNC_VER 2 #define SDB_FUNC_RESERVE_SIZE 64 static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); @@ -83,6 +83,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) } SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->funcVersion, _OVER) SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER); @@ -107,7 +108,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != SDB_FUNC_VER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -144,8 +145,15 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { goto _OVER; } SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + + if (sver >= 2) { + SDB_GET_INT32(pRaw, dataPos, &pFunc->funcVersion, _OVER) + } + SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pFunc->lock); + terrno = 0; _OVER: @@ -173,6 +181,44 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + + taosWLockLatch(&pOld->lock); + + pOld->align = pNew->align; + pOld->bufSize = pNew->bufSize; + pOld->codeSize = pNew->codeSize; + pOld->commentSize = pNew->commentSize; + pOld->createdTime = pNew->createdTime; + pOld->funcType = pNew->funcType; + pOld->funcVersion = pNew->funcVersion; + pOld->outputLen = pNew->outputLen; + pOld->outputType = pNew->outputType; + + if (pOld->pComment != NULL) { + taosMemoryFree(pOld->pComment); + pOld->pComment = NULL; + } + if (pNew->commentSize > 0 && pNew->pComment != NULL) { + pOld->commentSize = pNew->commentSize; + pOld->pComment = taosMemoryMalloc(pOld->commentSize); + memcpy(pOld->pComment, pNew->pComment, pOld->commentSize); + } + + if (pOld->pCode != NULL) { + taosMemoryFree(pOld->pCode); + pOld->pCode = NULL; + } + if (pNew->codeSize > 0 && pNew->pCode != NULL) { + pOld->codeSize = pNew->codeSize; + pOld->pCode = taosMemoryMalloc(pOld->codeSize); + memcpy(pOld->pCode, pNew->pCode, pOld->codeSize); + } + + pOld->scriptType = pNew->scriptType; + pOld->signature = pNew->signature; + + taosWUnLockLatch(&pOld->lock); + return 0; } @@ -225,26 +271,47 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); if (pTrans == NULL) goto _OVER; - mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; + SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name); + if (pCreate->orReplace == 1 && oldFunc != NULL) { + func.funcVersion = oldFunc->funcVersion + 1; + func.createdTime = oldFunc->createdTime; + + SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER; + + SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER; - SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + } else { + SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; - SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; + + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + } if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; _OVER: + if (oldFunc != NULL) { + mndReleaseFunc(pMnode, oldFunc); + } + taosMemoryFree(func.pCode); taosMemoryFree(func.pComment); mndTransDrop(pTrans); @@ -304,6 +371,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { mInfo("func:%s, already exist, ignore exist is set", createReq.name); code = 0; goto _OVER; + } else if (createReq.orReplace) { + mInfo("func:%s, replace function is set", createReq.name); + code = 0; } else { terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; goto _OVER; @@ -413,6 +483,12 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { goto RETRIEVE_FUNC_OVER; } + retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); + if (retrieveRsp.pFuncExtraInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + for (int32_t i = 0; i < retrieveReq.numOfFuncs; ++i) { char *funcName = taosArrayGet(retrieveReq.pFuncNames, i); @@ -451,6 +527,11 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { } } taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); + SFuncExtraInfo extraInfo = {0}; + extraInfo.funcVersion = pFunc->funcVersion; + extraInfo.funcCreatedTime = pFunc->createdTime; + taosArrayPush(retrieveRsp.pFuncExtraInfos, &extraInfo); + mndReleaseFunc(pMnode, pFunc); } @@ -519,6 +600,7 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)b2, false); + taosMemoryFree(b2); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, NULL, true); @@ -545,6 +627,31 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char *language = ""; + if (pFunc->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { + language = "C"; + } else if (pFunc->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { + language = "Python"; + } + char varLang[TSDB_TYPE_STR_MAX_LEN + 1] = {0}; + varDataSetLen(varLang, strlen(language)); + strcpy(varDataVal(varLang), language); + colDataSetVal(pColInfo, numOfRows, (const char *)varLang, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + int32_t varCodeLen = (pFunc->codeSize + VARSTR_HEADER_SIZE) > TSDB_MAX_BINARY_LEN + ? TSDB_MAX_BINARY_LEN + : pFunc->codeSize + VARSTR_HEADER_SIZE; + char *b4 = taosMemoryMalloc(varCodeLen); + memcpy(varDataVal(b4), pFunc->pCode, varCodeLen - VARSTR_HEADER_SIZE); + varDataSetLen(b4, varCodeLen - VARSTR_HEADER_SIZE); + colDataSetVal(pColInfo, numOfRows, (const char *)b4, false); + taosMemoryFree(b4); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->funcVersion, false); + numOfRows++; sdbRelease(pSdb, pFunc); } diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c index 8782fd823f36fdfcad114d4cf2f6bbac5d754bad..83172acf641608427a045623b835b9300901ad2c 100644 --- a/source/dnode/mnode/impl/src/mndIndex.c +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -138,7 +138,7 @@ static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbOb mInfo("idx: %s start to build drop index req", pIdx->name); len = tSerializeSDropIdxReq(NULL, 0, &req); - if (ret < 0) { + if (len < 0) { goto _err; } @@ -672,7 +672,7 @@ _OVER: static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; SIdxObj idxObj = {0}; - memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.name, req->idxName, TSDB_INDEX_FNAME_LEN); memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN); memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d83b969e2debfd3848b39c28ca316e4ef9eb3fc1..a9f52128a685215fed934eaddf684cc77ea1ff30 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -380,11 +380,13 @@ static int32_t mndInitSdb(SMnode *pMnode) { } static int32_t mndOpenSdb(SMnode *pMnode) { + int32_t code = 0; if (!pMnode->deploy) { - return sdbReadFile(pMnode->pSdb); - } else { - return 0; + code = sdbReadFile(pMnode->pSdb); } + + atomic_store_64(&pMnode->applied, pMnode->pSdb->commitIndex); + return code; } static void mndCleanupSdb(SMnode *pMnode) { @@ -489,7 +491,10 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->selfDnodeId = pOption->dnodeId; pMnode->syncMgmt.selfIndex = pOption->selfIndex; pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + pMnode->syncMgmt.numOfTotalReplicas = pOption->numOfTotalReplicas; + pMnode->syncMgmt.lastIndex = pOption->lastIndex; memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); + memcpy(pMnode->syncMgmt.nodeRoles, pOption->nodeRoles, sizeof(pOption->nodeRoles)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -580,6 +585,16 @@ int32_t mndStart(SMnode *pMnode) { return mndInitTimer(pMnode); } +int32_t mndIsCatchUp(SMnode *pMnode) { + int64_t rid = pMnode->syncMgmt.sync; + return syncIsCatchUp(rid); +} + +ESyncRole mndGetRole(SMnode *pMnode){ + int64_t rid = pMnode->syncMgmt.sync; + return syncGetRole(rid); +} + void mndStop(SMnode *pMnode) { mndSetStop(pMnode); mndSyncStop(pMnode); @@ -651,7 +666,7 @@ _OVER: pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, - pMnode->stopped, state.restored, syncStr(state.restored)); + pMnode->stopped, state.restored, syncStr(state.state)); return -1; } @@ -759,6 +774,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr pClusterInfo->connections_total = mndGetNumOfConnections(pMnode); pClusterInfo->dbs_total = sdbGetSize(pSdb, SDB_DB); pClusterInfo->stbs_total = sdbGetSize(pSdb, SDB_STB); + pClusterInfo->topics_toal = sdbGetSize(pSdb, SDB_TOPIC); + pClusterInfo->streams_total = sdbGetSize(pSdb, SDB_STREAM); void *pIter = NULL; while (1) { @@ -862,7 +879,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr } // grant info - pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 86400000.0f; + pGrantInfo->expire_time = (pMnode->grant.expireTimeMS - ms) / 1000; pGrantInfo->timeseries_total = pMnode->grant.timeseriesAllowed; if (pMnode->grant.expireTimeMS == 0) { pGrantInfo->expire_time = INT32_MAX; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 50fab447e3c7b8fb85ece7c025059358f6ab9e6f..5e3476859a446340212ed9d687f84a87dd16c3ea 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,7 +23,7 @@ #include "mndTrans.h" #include "tmisce.h" -#define MNODE_VER_NUMBER 1 +#define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); @@ -53,6 +53,7 @@ int32_t mndInitMnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE, mndProcessAlterMnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); @@ -126,6 +127,8 @@ static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pObj->role, _OVER) + SDB_SET_INT64(pRaw, dataPos, pObj->lastIndex, _OVER) SDB_SET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -149,7 +152,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != MNODE_VER_NUMBER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -164,6 +167,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER) + if(sver >=2){ + SDB_GET_INT32(pRaw, dataPos, &pObj->role, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pObj->lastIndex, _OVER) + } SDB_GET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -204,7 +211,9 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); + pOld->role = pNew->role; pOld->updateTime = pNew->updateTime; + pOld->lastIndex = pNew->lastIndex; mndReloadSyncConfig(pSdb->pMnode); return 0; @@ -302,6 +311,28 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p return 0; } +static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, + SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); + + STransAction action = { + .epSet = *pAlterMnodeTypeEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_ALTER_MNODE_TYPE, + .retryCode = TSDB_CODE_MNODE_NOT_CATCH_UP, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); void *pReq = taosMemoryMalloc(contLen); @@ -347,6 +378,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; + int32_t numOfLearnerReplicas = 0; SDCreateMnodeReq createReq = {0}; SEpSet createEpset = {0}; @@ -355,18 +387,29 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - createReq.replicas[numOfReplicas].id = pMObj->id; - createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfReplicas++; + } + else{ + createReq.learnerReplicas[numOfLearnerReplicas].id = pMObj->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pMObj->pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfLearnerReplicas++; + } - numOfReplicas++; sdbRelease(pSdb, pMObj); } - createReq.replica = numOfReplicas + 1; - createReq.replicas[numOfReplicas].id = pDnode->id; - createReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas; + createReq.learnerReplica = numOfLearnerReplicas + 1; + createReq.learnerReplicas[numOfLearnerReplicas].id = pDnode->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + createReq.lastIndex = pObj->lastIndex; createEpset.inUse = 0; createEpset.numOfEps = 1; @@ -378,23 +421,78 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } +static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SDAlterMnodeTypeReq alterReq = {0}; + SEpSet createEpset = {0}; + + while (1) { + SMnodeObj *pMObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); + if (pIter == NULL) break; + + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + alterReq.replicas[alterReq.replica].id = pMObj->id; + alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + } + else{ + alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id; + alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port; + memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.learnerReplica++; + } + + sdbRelease(pSdb, pMObj); + } + + alterReq.replicas[alterReq.replica].id = pDnode->id; + alterReq.replicas[alterReq.replica].port = pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + + alterReq.lastIndex = pObj->lastIndex; + + createEpset.inUse = 0; + createEpset.numOfEps = 1; + createEpset.eps[0].port = pDnode->port; + memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; + + return 0; +} + static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; - SMnodeObj mnodeObj = {0}; - mnodeObj.id = pDnode->id; - mnodeObj.createdTime = taosGetTimestampMs(); - mnodeObj.updateTime = mnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + SMnodeObj mnodeObj = {0}; + mnodeObj.id = pDnode->id; + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + mnodeObj.role = TAOS_SYNC_ROLE_LEARNER; + mnodeObj.lastIndex = pMnode->applied; + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; + + SMnodeObj mnodeLeaderObj = {0}; + mnodeLeaderObj.id = pDnode->id; + mnodeLeaderObj.createdTime = taosGetTimestampMs(); + mnodeLeaderObj.updateTime = mnodeLeaderObj.createdTime; + mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER; + mnodeLeaderObj.lastIndex = pMnode->applied + 1; + + if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER; + if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -514,6 +612,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) { if (pObj == NULL) return 0; + pObj->lastIndex = pMnode->applied; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; return 0; @@ -730,7 +829,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) { void *pIter = NULL; int32_t updatingMnodes = 0; int32_t readyMnodes = 0; - SSyncCfg cfg = {.myIndex = -1}; + SSyncCfg cfg = {.myIndex = -1, .lastIndex = 0,}; + SyncIndex maxIndex = 0; while (1) { pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); @@ -745,26 +845,41 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { - SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + SNodeInfo *pNode = &cfg.nodeInfo[cfg.totalReplicaNum]; pNode->nodeId = pObj->pDnode->id; pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; + pNode->nodeRole = pObj->role; tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { - cfg.myIndex = cfg.replicaNum; + cfg.myIndex = cfg.totalReplicaNum; + } + if(pNode->nodeRole == TAOS_SYNC_ROLE_VOTER){ + cfg.replicaNum++; + } + cfg.totalReplicaNum++; + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; + } + } + + if (objStatus == SDB_STATUS_DROPPING) { + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; } - cfg.replicaNum++; } + mInfo("vgId:1, mnode:%d, role:%d, lastIndex:%" PRId64, pObj->id, pObj->role, pObj->lastIndex); + sdbReleaseLock(pSdb, pObj, false); } - if (readyMnodes <= 0 || updatingMnodes <= 0) { - mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); - return; - } + //if (readyMnodes <= 0 || updatingMnodes <= 0) { + // mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); + // return; + //} if (cfg.myIndex == -1) { #if 1 @@ -777,12 +892,14 @@ static void mndReloadSyncConfig(SMnode *pMnode) { return; } - if (updatingMnodes > 0) { - mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); - for (int32_t i = 0; i < cfg.replicaNum; ++i) { + if (pMnode->syncMgmt.sync > 0) { + mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", + cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); + + for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, - pNode->clusterId); + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId, pNode->clusterId, pNode->nodeRole); } int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index ccb4140b83bc6bcc58eacdd52b357fc19ed3455b..de0374c6e89c506ce1b9cad966e2a4dd8905b7b7 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -35,6 +35,7 @@ int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = 1; + pRsp->enable = pUser->enable; pRsp->version = pUser->authVersion; return 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 41dea5073146076e5783f763fbe43c9488c5d12d..2ebb5aeb9904cbabcc95086846b9ded2cdd2f1ef 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -226,7 +226,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 2)) != 0) { + if ((code = taosCheckVersionCompatibleFromStr(connReq.sVer, version, 3)) != 0) { terrno = code; goto _OVER; } @@ -283,6 +283,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); connectRsp.svrTimestamp = taosGetTimestampSec(); + connectRsp.passVer = pUser->passVersion; strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, @@ -547,6 +548,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserPassInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserPassVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_PASSINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_APP_ERROR; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index d1671aa12a5b4c5dfbc2a3a1ede49825b7e384a2..734f624be0e19c942c10244f28263570d6ea4504 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -138,7 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->taskId; + pVgInfo->taskId = pLastLevelTask->id.taskId; break; } } @@ -149,7 +149,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; + pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; } @@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -350,12 +350,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - pInnerTask = tNewSStreamTask(pStream->uid); + pInnerTask = tNewStreamTask(pStream->uid); if (pInnerTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; qDestroyQueryPlan(pPlan); return -1; } + pInnerTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); @@ -421,7 +422,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); @@ -440,7 +441,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pTask->fixedEpDispatcher.taskId = pInnerTask->taskId; + pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; @@ -460,7 +461,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pEpInfo->childId = pTask->selfChildId; pEpInfo->epSet = pTask->epSet; pEpInfo->nodeId = pTask->nodeId; - pEpInfo->taskId = pTask->taskId; + pEpInfo->taskId = pTask->id.taskId; taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); sdbRelease(pSdb, pVgroup); } @@ -491,7 +492,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 7fe08514f62108ae08f75c2d875cb117881b9298..c50b205f37f569fced5a690f329ae3ba26c1ac6a 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -324,7 +324,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { pReq->info.rsp = pRsp; pReq->info.rspLen = size; - if (rowsRead == 0 || rowsRead < rowsToRead) { + if (rowsRead == 0 || ((rowsRead < rowsToRead) && !pShow->restore)) { pRsp->completed = 1; mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mndReleaseShowObj(pShow, true); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 09aef2cd967fb094b553dec38a837e59050c5e44..3d654a23d87b082aa441012375d0691089e391d8 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -470,6 +470,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, return -1; } + action.mTraceId = pTrans->mTraceId; action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; @@ -693,6 +694,8 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { SDbObj *pDb = NULL; SMCreateSmaReq createReq = {0}; + int64_t mTraceId = TRACE_GET_ROOTID(&pReq->info.traceId); + if (tDeserializeSMCreateSmaReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2a369a863a8f6a79356d058992c0717d16c2a557..63bcef2a5bd80839ad5a7e6e052f18b46f1106c4 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -668,6 +668,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj } STransAction action = {0}; + action.mTraceId = pTrans->mTraceId; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; action.contLen = contLen; @@ -877,7 +878,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { pHead->vgId = htonl(pVgroup->vgId); tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), contLen, &ttlReq); - SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen}; + SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); int32_t code = tmsgSendReq(&epSet, &rpcMsg); if (code != 0) { @@ -2613,6 +2614,13 @@ void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { tNameGetFullDbName(&name, dst); } +void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + tNameGetDbName(&name, dst); +} + void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { int32_t pos = -1; int32_t num = 0; @@ -3113,9 +3121,18 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(typeName, "SUPER_TABLE"); + bool fetch = pShow->restore ? false : true; + pShow->restore = false; while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); - if (pShow->pIter == NULL) break; + if (fetch) { + pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); + if (pShow->pIter == NULL) break; + } else { + fetch = true; + void *pKey = taosHashGetKey(pShow->pIter, NULL); + pStb = sdbAcquire(pSdb, SDB_STB, pKey); + if (!pStb) continue; + } if (pDb != NULL && pStb->dbUid != pDb->uid) { sdbRelease(pSdb, pStb); @@ -3129,6 +3146,17 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB sdbRelease(pSdb, pStb); continue; } + + if ((numOfRows + pStb->numOfColumns) > rows) { + pShow->restore = true; + if (numOfRows == 0) { + mError("mndRetrieveStbCol failed to get stable cols since buf:%d less than result:%d, stable name:%s, db:%s", + rows, pStb->numOfColumns, pStb->name, pStb->db); + } + sdbRelease(pSdb, pStb); + break; + } + varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b7f80f6b0e5e84be4806268c9b6c301488a02860..76bb144fcb0b33ff5a832b01c9dd0b9ad9b95175 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -35,12 +35,12 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); -static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); +static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); -// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); -/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -418,7 +418,7 @@ FAIL: int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tEncoderClear(&encoder); @@ -430,10 +430,11 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); STransAction action = {0}; + action.mTraceId = pTrans->mTraceId; memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); action.pCont = buf; action.contLen = tlen; @@ -600,7 +601,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return -1; } pReq->head.vgId = htonl(pTask->nodeId); - pReq->taskId = pTask->taskId; + pReq->taskId = pTask->id.taskId; STransAction action = {0}; memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); action.pCont = pReq; @@ -1208,7 +1209,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 4d19110f31ffa9e01a38c4bd60ab375e608a65d4..49921c9a1a0572c8498ca1ff01fb5491b23cd595 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -197,24 +197,20 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t totalVgNum = pOutput->pSub->vgNum; - const char *sub = pOutput->pSub->key; - mInfo("sub:%s mq re-balance %d vgroups", sub, pOutput->pSub->vgNum); - - // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg - SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); +static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { + int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + const char *pSubKey = pOutput->pSub->key; - // 2. check and get actual removed consumers, put their vg into hash - int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; - for (int32_t i = 0; i < removedNum; i++) { + for (int32_t i = 0; i < numOfRemoved; i++) { uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // consumer exists till now if (pConsumerEp) { actualRemoved++; + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); @@ -223,52 +219,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .newConsumerId = -1, .pVgEp = pVgEp, }; + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, sub, pVgEp->vgId, consumerId); + mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId); } + taosArrayDestroy(pConsumerEp->vgs); taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); + // put into removed taosArrayPush(pOutput->removedConsumers, &consumerId); } } - if (removedNum != actualRemoved) { - mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved); + if (numOfRemoved != actualRemoved) { + mError("sub:%s mq re-balance removedNum:%d not matched with actual:%d", pSubKey, numOfRemoved, actualRemoved); + } else { + mInfo("sub:%s removed %d consumers", pSubKey, numOfRemoved); } +} - // if previously no consumer, there are vgs not assigned - { - int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs); - for (int32_t i = 0; i < consumerVgNum; i++) { - SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs); - SMqRebOutputVg rebOutput = { - .oldConsumerId = -1, - .newConsumerId = -1, - .pVgEp = pVgEp, - }; - taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", sub, pVgEp->vgId); - } +static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { + int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); + const char *pSubKey = pOutput->pSub->key; + + for (int32_t i = 0; i < numOfNewConsumers; i++) { + int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); + + SMqConsumerEp newConsumerEp; + newConsumerEp.consumerId = consumerId; + newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); + + taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); + taosArrayPush(pOutput->newConsumers, &consumerId); + mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, pSubKey, consumerId); } +} - // 3. calc vg number of each consumer - int32_t afterRebConsumerNum = pInput->oldConsumerNum + taosArrayGetSize(pInput->pRebInfo->newConsumers) - - taosArrayGetSize(pInput->pRebInfo->removedConsumers); - int32_t minVgCnt = 0; - int32_t imbConsumerNum = 0; - // calc num - if (afterRebConsumerNum) { - minVgCnt = totalVgNum / afterRebConsumerNum; - imbConsumerNum = totalVgNum % afterRebConsumerNum; +static void addUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { + const char *pSubKey = pOutput->pSub->key; + int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); + + for (int32_t i = 0; i < numOfVgroups; i++) { + SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs); + SMqRebOutputVg rebOutput = { + .oldConsumerId = -1, + .newConsumerId = -1, + .pVgEp = pVgEp, + }; + + taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); + mInfo("sub:%s mq re-balance remove vgId:%d from unassigned", pSubKey, pVgEp->vgId); } +} - mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has more vgs", sub, - afterRebConsumerNum, minVgCnt, imbConsumerNum); +static void transferVgroupsForConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, + int32_t imbConsumerNum) { + const char *pSubKey = pOutput->pSub->key; - // 4. first scan: remove consumer more than wanted, put to remove hash int32_t imbCnt = 0; void *pIter = NULL; + while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); if (pIter == NULL) { @@ -276,8 +287,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); - int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); // all old consumers still existing are touched // TODO optimize: touch only consumer whose vgs changed taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId); @@ -296,13 +307,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } } else { - // pop until equal minVg + // all the remain consumers should only have the number of vgroups, which is equalled to the value of minVg while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) { SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs); SMqRebOutputVg outputVg = { @@ -311,36 +322,67 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", sub, pVgEp->vgId, + mInfo("sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64 ",(first scan)", pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } } } } +} - // 5. add new consumer into sub - { - int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers); - for (int32_t i = 0; i < consumerNum; i++) { - int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); - - SMqConsumerEp newConsumerEp; - newConsumerEp.consumerId = consumerId; - newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); - taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); - taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("sub:%s mq rebalance add new consumer:%" PRIx64, sub, consumerId); - } +static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + int32_t totalVgNum = pOutput->pSub->vgNum; + const char *pSubKey = pOutput->pSub->key; + + int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); + int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers); + mInfo("sub:%s mq re-balance %d vgroups, existed consumers:%d, added:%d, removed:%d", pSubKey, totalVgNum, + pInput->oldConsumerNum, numOfAdded, numOfRemoved); + + // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + // 2. check and get actual removed consumers, put their vg into hash + doRemoveExistedConsumers(pOutput, pHash, pInput); + + // 3. if previously no consumer, there are vgs not assigned + addUnassignedVgroups(pOutput, pHash); + + // 4. calc vg number of each consumer + int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved; + + int32_t minVgCnt = 0; + int32_t imbConsumerNum = 0; + + // calc num + if (numOfFinal) { + minVgCnt = totalVgNum / numOfFinal; + imbConsumerNum = totalVgNum % numOfFinal; + mInfo("sub:%s mq re-balance %d consumers: at least %d vgs each, %d consumers has 1 more vgroups than avg value", + pSubKey, numOfFinal, minVgCnt, imbConsumerNum); + } else { + mInfo("sub:%s no consumer subscribe this topic", pSubKey); } - // 6. second scan: find consumer do not have enough vg, extract from temporary hash and assign to new consumer. + // 5. first scan: remove vgroups from te consumers, who have more vgroups than the threashold value that is + // minVgCnt, and then put them into the recycled hash list + transferVgroupsForConsumers(pOutput, pHash, minVgCnt, imbConsumerNum); + + // 6. add new consumer into sub + doAddNewConsumers(pOutput, pInput); + + // 7. second scan: find consumer do not have enough vgroups, extract from temporary hash and assign to them // All related vg should be put into rebVgs SMqRebOutputVg *pRebVg = NULL; void *pRemovedIter = NULL; - pIter = NULL; + void *pIter = NULL; + while (1) { pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; // push until equal minVg @@ -348,8 +390,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR // iter hash and find one vg pRemovedIter = taosHashIterate(pHash, pRemovedIter); if (pRemovedIter == NULL) { - mError("sub:%s removed iter is null", sub); - continue; + mError("sub:%s removed iter is null", pSubKey); + break; } pRebVg = (SMqRebOutputVg *)pRemovedIter; @@ -357,7 +399,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId, + mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (not enough)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } @@ -387,12 +429,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; if (pRebVg->newConsumerId == pRebVg->oldConsumerId) { - mInfo("mq rebalance: skip vg %d for same consumer:%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId, + mInfo("mq rebalance: skip vg %d for same consumer:0x%" PRIx64 " (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); continue; } taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vgId:%d to consumer:%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId, + mInfo("mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " (second scan) (unassigned)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } else { @@ -409,15 +451,15 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId); + mInfo("sub:%s mq re-balance unassign vgId:%d (second scan)", pSubKey, pRebOutput->pVgEp->vgId); } } // 8. generate logs - mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", sub); + mInfo("sub:%s mq re-balance calculation completed, re-balanced vg", pSubKey); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, sub, + mInfo("sub:%s mq re-balance vgId:%d, moved from consumer:0x%" PRIx64 ", to consumer:0x%" PRIx64, pSubKey, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } { @@ -427,10 +469,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", sub, pConsumerEp->consumerId, sz); + mInfo("sub:%s mq re-balance final cfg: consumer:0x%" PRIx64 " has %d vg", pSubKey, pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); - mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, sub, pVgEp->vgId, + mInfo("sub:%s mq re-balance final cfg: vg %d to consumer:0x%" PRIx64, pSubKey, pVgEp->vgId, pConsumerEp->consumerId); } } @@ -555,17 +597,23 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqDoRebalanceMsg *pReq = pMsg->pCont; void *pIter = NULL; + bool rebalanceOnce = false; // to ensure only once. - mInfo("mq re-balance start"); + mInfo("mq re-balance start, total required re-balanced trans:%d", taosHashGetSize(pReq->rebSubHash)); + // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. while (1) { + if (rebalanceOnce) { + break; + } + pIter = taosHashIterate(pReq->rebSubHash, pIter); if (pIter == NULL) { break; } - SMqRebInputObj rebInput = {0}; - + // todo handle the malloc failure + SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.removedConsumers = taosArrayInit(0, sizeof(int64_t)); @@ -582,9 +630,10 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char topic[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { - mError("mq re-balance %s ignored since topic %s not exist", pRebInfo->key, topic); + mError("mq re-balance %s ignored since topic %s doesn't exist", pRebInfo->key, topic); continue; } @@ -604,11 +653,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { mndReleaseTopic(pMnode, pTopic); rebInput.oldConsumerNum = 0; + mInfo("topic:%s has no consumers sub yet", topic); } else { taosRLockLatch(&pSub->lock); rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); rebOutput.pSub = tCloneSubscribeObj(pSub); taosRUnLockLatch(&pSub->lock); + mInfo("topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } @@ -620,19 +671,17 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // possibly no vg is changed // when each topic is re-balanced, issue an trans to save the results in sdb. if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { - mError("mq re-balance persist output error, possibly vnode splitted or dropped"); + mError("mq re-balance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()); } - taosArrayDestroy(pRebInfo->lostConsumers); - taosArrayDestroy(pRebInfo->newConsumers); - taosArrayDestroy(pRebInfo->removedConsumers); - taosArrayDestroy(rebOutput.newConsumers); taosArrayDestroy(rebOutput.touchedConsumers); taosArrayDestroy(rebOutput.removedConsumers); taosArrayDestroy(rebOutput.rebVgs); tDeleteSubscribeObj(rebOutput.pSub); taosMemoryFree(rebOutput.pSub); + + rebalanceOnce = true; } // reset flag @@ -1019,7 +1068,7 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumerEp->consumerId, false); - mDebug("mnd show subscriptions: topic %s, consumer:%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), + mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), pConsumerEp->consumerId, varDataVal(cgroup), pVgEp->vgId); // offset diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index f702d8f14858f13525f893737f02cc40beab8355..0a6df02f5fbf2636de06358831b577917846b797 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -33,7 +33,7 @@ static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { return -1; } - int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; @@ -129,6 +129,14 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { int32_t code = 0; + pMsg->info.conn.applyIndex = pMeta->index; + pMsg->info.conn.applyTerm = pMeta->term; + + if (pMsg->code == 0) { + SMnode *pMnode = pFsm->data; + atomic_store_64(&pMnode->applied, pMsg->info.conn.applyIndex); + } + if (!syncUtilUserCommit(pMsg->msgType)) { goto _out; } @@ -140,6 +148,11 @@ _out: return code; } +SyncIndex mndSyncAppliedIndex(const SSyncFSM *pFSM) { + SMnode *pMnode = pFSM->data; + return atomic_load_64(&pMnode->applied); +} + int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { mInfo("start to read snapshot from sdb in atomic way"); SMnode *pMnode = pFsm->data; @@ -153,7 +166,7 @@ static void mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) { sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex); } -void mndRestoreFinish(const SSyncFSM *pFsm) { +void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { SMnode *pMnode = pFsm->data; if (!pMnode->deploy) { @@ -167,6 +180,8 @@ void mndRestoreFinish(const SSyncFSM *pFsm) { } else { mInfo("vgId:1, sync restore finished"); } + + ASSERT(commitIdx == mndSyncAppliedIndex(pFsm)); } int32_t mndSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) { @@ -222,6 +237,23 @@ static void mndBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pMgmt->lock); } +static void mndBecomeLearner(const SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + mInfo("vgId:1, become learner"); + + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + mInfo("vgId:1, become learner and post sem, trans:%d, failed to propose since not leader", pMgmt->transId); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; + tsem_post(&pMgmt->syncSem); + } + taosThreadMutexUnlock(&pMgmt->lock); +} + static void mndBecomeLeader(const SSyncFSM *pFsm) { mInfo("vgId:1, become leader"); SMnode *pMnode = pFsm->data; @@ -253,6 +285,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; pFsm->FpCommitCb = mndSyncCommitMsg; + pFsm->FpAppliedIndexCb = mndSyncAppliedIndex; pFsm->FpPreCommitCb = NULL; pFsm->FpRollBackCb = NULL; pFsm->FpRestoreFinishCb = mndRestoreFinish; @@ -262,6 +295,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpReConfigCb = NULL; pFsm->FpBecomeLeaderCb = mndBecomeLeader; pFsm->FpBecomeFollowerCb = mndBecomeFollower; + pFsm->FpBecomeLearnerCb = mndBecomeLearner; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; @@ -301,13 +335,16 @@ int32_t mndInitSync(SMnode *pMnode) { mInfo("vgId:1, start to open sync, replica:%d selfIndex:%d", pMgmt->numOfReplicas, pMgmt->selfIndex); SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->totalReplicaNum = pMgmt->numOfTotalReplicas; pCfg->replicaNum = pMgmt->numOfReplicas; pCfg->myIndex = pMgmt->selfIndex; - for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + pCfg->lastIndex = pMgmt->lastIndex; + for (int32_t i = 0; i < pMgmt->numOfTotalReplicas; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = pMgmt->nodeRoles[i]; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 991f1099a63aee39526b27371d72cd9699ea83fb..f6da3709164158acc008708121873da5bb2b78d1 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -605,6 +605,12 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } + if (pDb->cfg.walRetentionPeriod == 0) { + terrno = TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO; + mError("db:%s, not allowed to create topic when WAL_RETENTION_PERIOD is zero", pDb->name); + goto _OVER; + } + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user); if (code == 0) { code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -793,7 +799,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { +int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { *pNumOfTopics = 0; SSdb *pSdb = pMnode->pSdb; @@ -943,4 +949,4 @@ int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return code; } -#endif \ No newline at end of file +#endif diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 39b4252618f33f4cc114eab0116e5617953ac35a..106eea031384a93641c002200ab0c084f53fc76a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -505,8 +505,8 @@ static TransCbFp mndTransGetCbFp(ETrnFunc ftype) { } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { - mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), - pTrans->startFunc); + mInfo("trans:%d, perform insert action, row:%p stage:%s, callfunc:1, startFunc:%d", pTrans->id, pTrans, + mndTransStr(pTrans->stage), pTrans->startFunc); if (pTrans->startFunc > 0) { TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); @@ -577,8 +577,7 @@ static void mndTransUpdateActions(SArray *pOldArray, SArray *pNewArray) { static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { mInfo("trans:%d, perform update action, old row:%p stage:%s create:%" PRId64 ", new row:%p stage:%s create:%" PRId64, - pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage), - pNew->createdTime); + pOld->id, pOld, mndTransStr(pOld->stage), pOld->createdTime, pNew, mndTransStr(pNew->stage), pNew->createdTime); if (pOld->createdTime != pNew->createdTime) { mError("trans:%d, failed to perform update action since createTime not match, old row:%p stage:%s create:%" PRId64 @@ -650,6 +649,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); + pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : 0; taosInitRWLatch(&pTrans->lockRpcArray); if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || @@ -706,7 +706,8 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { } int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; + STransAction action = { + .stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw, .mTraceId = pTrans->mTraceId}; return mndTransAppendAction(pTrans->redoActions, &action); } @@ -728,6 +729,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_REDO_ACTION; pAction->actionType = TRANS_ACTION_MSG; + pAction->mTraceId = pTrans->mTraceId; return mndTransAppendAction(pTrans->redoActions, pAction); } @@ -875,7 +877,6 @@ int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { } } - if (mndCheckTransConflict(pMnode, pTrans)) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -912,6 +913,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pNew->pRpcArray = pTrans->pRpcArray; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; + pNew->mTraceId = pTrans->mTraceId; pTrans->pRpcArray = NULL; pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; @@ -1091,10 +1093,13 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; pTrans->lastErrorNo = pRsp->code; + + mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, + mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode); + } else { + mInfo("trans:%d, invalid action, index:%d, code:0x%x", transId, action, pRsp->code); } - mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage), - action, pRsp->code, pAction->acceptableCode, pAction->retryCode); mndTransExecute(pMnode, pTrans, true); _OVER: @@ -1167,6 +1172,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + rpcMsg.info.traceId.rootId = pTrans->mTraceId; + memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); char detail[1024] = {0}; @@ -1457,7 +1464,7 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->code = 0; - pTrans->stage = TRN_STAGE_FINISHED; // TRN_STAGE_PRE_FINISH is not necessary + pTrans->stage = TRN_STAGE_FINISHED; // TRN_STAGE_PRE_FINISH is not necessary mInfo("trans:%d, stage from commitAction to finished", pTrans->id); continueExec = true; } else { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 23822c8f208deb9761f66b32d0b0b31fc0f0e55a..f6e5895cda2ee31b838f1203c861a04f3c49ac17 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -18,11 +18,12 @@ #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" +#include "mndStb.h" #include "mndTopic.h" #include "mndTrans.h" #include "tbase64.h" -#define USER_VER_NUMBER 2 +#define USER_VER_NUMBER 4 #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -124,9 +125,40 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); + int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs); + int32_t numOfWriteStbs = taosHashGetSize(pUser->writeTbs); int32_t numOfTopics = taosHashGetSize(pUser->topics); - int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN + - numOfTopics * TSDB_TOPIC_FNAME_LEN; + int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + + (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN; + + char *stb = taosHashIterate(pUser->readTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + + size_t valueLen = 0; + valueLen = strlen(stb); + size += sizeof(int32_t); + size += valueLen; + stb = taosHashIterate(pUser->readTbs, stb); + } + + stb = taosHashIterate(pUser->writeTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + + size_t valueLen = 0; + valueLen = strlen(stb); + size += sizeof(int32_t); + size += valueLen; + stb = taosHashIterate(pUser->writeTbs, stb); + } SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -142,6 +174,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->reserve, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER) @@ -164,6 +197,49 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { topic = taosHashIterate(pUser->topics, topic); } + SDB_SET_INT32(pRaw, dataPos, numOfReadStbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfWriteStbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER) + + stb = taosHashIterate(pUser->readTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + size_t valueLen = 0; + valueLen = strlen(stb) + 1; + SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER); + stb = taosHashIterate(pUser->readTbs, stb); + } + + stb = taosHashIterate(pUser->writeTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + size_t valueLen = 0; + valueLen = strlen(stb) + 1; + SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER); + stb = taosHashIterate(pUser->writeTbs, stb); + } + + int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + SDB_SET_INT32(pRaw, dataPos, *useDb, _OVER) + useDb = taosHashIterate(pUser->writeTbs, useDb); + } + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -188,7 +264,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -210,6 +286,9 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->reserve, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) + if (sver >= 4) { + SDB_GET_INT32(pRaw, dataPos, &pUser->passVersion, _OVER) + } int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -249,6 +328,75 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } } + if (sver >= 3) { + int32_t numOfReadStbs = 0; + int32_t numOfWriteStbs = 0; + int32_t numOfUseDbs = 0; + SDB_GET_INT32(pRaw, dataPos, &numOfReadStbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfWriteStbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER) + + pUser->readTbs = + taosHashInit(numOfReadStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pUser->writeTbs = + taosHashInit(numOfWriteStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < numOfReadStbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t valuelen = 0; + SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); + char *value = taosMemoryCalloc(valuelen, sizeof(char)); + memset(value, 0, valuelen); + SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) + + taosHashPut(pUser->readTbs, key, keyLen, value, valuelen); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfWriteStbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t valuelen = 0; + SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); + char *value = taosMemoryCalloc(valuelen, sizeof(char)); + memset(value, 0, valuelen); + SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) + + taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfUseDbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t ref = 0; + SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); + + taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)); + } + } + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) taosInitRWLatch(&pUser->lock); @@ -261,6 +409,9 @@ _OVER: taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); + taosHashCleanup(pUser->readTbs); + taosHashCleanup(pUser->writeTbs); + taosHashCleanup(pUser->useDbs); } taosMemoryFreeClear(pRow); return NULL; @@ -285,6 +436,57 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return 0; } +SHashObj *mndDupTableHash(SHashObj *pOld) { + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + char *tb = taosHashIterate(pOld, NULL); + while (tb != NULL) { + size_t keyLen = 0; + char *key = taosHashGetKey(tb, &keyLen); + + int32_t valueLen = strlen(tb) + 1; + if (taosHashPut(pNew, key, keyLen, tb, valueLen) != 0) { + taosHashCancelIterate(pOld, tb); + taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + tb = taosHashIterate(pOld, tb); + } + + return pNew; +} + +SHashObj *mndDupUseDbHash(SHashObj *pOld) { + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int32_t *db = taosHashIterate(pOld, NULL); + while (db != NULL) { + size_t keyLen = 0; + char *key = taosHashGetKey(db, &keyLen); + + if (taosHashPut(pNew, key, keyLen, db, sizeof(*db)) != 0) { + taosHashCancelIterate(pOld, db); + taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + db = taosHashIterate(pOld, db); + } + + return pNew; +} + static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; @@ -293,7 +495,10 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { taosRLockLatch(&pUser->lock); pNew->readDbs = mndDupDbHash(pUser->readDbs); pNew->writeDbs = mndDupDbHash(pUser->writeDbs); + pNew->readTbs = mndDupTableHash(pUser->readTbs); + pNew->writeTbs = mndDupTableHash(pUser->writeTbs); pNew->topics = mndDupTopicHash(pUser->topics); + pNew->useDbs = mndDupUseDbHash(pUser->useDbs); taosRUnLockLatch(&pUser->lock); if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { @@ -306,9 +511,15 @@ static void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); + taosHashCleanup(pUser->readTbs); + taosHashCleanup(pUser->writeTbs); + taosHashCleanup(pUser->useDbs); pUser->readDbs = NULL; pUser->writeDbs = NULL; pUser->topics = NULL; + pUser->readTbs = NULL; + pUser->writeTbs = NULL; + pUser->useDbs = NULL; } static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { @@ -322,12 +533,16 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; pOld->authVersion = pNew->authVersion; + pOld->passVersion = pNew->passVersion; pOld->sysInfo = pNew->sysInfo; pOld->enable = pNew->enable; memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); TSWAP(pOld->topics, pNew->topics); + TSWAP(pOld->readTbs, pNew->readTbs); + TSWAP(pOld->writeTbs, pNew->writeTbs); + TSWAP(pOld->useDbs, pNew->useDbs); taosWUnLockLatch(&pOld->lock); return 0; @@ -337,7 +552,11 @@ SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; + if (terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + } else { + terrno = TSDB_CODE_MND_USER_NOT_AVAILABLE; + } } return pUser; } @@ -494,6 +713,71 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAM SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); } +static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, + SSdb *pSdb) { + void *pIter = NULL; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + + snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName); + int32_t len = strlen(tbFName) + 1; + + if (alterReq->tagCond != NULL && alterReq->tagCondLen != 0) { + char *value = taosHashGet(hash, tbFName, len); + if (value != NULL) { + terrno = TSDB_CODE_MND_PRIVILEDGE_EXIST; + return -1; + } + + int32_t condLen = alterReq->tagCondLen; + if (taosHashPut(hash, tbFName, len, alterReq->tagCond, condLen) != 0) { + return -1; + } + } else { + if (taosHashPut(hash, tbFName, len, "t", 2) != 0) { + return -1; + } + } + + int32_t dbKeyLen = strlen(alterReq->objname) + 1; + int32_t ref = 1; + int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen); + if (NULL != currRef) { + ref = (*currRef) + 1; + } + if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { + return -1; + } + + return 0; +} + +static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, + SSdb *pSdb) { + void *pIter = NULL; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName); + int32_t len = strlen(tbFName) + 1; + + if (taosHashRemove(hash, tbFName, len) != 0) { + return -1; + } + + int32_t dbKeyLen = strlen(alterReq->objname) + 1; + int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen); + if (NULL == currRef || 1 == *currRef) { + if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) { + return -1; + } + return 0; + } + int32_t ref = (*currRef) - 1; + if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { + return -1; + } + + return 0; +} + static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -539,10 +823,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; + newUser.passVersion = pUser->passVersion; if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN); + if (0 != strncmp(pUser->pass, pass, TSDB_PASSWORD_LEN)) { + ++newUser.passVersion; + } } if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) { @@ -633,6 +921,22 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE) { + if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE) { + if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE) { + if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) { + if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) { int32_t len = strlen(alterReq.objname) + 1; SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname); @@ -826,6 +1130,70 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int32_t *numOfRows, char *user, + SShowObj *pShow) { + char *value = taosHashIterate(hash, NULL); + int32_t cols = 0; + + while (value != NULL) { + cols = 0; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(userName, user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)userName, false); + + char privilege[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(privilege, priType, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)privilege, false); + + size_t keyLen = 0; + void *key = taosHashGetKey(value, &keyLen); + + char dbName[TSDB_DB_NAME_LEN] = {0}; + mndExtractShortDbNameFromStbFullName(key, dbName); + char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)dbNameContent, false); + + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + mndExtractTbNameFromStbFullName(key, tableName, TSDB_TABLE_NAME_LEN); + char tableNameContent[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableNameContent, tableName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false); + + if (strcmp("t", value) != 0) { + SNode *pAst = NULL; + int32_t sqlLen = 0; + char sql[TSDB_EXPLAIN_RESULT_ROW_SIZE] = {0}; + + if (nodesStringToNode(value, &pAst) == 0) { + nodesNodeToSQL(pAst, sql, TSDB_EXPLAIN_RESULT_ROW_SIZE, &sqlLen); + nodesDestroyNode(pAst); + } else { + sqlLen = 5; + sprintf(sql, "error"); + } + + char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); + } else { + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); + } + + (*numOfRows)++; + value = taosHashIterate(hash, value); + } +} + static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -841,7 +1209,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfTopics = taosHashGetSize(pUser->topics); - if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break; + int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs); + int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs); + if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) break; if (pUser->superUser) { cols = 0; @@ -860,6 +1230,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; } @@ -884,6 +1264,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; db = taosHashIterate(pUser->readDbs, db); } @@ -909,10 +1299,24 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; db = taosHashIterate(pUser->writeDbs, db); } + mndLoopHash(pUser->readTbs, "read", pBlock, &numOfRows, pUser->user, pShow); + + mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow); + char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { cols = 0; @@ -932,6 +1336,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; topic = taosHashIterate(pUser->topics, topic); } @@ -1015,6 +1429,69 @@ _OVER: return code; } +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { + int32_t code = 0; + SUserPassBatchRsp batchRsp = {0}; + + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to validate user pass since %s", pUsers[i].user, terrstr()); + continue; + } + + pUsers[i].version = ntohl(pUsers[i].version); + if (pUser->passVersion <= pUsers[i].version) { + mTrace("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, + pUsers[i].version); + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserPassRsp rsp = {0}; + memcpy(rsp.user, pUser->user, TSDB_USER_LEN); + rsp.version = pUser->passVersion; + + if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + mndReleaseUser(pMnode, pUser); + goto _OVER; + } + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + goto _OVER; + } + + int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp); + if (rspLen < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + +_OVER: + if (code) { + *ppRsp = NULL; + *pRspLen = 0; + } + + tFreeSUserPassBatchRsp(&batchRsp); + return code; +} + int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b5d9353b6847163244241ea35dd2dd82619b82fb..f0bece6e5eb8d77538e2d02eeb4787e2f3c55049 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -40,6 +40,7 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq); static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq); +static int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq); int32_t mndInitVgroup(SMnode *pMnode) { SSdbTable table = { @@ -60,10 +61,14 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DISABLE_WRITE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_SYNC_FORCE_FOLLOWER_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg); + //mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessVgroupBalanceLeaderMsg); mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP, mndProcessBalanceVgroupMsg); + mndSetMsgHandle(pMnode, TDMT_MND_BALANCE_VGROUP_LEADER, mndProcessVgroupBalanceLeaderMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -199,7 +204,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { pNew->compStorage = pOld->compStorage; pNew->pointsWritten = pOld->pointsWritten; pNew->compact = pOld->compact; - memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); + memcpy(pOld->vnodeGid, pNew->vnodeGid, (TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA) * sizeof(SVnodeGid)); return 0; } @@ -240,8 +245,10 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.compression = pDb->cfg.compression; createReq.strict = pDb->cfg.strict; createReq.cacheLast = pDb->cfg.cacheLast; - createReq.replica = pVgroup->replica; + createReq.replica = 0; + createReq.learnerReplica = 0; createReq.selfIndex = -1; + createReq.learnerSelfIndex = -1; createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->cfg.hashMethod; @@ -259,7 +266,15 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &createReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &createReq.replicas[createReq.replica]; + } + else{ + pReplica = &createReq.learnerReplicas[createReq.learnerReplica]; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { @@ -271,21 +286,40 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (pDnode->id == pVgid->dnodeId) { - createReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (pDnode->id == pVgid->dnodeId) { + createReq.selfIndex = createReq.replica; + } + } + else{ + if (pDnode->id == pVgid->dnodeId) { + createReq.learnerSelfIndex = createReq.learnerReplica; + } + } + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + createReq.replica++; + } + else{ + createReq.learnerReplica++; } } - if (createReq.selfIndex == -1) { + if (createReq.selfIndex == -1 && createReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } - mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d strict:%d", createReq.vgId, createReq.replica, - createReq.selfIndex, createReq.strict); + mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica, + createReq.learnerReplica, createReq.strict); for (int32_t i = 0; i < createReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.replicas[i].fqdn, createReq.replicas[i].port); } + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.learnerReplicas[i].fqdn, + createReq.learnerReplicas[i].port); + } int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); if (contLen < 0) { @@ -319,6 +353,10 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV alterReq.walLevel = pDb->cfg.walLevel; alterReq.strict = pDb->cfg.strict; alterReq.cacheLast = pDb->cfg.cacheLast; + alterReq.sttTrigger = pDb->cfg.sstTrigger; + alterReq.minRows = pDb->cfg.minRows; + alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod; + alterReq.walRetentionSize = pDb->cfg.walRetentionSize; mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId); int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq); @@ -348,12 +386,24 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p SAlterVnodeReplicaReq alterReq = { .vgId = pVgroup->vgId, .strict = pDb->cfg.strict, - .replica = pVgroup->replica, + .replica = 0, + .learnerReplica = 0, .selfIndex = -1, + .learnerSelfIndex = -1, }; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &alterReq.replicas[alterReq.replica]; + alterReq.replica++; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerReplica]; + alterReq.learnerReplica++; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) return NULL; @@ -363,18 +413,30 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (dnodeId == pVgid->dnodeId) { - alterReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (dnodeId == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + else{ + if (dnodeId == pVgid->dnodeId) { + alterReq.learnerSelfIndex = v; + } } } - alterReq.replica = pVgroup->replica; - mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, - alterReq.selfIndex, alterReq.strict); + + mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port); } + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + mInfo("vgId:%d, learnerReplica:%d ep:%s:%u", alterReq.vgId, i, + alterReq.learnerReplicas[i].fqdn, alterReq.learnerReplicas[i].port); + } - if (alterReq.selfIndex == -1) { + if (alterReq.selfIndex == -1 && alterReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -801,6 +863,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t cacheUsage = (int32_t)pVgroup->cacheUsage; colDataSetVal(pColInfo, numOfRows, (const char *)&cacheUsage, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->numOfCachedTables, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); @@ -1183,6 +1248,32 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } +int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode == NULL) return -1; + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_ALTER_VNODE_TYPE; + action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; + action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); @@ -1768,6 +1859,130 @@ _OVER: return code; } +static void *mndBuildSForceBecomeFollowerReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dnodeId, + int32_t *pContLen) { + SForceBecomeFollowerReq balanceReq = { + .vgId = pVgroup->vgId, + }; + + int32_t contLen = tSerializeSForceBecomeFollowerReq(NULL, 0, &balanceReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSForceBecomeFollowerReq((char *)pReq + sizeof(SMsgHead), contLen, &balanceReq); + *pContLen = contLen; + return pReq; +} + +int32_t mndAddBalanceVgroupLeaderAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dnodeId) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode == NULL) return -1; + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildSForceBecomeFollowerReq(pMnode, pVgroup, dnodeId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_SYNC_FORCE_FOLLOWER; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTrans){ + SSdb *pSdb = pMnode->pSdb; + + int32_t vgid = pVgroup->vgId; + int8_t replica = pVgroup->replica; + + if(pVgroup->replica <= 1) { + mInfo("trans:%d, vgid:%d no need to balance, replica:%d", pTrans->id, vgid, replica); + return -1; + } + + int32_t dnodeId = pVgroup->vnodeGid[0].dnodeId; + + for(int i = 0; i < replica; i++) + { + if(pVgroup->vnodeGid[i].syncState == TAOS_SYNC_STATE_LEADER){ + dnodeId = pVgroup->vnodeGid[i].dnodeId; + break; + } + } + + bool exist = false; + bool online = false; + int64_t curMs = taosGetTimestampMs(); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode != NULL) { + exist = true; + online = mndIsDnodeOnline(pDnode, curMs); + mndReleaseDnode(pMnode, pDnode); + } + + if(exist && online) + { + mInfo("trans:%d, vgid:%d leader to dnode:%d", pTrans->id, vgid, dnodeId); + + if (mndAddBalanceVgroupLeaderAction(pMnode, pTrans, pVgroup, dnodeId) != 0) { + mError("trans:%d, vgid:%d failed to be balanced to dnode:%d", pTrans->id, vgid, dnodeId); + return -1; + } + + SSdbRaw *pRaw = mndVgroupActionEncode(pVgroup); + if (pRaw == NULL) { + mError("trans:%d, vgid:%d failed to encode action to dnode:%d", pTrans->id, vgid, dnodeId); + return -1; + } + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + mError("trans:%d, vgid:%d failed to append commit log dnode:%d", pTrans->id, vgid, dnodeId); + return -1; + } + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + else + { + mInfo("trans:%d, vgid:%d cant be balanced to dnode:%d, exist:%d, online:%d", pTrans->id, vgid, dnodeId, exist, online); + } + + return 0; +} + +extern int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq); + +int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) { + return mndProcessVgroupBalanceLeaderMsgImp(pReq); +} + +#ifndef TD_ENTERPRISE +int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq) { + return 0; +} +#endif + static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup, SVgObj *pNewVgroup, SArray *pArray) { for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pArray); ++i) { @@ -1818,18 +2033,33 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); + //add first if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) + return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + //add second if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; } else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) { mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName, @@ -2265,4 +2495,4 @@ int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STimeWindow tw) { if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs, tw) != 0) return -1; return 0; -} \ No newline at end of file +} diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index d4ca81a6a944a849565f3d86aa0f47bcf00bfc66..cefc4fa63eb7ba79a46a8527ba0f7c2652b0e6ab 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -32,6 +32,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { tDecoderClear(&decoder); goto FAIL; } + tDecoderClear(&decoder); int32_t taskId = req.taskId; @@ -65,7 +66,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -75,26 +76,22 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; - pTask->pMsgCb = &pSnode->msgCb; - - pTask->startVer = ver; + pTask->chkInfo.version = ver; + pTask->pMeta = pSnode->pMeta; pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); - ASSERT(pTask->exec.executor); + int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; - streamSetupTrigger(pTask); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + ASSERT(pTask->exec.pExecutor); + streamSetupTrigger(pTask); return 0; } @@ -142,9 +139,10 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { if (pTask == NULL) { return -1; } + SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); @@ -155,7 +153,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); // 2.save task - code = streamMetaAddTask(pSnode->pMeta, -1, pTask); + code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); if (code < 0) { return -1; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 8dc3f46ae3d53d4e78c32a168e97defdcb87baf1..c713d1e247a8cd358fbe0c8a4a529f884e8810b6 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -57,13 +57,15 @@ target_sources( # tq "src/tq/tq.c" - "src/tq/tqExec.c" + "src/tq/tqUtil.c" + "src/tq/tqScan.c" "src/tq/tqMeta.c" "src/tq/tqRead.c" "src/tq/tqOffset.c" "src/tq/tqPush.c" "src/tq/tqSink.c" "src/tq/tqCommit.c" + "src/tq/tqRestore.c" "src/tq/tqSnapshot.c" "src/tq/tqOffsetSnapshot.c" ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1d14829891c602587f90f7aa38d1eeefb4b7b875..828a173108f8c3a97529cf31be09f668d47b98aa 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,6 +68,8 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); +int32_t vnodeIsCatchUp(SVnode *pVnode); +ESyncRole vnodeGetRole(SVnode *pVnode); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); @@ -92,7 +94,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); -void vnodeProposeCommitOnNeed(SVnode *pVnode); +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); // meta typedef struct SMeta SMeta; // todo: remove @@ -157,7 +159,7 @@ typedef struct SMTbCursor SMTbCursor; SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); -int32_t metaTbCursorPrev(SMTbCursor *pTbCur); +int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); #endif @@ -178,10 +180,11 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); +void tsdbReaderSetId(STsdbReader* pReader, const char* idstr); void tsdbReaderClose(STsdbReader *pReader); -bool tsdbNextDataBlock(STsdbReader *pReader); +int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); @@ -198,9 +201,10 @@ int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32 void *tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); -void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); -size_t tsdbCacheGetCapacity(SVnode *pVnode); -size_t tsdbCacheGetUsage(SVnode *pVnode); +void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); +size_t tsdbCacheGetCapacity(SVnode *pVnode); +size_t tsdbCacheGetUsage(SVnode *pVnode); +int32_t tsdbCacheGetElems(SVnode *pVnode); // tq typedef struct SMetaTableInfo { @@ -226,19 +230,12 @@ typedef struct SSnapContext { SArray *idList; int32_t index; bool withMeta; - bool queryMetaOrData; // true-get meta, false-get data + bool queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct STqReader { - // const SSubmitReq *pMsg; - // SSubmitBlk *pBlock; - // SSubmitMsgIter msgIter; - // SSubmitBlkIter blkIter; - - int64_t ver; SPackedData msg2; - int8_t setMsg; SSubmitReq2 submit; int32_t nextBlk; @@ -261,15 +258,16 @@ void tqCloseReader(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char* id); -int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); +int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); +void tqNextBlock(STqReader *pReader, SFetchRet *ret); +int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); -int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 792fed2309e95d89fdf0616cd4f28e9bf58019e6..142a1f6fb3d9fe064d8b8544f2bfd6c622b770d1 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -79,63 +79,49 @@ typedef struct { } STqExecDb; typedef struct { - int8_t subType; - - STqReader* pExecReader; - qTaskInfo_t task; + int8_t subType; + STqReader* pTqReader; + qTaskInfo_t task; union { STqExecCol execCol; STqExecTb execTb; STqExecDb execDb; }; - int32_t numOfCols; // number of out pout column, temporarily used + int32_t numOfCols; // number of out pout column, temporarily used } STqExecHandle; typedef struct { - // info - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int64_t consumerId; - int32_t epoch; - int8_t fetchMeta; - - int64_t snapshotVer; - - SWalReader* pWalReader; - - SWalRef* pRef; - - // push - STqPushHandle pushHandle; - - // exec - STqExecHandle execHandle; - + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int64_t consumerId; + int32_t epoch; + int8_t fetchMeta; + int64_t snapshotVer; + SWalReader* pWalReader; + SWalRef* pRef; + STqPushHandle pushHandle; // push + STqExecHandle execHandle; // exec } STqHandle; typedef struct { - SMqDataRsp dataRsp; + SMqDataRsp* pDataRsp; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - SRpcHandleInfo pInfo; + SRpcHandleInfo info; + STqHandle* pHandle; } STqPushEntry; struct STQ { - SVnode* pVnode; - char* path; - int64_t walLogLastVer; - - SRWLatch pushLock; - - SHashObj* pPushMgr; // consumerId -> STqPushEntry - SHashObj* pHandle; // subKey -> STqHandle - SHashObj* pCheckInfo; // topic -> SAlterCheckInfo - + SVnode* pVnode; + char* path; + int64_t walLogLastVer; + SRWLatch lock; + SHashObj* pPushMgr; // consumerId -> STqPushEntry + SHashObj* pHandle; // subKey -> STqHandle + SHashObj* pCheckInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; - - TDB* pMetaDB; - TTB* pExecStore; - TTB* pCheckStore; - - SStreamMeta* pStreamMeta; + TDB* pMetaDB; + TTB* pExecStore; + TTB* pCheckStore; + SStreamMeta* pStreamMeta; }; typedef struct { @@ -143,6 +129,10 @@ typedef struct { tmr_h timer; } STqMgmt; +typedef struct { + int32_t size; +} STqOffsetHead; + static STqMgmt tqMgmt = {0}; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); @@ -151,14 +141,14 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); // tqRead int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); -int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); +int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum, uint64_t reqId); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); -// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, + int32_t type, int32_t vgId); +int32_t tqPushDataRsp(STqPushEntry* pPushEntry, int32_t vgId); // tqMeta int32_t tqMetaOpen(STQ* pTq); @@ -170,11 +160,7 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); -typedef struct { - int32_t size; -} STqOffsetHead; - -STqOffsetStore* tqOffsetOpen(); +STqOffsetStore* tqOffsetOpen(STQ* pTq); void tqOffsetClose(STqOffsetStore*); STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset); @@ -192,6 +178,20 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); +int32_t tqStreamTasksScanWal(STQ* pTq); + +// tq util +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, + int32_t type, int64_t sver, int64_t ever); + +int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq); +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver); +void saveOffsetForAllTasks(STQ* pTq, int64_t ver); +void initOffsetForAllRestoreTasks(STQ* pTq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 15087e90f013f828eccd69012504555b8bde11f2..2a85b191a42839a580da6ff8b65ee4e9c65430b5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,12 +123,12 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tsdbRowMergerClear(SRowMerger *pMerger); -int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); @@ -224,6 +224,8 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); + // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== @@ -687,6 +689,8 @@ typedef struct SSttBlockLoadInfo { STSchema *pSchema; int16_t *colIds; int32_t numOfCols; + bool checkRemainingRow; + bool isLast; bool sttBlockLoaded; int32_t numOfStt; @@ -706,6 +710,7 @@ typedef struct SMergeTree { bool destroyLoadInfo; SSttBlockLoadInfo *pLoadInfo; const char *idStr; + bool ignoreEarlierTs; } SMergeTree; typedef struct { @@ -748,9 +753,10 @@ struct SDiskDataBuilder { int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr); + bool destroyLoadInfo, const char *idStr, bool strictTimeRange); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); +bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); @@ -780,6 +786,7 @@ typedef struct SCacheRowsReader { SDataFReader *pDataFReader; SDataFReader *pDataFReaderLast; const char *idstr; + int64_t lastTs; } SCacheRowsReader; typedef struct { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 134909090f1c14f2972b98bf00d9d7c0486b5588..a67f246e73d9656f9c184e99b796edf8ee8dc954 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -96,8 +96,7 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); -int32_t vnodeShouldCommit(SVnode* pVnode); -void vnodeUpdCommitSched(SVnode* pVnode); +int32_t vnodeShouldCommit(SVnode* pVnode, bool atExit); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir); @@ -115,7 +114,6 @@ void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); -int vnodeShouldCommit(SVnode* pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0dff2420ec4013c251de993c8195273dc29c91ac..70b997aba2cadb4e9088c11cb5115a4079382ea4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -187,11 +187,16 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); // tq -int tqInit(); -void tqCleanUp(); -STQ* tqOpen(const char* path, SVnode* pVnode); -void tqClose(STQ*); -int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); +int tqInit(); +void tqCleanUp(); +STQ* tqOpen(const char* path, SVnode* pVnode); +void tqClose(STQ*); +int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); +int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, + int32_t type); +int tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. + int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); @@ -201,7 +206,10 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); + // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); @@ -373,7 +381,6 @@ struct SVnode { STQ* pTq; SSink* pSink; tsem_t canCommit; - SVCommitSched commitSched; int64_t sync; TdThreadMutex lock; bool blocked; @@ -382,9 +389,6 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; -#if 0 - SRpcHandleInfo blockInfo; -#endif }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 05889e47673aa3ed41f5c55693dce0f2d63fa407..795f281ab2107acdce93ec2373ab99e1391b61ba 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -14,6 +14,7 @@ */ #include "meta.h" +#define TAG_FILTER_RES_KEY_LEN 32 #define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_STATS_BUCKET 16 @@ -34,7 +35,6 @@ typedef struct SMetaStbStatsEntry { typedef struct STagFilterResEntry { SList list; // the linked list of md5 digest, extracted from the serialized tag query condition uint32_t hitTimes; // queried times for current super table - uint32_t accTime; } STagFilterResEntry; struct SMetaCache { @@ -455,26 +455,37 @@ static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInv return 0; } +static FORCE_INLINE void setMD5DigestInKey(uint64_t* pBuf, const char* key, int32_t keyLen) { +// ASSERT(keyLen == sizeof(int64_t) * 2); + memcpy(&pBuf[2], key, keyLen); +} + +// the format of key: +// hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) +static void initCacheKey(uint64_t* buf, const SHashObj* pHashMap, uint64_t suid, const char* key, int32_t keyLen) { + buf[0] = (uint64_t) pHashMap; + buf[1] = suid; + setMD5DigestInKey(buf, key, keyLen); + ASSERT(keyLen == sizeof(uint64_t) * 2); +} + int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { + int32_t vgId = TD_VID(pMeta->pVnode); + // generate the composed key for LRU cache SLRUCache* pCache = pMeta->pCache->sTagFilterResCache.pUidResCache; SHashObj* pTableMap = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - uint64_t buf[4]; - *acquireRes = 0; - - buf[0] = (uint64_t)pTableMap; - buf[1] = suid; - memcpy(&buf[2], pKey, keyLen); + uint64_t key[4]; + initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen); taosThreadMutexLock(pLock); pMeta->pCache->sTagFilterResCache.accTimes += 1; - int32_t len = keyLen + sizeof(uint64_t) * 2; - LRUHandle* pHandle = taosLRUCacheLookup(pCache, buf, len); + LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN); if (pHandle == NULL) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; @@ -499,7 +510,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK uint32_t acc = pMeta->pCache->sTagFilterResCache.accTimes; if ((*pEntry)->hitTimes % 5000 == 0 && (*pEntry)->hitTimes > 0) { - metaInfo("cache hit:%d, total acc:%d, rate:%.2f", (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); + metaInfo("vgId:%d cache hit:%d, total acc:%d, rate:%.2f", vgId, (*pEntry)->hitTimes, acc, ((double)(*pEntry)->hitTimes) / acc); } taosLRUCacheRelease(pCache, pHandle, false); @@ -520,10 +531,11 @@ static void freePayload(const void* key, size_t keyLen, void* value) { return; } - SHashObj* pHashObj = (SHashObj*)p[0]; + SHashObj* pHashObj = (SHashObj*)p[0]; + STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t)); - { + if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); SListIter iter = {0}; @@ -536,9 +548,9 @@ static void freePayload(const void* key, size_t keyLen, void* value) { void* tmp = tdListPopNode(&((*pEntry)->list), pNode); taosMemoryFree(tmp); - int64_t et = taosGetTimestampUs(); - metaInfo("clear items in cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), - (et - st) / 1000.0); + double el = (taosGetTimestampUs() - st) / 1000.0; + metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), + el); break; } } @@ -563,10 +575,13 @@ static int32_t addNewEntry(SHashObj* pTableEntry, const void* pKey, int32_t keyL // check both the payload size and selectivity ratio int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { + int32_t code = 0; + int32_t vgId = TD_VID(pMeta->pVnode); + if (selectivityRatio > tsSelectivityRatio) { metaDebug("vgId:%d, suid:%" PRIu64 " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f", - TD_VID(pMeta->pVnode), suid, selectivityRatio, tsSelectivityRatio); + vgId, suid, selectivityRatio, tsSelectivityRatio); taosMemoryFree(pPayload); return TSDB_CODE_SUCCESS; } @@ -574,7 +589,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int if (payloadLen > tsTagFilterResCacheSize) { metaDebug("vgId:%d, suid:%" PRIu64 " failed to add to uid list cache, due to payload length %d greater than threshold %d", - TD_VID(pMeta->pVnode), suid, payloadLen, tsTagFilterResCacheSize); + vgId, suid, payloadLen, tsTagFilterResCacheSize); taosMemoryFree(pPayload); return TSDB_CODE_SUCCESS; } @@ -583,26 +598,17 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int SHashObj* pTableEntry = pMeta->pCache->sTagFilterResCache.pTableEntry; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - // the format of key: - // hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) + uint64_t key[4] = {0}; + initCacheKey(key, pTableEntry, suid, pKey, keyLen); - uint64_t buf[4] = {0}; - buf[0] = (uint64_t)pTableEntry; - buf[1] = suid; - memcpy(&buf[2], pKey, keyLen); - ASSERT(keyLen == 16); - - int32_t code = 0; taosThreadMutexLock(pLock); - STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t)); if (pEntry == NULL) { code = addNewEntry(pTableEntry, pKey, keyLen, suid); if (code != TSDB_CODE_SUCCESS) { goto _end; } - } else { - // check if it exists or not + } else { // check if it exists or not size_t size = listNEles(&(*pEntry)->list); if (size == 0) { tdListAppend(&(*pEntry)->list, pKey); @@ -620,12 +626,11 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL, + taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW); _end: taosThreadMutexUnlock(pLock); - - metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, + metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid, (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); return code; @@ -633,33 +638,36 @@ _end: // remove the lru cache that are expired due to the tags value update, or creating, or dropping, of child tables int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { - int32_t keyLen = sizeof(uint64_t) * 3; - uint64_t p[4] = {0}; + uint64_t p[4] = {0}; + int32_t vgId = TD_VID(pMeta->pVnode); + SHashObj* pEntryHashMap = pMeta->pCache->sTagFilterResCache.pTableEntry; - p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry; - p[1] = suid; + uint64_t dummy[2] = {0}; + initCacheKey(p, pEntryHashMap, suid, (char*) &dummy[0], 16); TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; - taosThreadMutexLock(pLock); - STagFilterResEntry** pEntry = taosHashGet(pMeta->pCache->sTagFilterResCache.pTableEntry, &suid, sizeof(uint64_t)); + + STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t)); if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) { taosThreadMutexUnlock(pLock); return TSDB_CODE_SUCCESS; } + (*pEntry)->hitTimes = 0; + SListIter iter = {0}; tdListInitIter(&(*pEntry)->list, &iter, TD_LIST_FORWARD); SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(&p[2], pNode->data, 16); - taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, keyLen); + setMD5DigestInKey(p, pNode->data, 2 * sizeof(uint64_t)); + taosLRUCacheErase(pMeta->pCache->sTagFilterResCache.pUidResCache, p, TAG_FILTER_RES_KEY_LEN); } - (*pEntry)->hitTimes = 0; tdListEmpty(&(*pEntry)->list); - taosThreadMutexUnlock(pLock); + + metaDebug("vgId:%d suid:%"PRId64" cached related tag filter uid list cleared", vgId, suid); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d3382373144aaef5efb72dce65e8d602e0064317..2359a165b7e131bde308fa1d68c7ba8d1f0c4a87 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -14,6 +14,8 @@ */ #include "meta.h" +#include "osMemory.h" +#include "tencode.h" void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); @@ -334,7 +336,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { return 0; } -int32_t metaTbCursorPrev(SMTbCursor *pTbCur) { +int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; STbCfg tbCfg; @@ -348,7 +350,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur) { tDecoderClear(&pTbCur->mr.coder); metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey); - if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { + if (pTbCur->mr.me.type == jumpTableType) { continue; } @@ -1235,9 +1237,14 @@ END: return 0; } int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; - + SMetaEntry oStbEntry = {0}; + int32_t ret = -1; + char *buf = NULL; + void *pData = NULL; + int nData = 0; + + SDecoder dc = {0}; + STbDbKey tbDbKey = {0}; STagIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1249,8 +1256,34 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { pCursor->type = param->type; metaRLock(pMeta); + + if (tdbTbGet(pMeta->pUidIdx, ¶m->suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + goto END; + } + tbDbKey.uid = param->suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + + if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { + ret = -1; + goto END; + } + ret = -1; + for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { + SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (schema->colId == param->cid && param->type == schema->type && (IS_IDX_ON(schema) || i == 0)) { + ret = 0; + } + } + if (ret != 0) { + goto END; + } + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); - if (ret < 0) { + if (ret != 0) { goto END; } @@ -1271,6 +1304,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { maxSize = 4 * nTagData + 1; buf = taosMemoryCalloc(1, maxSize); if (false == taosMbsToUcs4(tagData, nTagData, (TdUcs4 *)buf, maxSize, &maxSize)) { + ret = -1; goto END; } @@ -1288,8 +1322,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (ret != 0) { goto END; } + int cmp = 0; - if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { + ret = tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp); + if (ret != 0) { goto END; } @@ -1356,6 +1392,10 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { END: if (pCursor->pMeta) metaULock(pCursor->pMeta); if (pCursor->pCur) tdbTbcClose(pCursor->pCur); + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + taosMemoryFree(buf); taosMemoryFree(pKey); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 67ade45732865893d730f4920179bdc96a9b4925..0126d29cc93f3b2f69f3f12539c8cda62f1fa19a 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -268,7 +268,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; - ctx->queryMetaOrData = withMeta; + ctx->queryMeta = withMeta; ctx->withMeta = withMeta; ctx->idVersion = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (ctx->idVersion == NULL) { @@ -475,7 +475,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in if (ctx->index >= taosArrayGetSize(ctx->idList)) { metaDebug("tmqsnap get meta end"); ctx->index = 0; - ctx->queryMetaOrData = false; // change to get data + ctx->queryMeta = false; // change to get data return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 99e171dde1785f7aab55ab0f97b28f248744545d..ce987ca88ec930db76fcb560ff79f758e2bed1a2 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -168,7 +168,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { + if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); @@ -282,7 +282,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat .initTqReader = 1, .pStateBackend = pStreamState, }; - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; return TSDB_CODE_FAILED; @@ -669,7 +669,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); @@ -864,7 +864,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t TSDB_CHECK_CODE(code, lino, _exit); } - dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode)); if (!dstTaskInfo) { code = TSDB_CODE_RSMA_QTASKINFO_CREATE; TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4d21a2e7f35f1081d9f425cfabba3c82bf86614f..4a9bf07429a641ceedf1c131e261f72f8cee7716 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,6 +15,13 @@ #include "tq.h" +// 0: not init +// 1: already inited +// 2: wait to be inited or cleaup +#define WAL_READ_TASKS_ID (-1) + +static int32_t tqInitialize(STQ* pTq); + int32_t tqInit() { int8_t old; while (1) { @@ -51,66 +58,85 @@ void tqCleanUp() { } } -static void destroySTqHandle(void* data) { +static void destroyTqHandle(void* data) { STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); } } static void tqPushEntryFree(void* data) { STqPushEntry* p = *(void**)data; - tDeleteSMqDataRsp(&p->dataRsp); + if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + tDeleteMqDataRsp(p->pDataRsp); + } else if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__TAOSX_RSP) { + tDeleteSTaosxRsp((STaosxRsp*)p->pDataRsp); + } + + taosMemoryFree(p->pDataRsp); taosMemoryFree(p); } +static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { + return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && + pLeft->val.version <= pRight->val.version; +} + STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - taosHashSetFreeFp(pTq->pHandle, destroySTqHandle); + taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); - taosInitRWLatch(&pTq->pushLock); + taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); taosHashSetFreeFp(pTq->pPushMgr, tqPushEntryFree); pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); + tqInitialize(pTq); + return pTq; +} + +int32_t tqInitialize(STQ* pTq) { if (tqMetaOpen(pTq) < 0) { - return NULL; + return -1; } pTq->pOffsetStore = tqOffsetOpen(pTq); if (pTq->pOffsetStore == NULL) { - return NULL; + return -1; } - pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); + pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); if (pTq->pStreamMeta == NULL) { - return NULL; + return -1; } - if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pVnode->pWal)) < 0) { - return NULL; + // the version is kept in task's meta data + // todo check if this version is required or not + if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pTq->pVnode->pWal)) < 0) { + return -1; } - return pTq; + return 0; } void tqClose(STQ* pTq) { @@ -128,267 +154,161 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } -int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) { - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code); - if (code < 0) { - return -1; - } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = pReq->consumerId; +int32_t tqPushDataRsp(STqPushEntry* pPushEntry, int32_t vgId) { + SMqDataRsp* pRsp = pPushEntry->pDataRsp; + SMqRspHead* pHeader = &pPushEntry->pDataRsp->head; - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSMqMetaRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg resp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&resp); + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pPushEntry->pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); - - return 0; -} - -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { - SMqDataRsp* pRsp = &pPushEntry->dataRsp; - -#if 0 - A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - - A(!pRsp->withSchema); - A(taosArrayGetSize(pRsp->blockSchema) == 0); - - if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - A(pRsp->rspOffset.version > pRsp->reqOffset.version); - } -#endif - - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); - - if (code < 0) { - return -1; - } - - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSMqDataRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = pPushEntry->pInfo, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - - tmsgSendRsp(&rsp); + tqDoSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType, sver, ever); char buf1[80] = {0}; char buf2[80] = {0}; tFormatOffset(buf1, tListLen(buf1), &pRsp->reqOffset); tFormatOffset(buf2, tListLen(buf2), &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", - TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); - + vgId, pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } -int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { -#if 0 - A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - - A(!pRsp->withSchema); - A(taosArrayGetSize(pRsp->blockSchema) == 0); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, + int32_t type, int32_t vgId) { + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - A(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - A(pRsp->rspOffset.version >= pRsp->reqOffset.version); - } - } -#endif - - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); - if (code < 0) { - return -1; - } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = pReq->consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSMqDataRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&rsp); + tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); char buf1[80] = {0}; char buf2[80] = {0}; tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); - tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d), block num:%d, req:%s, rsp:%s", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); + + tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, + vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return 0; } -int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) { -#if 0 - A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + SMqVgOffset vgOffset = {0}; + int32_t vgId = TD_VID(pTq->pVnode); - if (pRsp->withSchema) { - A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); - } else { - A(taosArrayGetSize(pRsp->blockSchema) == 0); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { + return -1; } - if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - if (pRsp->blockNum > 0) { - A(pRsp->rspOffset.version > pRsp->reqOffset.version); - } else { - A(pRsp->rspOffset.version >= pRsp->reqOffset.version); + tDecoderClear(&decoder); + + STqOffset* pOffset = &vgOffset.offset; + + if (pOffset->val.type == TMQ_OFFSET__SNAPSHOT_DATA || pOffset->val.type == TMQ_OFFSET__SNAPSHOT_META) { + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, + pOffset->subKey, vgId, pOffset->val.uid, pOffset->val.ts); + } else if (pOffset->val.type == TMQ_OFFSET__LOG) { + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, + pOffset->val.version); + if (pOffset->val.version + 1 == sversion) { + pOffset->val.version += 1; } + } else { + tqError("invalid commit offset type:%d", pOffset->val.type); + return -1; } -#endif - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code); - if (code < 0) { - return -1; + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset != NULL && tqOffsetLessOrEqual(pOffset, pSavedOffset)) { + tqDebug("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, + vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); + return 0; // no need to update the offset value } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { + + // save the new offset value + if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { return -1; } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__TAOSX_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = pReq->consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSTaosxRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&rsp); - - char buf1[80] = {0}; - char buf2[80] = {0}; - tFormatOffset(buf1, 80, &pRsp->reqOffset); - tFormatOffset(buf2, 80, &pRsp->rspOffset); - tqDebug("taosx rsp, vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, numOfBlks:%d, req:%s, rsp:%s", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); + if (pOffset->val.type == TMQ_OFFSET__LOG) { + STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + if (pHandle && (walRefVer(pHandle->pRef, pOffset->val.version) < 0)) { + return -1; + } + } return 0; } -static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { - return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && - pLeft->val.version <= pRight->val.version; -} - -int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - STqOffset offset = {0}; +int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + SMqVgOffset vgOffset = {0}; + int32_t vgId = TD_VID(pTq->pVnode); SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSTqOffset(&decoder, &offset) < 0) { + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { return -1; } tDecoderClear(&decoder); - if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA || offset.val.type == TMQ_OFFSET__SNAPSHOT_META) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, - offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); - } else if (offset.val.type == TMQ_OFFSET__LOG) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, - TD_VID(pTq->pVnode), offset.val.version); - if (offset.val.version + 1 == sversion) { - offset.val.version += 1; - } - } else { - tqError("invalid commit offset type:%d", offset.val.type); + STqOffset* pOffset = &vgOffset.offset; + if (pOffset->val.type != TMQ_OFFSET__LOG) { + tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); return -1; } - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); - if (pSavedOffset != NULL && tqOffsetLessOrEqual(&offset, pSavedOffset)) { - return 0; // no need to update the offset value + STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + if (pHandle == NULL) { + tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, + pOffset->subKey); + terrno = TSDB_CODE_INVALID_MSG; + return -1; } - // save the new offset value - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + // 2. check consumer-vg assignment status + taosRLockLatch(&pTq->lock); + if (pHandle->consumerId != vgOffset.consumerId) { + tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->lock); return -1; } + taosRUnLockLatch(&pTq->lock); - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { - return -1; + //3. check the offset info + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset != NULL) { + if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { + tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); + return 0; // no need to update the offset value + } + + if (pSavedOffset->val.version == pOffset->val.version) { + tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, + pOffset->val.version, pSavedOffset->val.version); + return 0; } } + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + if (pOffset->val.version < sver) { + pOffset->val.version = sver; + } else if (pOffset->val.version > ever) { + pOffset->val.version = ever; + } + + // save the new offset value + tqDebug("vgId:%d sub:%s seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, + pSavedOffset->val.version); + + if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { + tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); + return -1; + } + return 0; } @@ -418,318 +338,128 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { return 0; } -static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { - return -1; - } - - pRsp->withTbName = 0; -#if 0 - pRsp->withTbName = pReq->withTbName; - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - if (pRsp->blockTbName == NULL) { - // TODO free - return -1; - } - } -#endif - - /*A(subType == TOPIC_SUB_TYPE__COLUMN);*/ - pRsp->withSchema = false; - - return 0; -} - -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->withTbName = 1; - pRsp->withSchema = 1; - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); - - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { - return -1; - } - - return 0; -} - int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { - SMqPollReq req = {0}; - int32_t code = 0; - STqOffsetVal fetchOffsetNew; - SWalCkHead* pCkHead = NULL; - + SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + terrno = TSDB_CODE_INVALID_MSG; return -1; } int64_t consumerId = req.consumerId; int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; + int32_t vgId = TD_VID(pTq->pVnode); // 1. find handle STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { - tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s not found", consumerId, TD_VID(pTq->pVnode), - req.subKey); + tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; return -1; } - // 2. check rebalance + // 2. check re-balance status + taosRLockLatch(&pTq->lock); if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->lock); return -1; } + taosRUnLockLatch(&pTq->lock); - // update epoch if need - int32_t savedEpoch = atomic_load_32(&pHandle->epoch); - while (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%"PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch); - savedEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, savedEpoch, reqEpoch); + // 3. update the epoch value + taosWLockLatch(&pTq->lock); + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + reqEpoch); + pHandle->epoch = reqEpoch; } + taosWUnLockLatch(&pTq->lock); char buf[80]; tFormatOffset(buf, 80, &reqOffset); - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s", consumerId, - req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); - - // 2.reset offset if needed - if (reqOffset.type > 0) { - fetchOffsetNew = reqOffset; - } else { - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); - if (pOffset != NULL) { - fetchOffsetNew = pOffset->val; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, &fetchOffsetNew); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vg %d, offset reset to %s", consumerId, pHandle->subKey, - TD_VID(pTq->pVnode), formatBuf); - } else { - if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (req.useSnapshot) { - if (pHandle->fetchMeta) { - tqOffsetResetToMeta(&fetchOffsetNew, 0); - } else { - tqOffsetResetToData(&fetchOffsetNew, 0, 0); - } - } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tqOffsetResetToLog(&fetchOffsetNew, pHandle->pRef->refVer - 1); - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, offset reset to %" PRId64, consumerId, - pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version); - if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { - code = -1; - } - tDeleteSMqDataRsp(&dataRsp); - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, &req); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { - code = -1; - } - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: subkey %s, no offset committed for consumer:0x%" PRIx64 - " in vg %d, subkey %s, reset none failed", - pHandle->subKey, consumerId, TD_VID(pTq->pVnode), req.subKey); - terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; - return -1; - } - } - } + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, + consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); - // lock - taosWLockLatch(&pTq->pushLock); - if (tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) { - return -1; - } - - // till now, all data has been rsp to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version) { - STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry)); - if (pPushEntry != NULL) { - pPushEntry->pInfo = pMsg->info; - memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); - dataRsp.withTbName = 0; - memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->dataRsp.head.consumerId = consumerId; - pPushEntry->dataRsp.head.epoch = reqEpoch; - pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); - - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr", - consumerId, pHandle->subKey, dataRsp.reqOffset.version, TD_VID(pTq->pVnode)); - // unlock - taosWUnLockLatch(&pTq->pushLock); - return 0; - } - } - taosWUnLockLatch(&pTq->pushLock); + return tqExtractDataForMq(pTq, pHandle, &req, pMsg); +} - if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { - code = -1; - } +int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { + SMqPollReq req = {0}; + if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 ", ts:%" PRId64 "", - consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type, - dataRsp.rspOffset.uid, dataRsp.rspOffset.ts); + int64_t consumerId = req.consumerId; + STqOffsetVal reqOffset = req.reqOffset; + int32_t vgId = TD_VID(pTq->pVnode); - tDeleteSMqDataRsp(&dataRsp); - return code; + // 1. find handle + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + return -1; } - // for taosx - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, &req); + // 2. check re-balance status + taosRLockLatch(&pTq->lock); + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, vgId, req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->lock); + return -1; + } + taosRUnLockLatch(&pTq->lock); - if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) { - return -1; - } + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - if (metaRsp.metaRspLen > 0) { - if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { - code = -1; - } - tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey %s, vg %d, send meta offset type:%d,uid:%" PRId64 - ",version:%" PRId64, - consumerId, pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid, - metaRsp.rspOffset.version); - taosMemoryFree(metaRsp.metaRsp); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, &req); - if (taosxRsp.blockNum > 0) { - if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { - code = -1; - } - tDeleteSTaosxRsp(&taosxRsp); - return code; - } else { - fetchOffsetNew = taosxRsp.rspOffset; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); + if (pOffset != NULL) { + if (pOffset->val.type != TMQ_OFFSET__LOG) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s use snapshot, no valid wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; } - tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",version:%" PRId64, - consumerId, pHandle->subKey, TD_VID(pTq->pVnode), taosxRsp.blockNum, taosxRsp.rspOffset.type, - taosxRsp.rspOffset.uid, taosxRsp.rspOffset.version); - } - - if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { - int64_t fetchVer = fetchOffsetNew.version + 1; - pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); - if (pCkHead == NULL) { - tDeleteSTaosxRsp(&taosxRsp); + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.rspOffset.version = pOffset->val.version; + } else { + if (req.useSnapshot == true) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); return -1; } - walSetReaderCapacity(pHandle->pWalReader, 2048); - - while (1) { - savedEpoch = atomic_load_32(&pHandle->epoch); - if (savedEpoch > reqEpoch) { - tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, vg %d offset %" PRId64 - ", found new consumer epoch %d, discard req epoch %d", - consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, savedEpoch, reqEpoch); - break; - } - - if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { - code = -1; - } - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } - - SWalCont* pHead = &pCkHead->head; + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, - req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); - - if (pHead->msgType == TDMT_VND_SUBMIT) { - SPackedData submit = { - .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), - .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), - .ver = pHead->version, - }; - if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, TD_VID(pTq->pVnode), - req.subKey); - return -1; - } - if (taosxRsp.blockNum > 0 /* threshold */) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { - code = -1; - } - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } else { - fetchVer++; - } - - } else { - /*A(pHandle->fetchMeta);*/ - /*A(IS_META_MSG(pHead->msgType));*/ - tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); - metaRsp.resMsgType = pHead->msgType; - metaRsp.metaRspLen = pHead->bodyLen; - metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { - code = -1; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - code = 0; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + dataRsp.rspOffset.version = sver; + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + dataRsp.rspOffset.version = ever; + } else { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, + reqOffset.type); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; } } - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); + tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever); return 0; } @@ -738,12 +468,12 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); - taosWLockLatch(&pTq->pushLock); + taosWLockLatch(&pTq->lock); int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); if (code != 0) { tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey); } - taosWUnLockLatch(&pTq->pushLock); + taosWUnLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { @@ -803,24 +533,28 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); - // todo lock - tqDebug("vgId:%d, tq process sub req %s", pTq->pVnode->config.vgId, req.subKey); + SVnode* pVnode = pTq->pVnode; + int32_t vgId = TD_VID(pVnode); + + tqDebug("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pVnode->config.vgId, req.subKey, + req.oldConsumerId, req.newConsumerId); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { - tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId is %" PRId64 "", + tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, req.vgId, req.subKey, req.newConsumerId, req.oldConsumerId); } + if (req.newConsumerId == -1) { - tqError("vgId:%d, tq invalid rebalance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); + tqError("vgId:%d, tq invalid re-balance request, new consumerId %" PRId64 "", req.vgId, req.newConsumerId); taosMemoryFree(req.qmsg); return 0; } + STqHandle tqHandle = {0}; pHandle = &tqHandle; - /*taosInitRWLatch(&pExec->lock);*/ uint64_t oldConsumerId = pHandle->consumerId; memcpy(pHandle->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -831,97 +565,112 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->fetchMeta = req.withMeta; // TODO version should be assigned and refed during preprocess - SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal); + SWalRef* pRef = walRefCommittedVer(pVnode->pWal); if (pRef == NULL) { + taosMemoryFree(req.qmsg); return -1; } + int64_t ver = pRef->refVer; pHandle->pRef = pRef; SReadHandle handle = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTableReader = true, - .initTqReader = true, - .version = ver, - }; + .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { pHandle->execHandle.execCol.qmsg = req.qmsg; req.qmsg = NULL; - pHandle->execHandle.task = - qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols, NULL); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, vgId, + &pHandle->execHandle.numOfCols, req.newConsumerId); void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.task, &scanner); - pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); + pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { - pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); + pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); buildSnapContext(handle.meta, handle.version, 0, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - + pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); pHandle->execHandle.execTb.suid = req.suid; SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); - vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList); - tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, req.suid); + vnodeGetCtbIdList(pVnode, req.suid, tbUidList); + tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pVnode->config.vgId, req.suid); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid); + tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext**)(&handle.sContext)); - pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL); + pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, vgId, NULL, req.newConsumerId); } taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); - tqDebug("try to persist handle %s consumer:0x%" PRIx64" , old consumer:0x%"PRIx64, req.subKey, pHandle->consumerId, - oldConsumerId); + tqDebug("try to persist handle %s consumer:0x%" PRIx64 " , old consumer:0x%" PRIx64, req.subKey, + pHandle->consumerId, oldConsumerId); if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { + taosMemoryFree(req.qmsg); return -1; } } else { - // TODO handle qmsg and exec modification - tqInfo("update the consumer info, old consumer id:0x%"PRIx64", new Id:0x%"PRIx64, pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, -1); - atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); - taosMemoryFree(req.qmsg); - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - qStreamCloseTsdbReader(pHandle->execHandle.task); - } + if (pHandle->consumerId == req.newConsumerId) { // do nothing + tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs", req.vgId, req.newConsumerId); + atomic_store_32(&pHandle->epoch, -1); + atomic_add_fetch_32(&pHandle->epoch, 1); + taosMemoryFree(req.qmsg); + return tqMetaSaveHandle(pTq, req.subKey, pHandle); + } else { + tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, + req.newConsumerId); - if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { - return -1; + // kill executing task + qTaskInfo_t pTaskInfo = pHandle->execHandle.task; + if (pTaskInfo != NULL) { + qKillTask(pTaskInfo, TSDB_CODE_SUCCESS); + } + + taosWLockLatch(&pTq->lock); + atomic_store_32(&pHandle->epoch, -1); + + // remove if it has been register in the push manager, and return one empty block to consumer + tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + + atomic_store_64(&pHandle->consumerId, req.newConsumerId); + atomic_add_fetch_32(&pHandle->epoch, 1); + + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pTaskInfo); + } + + taosWUnLockLatch(&pTq->lock); + if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { + taosMemoryFree(req.qmsg); + return -1; + } } - // close handle } + taosMemoryFree(req.qmsg); return 0; } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { -#if 0 - if (pTask->taskLevel == TASK_LEVEL__AGG) { - A(taosArrayGetSize(pTask->childEpInfo) != 0); - } -#endif - + int32_t vgId = TD_VID(pTq->pVnode); + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; - + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -931,14 +680,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; - pTask->pMsgCb = &pTq->pVnode->msgCb; - - pTask->startVer = ver; + pTask->pMeta = pTq->pStreamMeta; + pTask->chkInfo.version = ver; // expand executor if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + } else { + pTask->status.taskStatus = TASK_STATUS__RESTORE; } if (pTask->taskLevel == TASK_LEVEL__SOURCE) { @@ -948,13 +698,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } SReadHandle handle = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTqReader = 1, - .pStateBackend = pTask->pState, - }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - if (pTask->exec.executor == NULL) { + .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } @@ -963,13 +710,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); - if (pTask->exec.executor == NULL) { + + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } } @@ -983,18 +729,30 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->tbSink.vnode = pTq->pVnode; pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; - /*A(pTask->tbSink.pSchemaWrapper);*/ - /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/ + int32_t ver1 = 1; + SMetaInfo info = {0}; + int32_t code = metaGetInfo(pTq->pVnode->pMeta, pTask->tbSink.stbUid, &info, NULL); + if (code == TSDB_CODE_SUCCESS) { + ver1 = info.skmVer; + } - pTask->tbSink.pTSchema = - tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); - ASSERT(pTask->tbSink.pTSchema); + SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper; + pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); + if(pTask->tbSink.pTSchema == NULL) { + return -1; + } + } + + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); } streamSetupTrigger(pTask); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, + pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); - tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", TD_VID(pTq->pVnode), pTask->taskId, - pTask->selfChildId, pTask->taskLevel); + // next valid version will add one + pTask->chkInfo.version += 1; return 0; } @@ -1017,18 +775,24 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { .upstreamNodeId = req.upstreamNodeId, .upstreamTaskId = req.upstreamTaskId, }; + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) { - rsp.status = 1; + if (pTask) { + rsp.status = (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) ? 1 : 0; + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + + tqDebug("tq recv task check req(reqId:0x%" PRIx64 + ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d", + rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId, + rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; + tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64 + ") %d at node %d, check req from task %d at node %d, rsp status %d", + taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, + rsp.status); } - if (pTask) streamMetaReleaseTask(pTq->pStreamMeta, pTask); - - tqDebug("tq recv task check req(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - SEncoder encoder; int32_t code; int32_t len; @@ -1046,13 +810,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tEncodeSStreamTaskCheckRsp(&encoder, &rsp); tEncoderClear(&encoder); - SRpcMsg rspMsg = { - .code = 0, - .pCont = buf, - .contLen = sizeof(SMsgHead) + len, - .info = pMsg->info, - }; - + SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info }; tmsgSendRsp(&rspMsg); return 0; } @@ -1068,8 +826,8 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32 tDecoderClear(&decoder); return -1; } - tDecoderClear(&decoder); + tDecoderClear(&decoder); tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); @@ -1101,17 +859,20 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); return -1; } + tDecoderClear(&decoder); - // 2.save task - code = streamMetaAddTask(pTq->pStreamMeta, sversion, pTask); + // 2.save task, use the newest commit version as the initial start version of stream task. + code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); if (code < 0) { + tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, + streamMetaGetNumOfTasks(pTq->pStreamMeta)); return -1; } @@ -1120,6 +881,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamTaskCheckDownstream(pTask, sversion); } + tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", TD_VID(pTq->pVnode), + pTask->id.idStr, pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta)); return 0; } @@ -1135,7 +898,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { } // check param - int64_t fillVer1 = pTask->startVer; + int64_t fillVer1 = pTask->chkInfo.version; if (fillVer1 <= 0) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return -1; @@ -1144,7 +907,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { // do recovery step 1 streamSourceRecoverScanStep1(pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1159,7 +922,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { return 0; } @@ -1201,7 +964,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1320,7 +1083,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); if (!failed) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -1329,8 +1092,8 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { pRefBlock->dataRef = pRef; atomic_add_fetch_32(pRefBlock->dataRef, 1); - if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); @@ -1338,7 +1101,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } @@ -1364,13 +1127,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayPush(pStreamBlock->blocks, &block); if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } } else { @@ -1383,18 +1146,32 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit2* pSubmit = NULL; +static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit, + const char* key, int64_t ver) { + doSaveTaskOffset(pOffsetStore, key, ver); + int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver); - pSubmit = streamDataSubmitNew(submit); + // remove the offset, if all functions are completed successfully. + if (code == TSDB_CODE_SUCCESS) { + tqOffsetDelete(pOffsetStore, key); + } + + return code; +} + +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { +#if 0 + void* pIter = NULL; + SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); - failed = true; + saveOffsetForAllTasks(pTq, submit.ver); + return -1; } + SArray* pInputQueueFullTasks = taosArrayInit(4, POINTER_BYTES); + while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -1402,46 +1179,79 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId, + pTask->status.taskStatus); + continue; + } - if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { - tqError("stream task input failed, task id %d", pTask->taskId); - continue; - } + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - if (streamSchedExec(pTask) < 0) { - tqError("stream task launch failed, task id %d", pTask->taskId); - continue; + if (tInputQueueIsFull(pTask)) { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + + int64_t ver = submit.ver; + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver); + } else { + ver = pOffset->val.version; } - } else { - streamTaskInputFail(pTask); + + tqDebug("s-task:%s input queue is full, discard submit block, ver:%" PRId64, pTask->id.idStr, ver); + taosArrayPush(pInputQueueFullTasks, &pTask); + continue; } - } - if (pSubmit) { - streamDataSubmitRefDec(pSubmit); - taosFreeQitem(pSubmit); + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + ASSERT(pOffset == NULL); + + addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver); } - return failed ? -1 : 0; + streamDataSubmitDestroy(pSubmit); + taosFreeQitem(pSubmit); +#endif + + tqStartStreamTasks(pTq); + return 0; } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRunReq(pTask); + + int32_t taskId = pReq->taskId; + int32_t vgId = TD_VID(pTq->pVnode); + + if (taskId == WAL_READ_TASKS_ID) { // all tasks are extracted submit data from the wal + tqStreamTasksScanWal(pTq); + return 0; + } + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + if (pTask != NULL) { + if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { + tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); + streamProcessRunReq(pTask); + } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { + tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.version); + streamProcessRunReq(pTask); + } else { + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); + } + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + tqStartStreamTasks(pTq); return 0; } else { + tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId); return -1; } } @@ -1454,14 +1264,10 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, exec); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; @@ -1474,7 +1280,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code: %x", pMsg->code); + tqDebug("recv dispatch rsp, code:%x", pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1502,10 +1308,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessRetrieveReq(pTask, &req, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); @@ -1541,15 +1344,14 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return 0; + } else { + tDeleteStreamDispatchReq(&req); } code = TSDB_CODE_STREAM_TASK_NOT_EXIST; @@ -1559,10 +1361,7 @@ FAIL: SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { - SRpcMsg rsp = { - .code = TSDB_CODE_OUT_OF_MEMORY, - .info = pMsg->info, - }; + SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info }; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1580,11 +1379,7 @@ FAIL: pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; SRpcMsg rsp = { - .code = code, - .info = pMsg->info, - .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), - .pCont = pRspHead, - }; + .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead}; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1593,3 +1388,45 @@ FAIL: } int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } + +int32_t tqStartStreamTasks(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + + SStreamMeta* pMeta = pTq->pStreamMeta; + taosWLockLatch(&pMeta->lock); + int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exists", vgId); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + + pMeta->walScan += 1; + + if (pMeta->walScan > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScan); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno)); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return -1; + } + + tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); + initOffsetForAllRestoreTasks(pTq); + + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = WAL_READ_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 7fc66c49192eb011ce674bc7a2d396825ba1b435..0f5daa31ad412c8ed0773cca8e09cdeeb0fa80f5 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -16,10 +16,13 @@ #include "tq.h" int tqCommit(STQ* pTq) { +#if 0 + // stream meta commit does not be aligned to the vnode commit if (streamMetaCommit(pTq->pStreamMeta) < 0) { tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr()); return -1; } +#endif return tqOffsetCommitFile(pTq->pOffsetStore); } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 25ee5637967229b5fb6b5aaa0e7daede6d9283c0..cd8cefb30741d2d42891380ede286aab6df91f0a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -269,12 +269,13 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { } int32_t tqMetaRestoreHandle(STQ* pTq) { - int code = 0; + int code = 0; TBC* pCur = NULL; if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { return -1; } + int32_t vgId = TD_VID(pTq->pVnode); void* pKey = NULL; int kLen = 0; void* pVal = NULL; @@ -284,7 +285,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { tdbTbcMoveToFirst(pCur); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - STqHandle handle; + STqHandle handle = {0}; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecodeSTqHandle(&decoder, &handle); tDecoderClear(&decoder); @@ -306,7 +307,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = - qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL); + qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, vgId, &handle.execHandle.numOfCols, 0); if (handle.execHandle.task == NULL) { tqError("cannot create exec task for %s", handle.subKey); code = -1; @@ -319,19 +320,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { code = -1; goto end; } - handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - if (handle.execHandle.pExecReader == NULL) { + handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); + if (handle.execHandle.pTqReader == NULL) { tqError("cannot extract exec reader for %s", handle.subKey); code = -1; goto end; } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, NULL, NULL); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__TABLE) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); @@ -340,17 +341,17 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { tqDebug("vgId:%d, tq try to get all ctb, suid:%" PRId64, pTq->pVnode->config.vgId, handle.execHandle.execTb.suid); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid); + tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); + tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); - handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, NULL, NULL); + handle.execHandle.task = qCreateQueueExecTaskInfo(NULL, &reader, vgId, NULL, 0); } - tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, TD_VID(pTq->pVnode)); + tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle.subKey, handle.consumerId, vgId); taosHashPut(pTq->pHandle, pKey, kLen, &handle, sizeof(STqHandle)); } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 66d1ac2c7e530e4b8c732deaaa83afc6191b5e44..e8051a140616d2c8296a42c0ca4dca11c3816cf9 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -128,31 +128,35 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { - if (!pStore->needCommit) return 0; + if (!pStore->needCommit) { + return 0; + } + // TODO file name should be with a newer version char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); - tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, - sysErrStr); + const char* err = strerror(errno); + tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err); taosMemoryFree(fname); return -1; } + taosMemoryFree(fname); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pStore->pHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + STqOffset* pOffset = (STqOffset*)pIter; int32_t bodyLen; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); + if (code < 0) { taosHashCancelIterate(pStore->pHash, pIter); return -1; @@ -166,6 +170,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { SEncoder encoder; tEncoderInit(&encoder, abuf, bodyLen); tEncodeSTqOffset(&encoder, pOffset); + // write file int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { @@ -174,8 +179,10 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { taosMemoryFree(buf); return -1; } + taosMemoryFree(buf); } + // close and rename file taosCloseFile(&pFile); pStore->needCommit = 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 5a25d7e89468b29e6c597c555d27afb259f09b63..43463f67b7dc80be6e0175c69f051b99f84e8670 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -30,7 +30,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); if (pRsp->blockNum > 0) { *ppSubmit = pSubmit; return 0; @@ -58,7 +58,7 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { } while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) { streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); if (pSubmit == NULL) break; } @@ -120,7 +120,7 @@ int32_t tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHan int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) { int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus); if (inputStatus == TASK_INPUT_STATUS__NORMAL) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); + SStreamDataSubmit* pSubmitClone = streamSubmitBlockClone(pSubmit); if (pSubmitClone == NULL) { return -1; } @@ -206,137 +206,154 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ } #endif -int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { +typedef struct { + void* pKey; + int64_t keyLen; +} SItem; + +static void recordPushedEntry(SArray* cachedKey, void* pIter); +static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq); + +static void freeItem(void* param) { + SItem* p = (SItem*)param; + taosMemoryFree(p->pKey); +} + +static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData, + int32_t dataLen, SArray* pCachedKey) { + STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; + + SMqDataRsp* pRsp = pPushEntry->pDataRsp; + if (pRsp->reqOffset.version >= ver) { + tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId, + pRsp->reqOffset.version, ver); + return; + } + + qTaskInfo_t pTaskInfo = pExec->task; + + // prepare scan mem data + SPackedData submit = {.msgStr = pData, .msgLen = dataLen, .ver = ver}; + + if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { + return; + } + qStreamSetOpen(pTaskInfo); + // here start to scan submit block to extract the subscribed data + int32_t totalRows = 0; + + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(pTaskInfo, &pDataBlock, &ts) < 0) { + tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr()); + } + + if (pDataBlock == NULL) { + break; + } + + tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + } + + tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d, rows:%d", vgId, pPushEntry->subKey, pRsp->blockNum, + totalRows); + + if (pRsp->blockNum > 0) { + tqOffsetResetToLog(&pRsp->rspOffset, ver); + tqPushDataRsp(pPushEntry, vgId); + recordPushedEntry(pCachedKey, pIter); + } +} + +int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); int32_t len = msgLen - sizeof(SSubmitReq2Msg); + int32_t vgId = TD_VID(pTq->pVnode); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost - taosWLockLatch(&pTq->pushLock); + taosWLockLatch(&pTq->lock); int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr); if (numOfRegisteredPush > 0) { tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d", - pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); - - SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); - SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); + vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); - void* data = taosMemoryMalloc(len); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); - taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); - taosArrayDestroy(cachedKeyLens); - - // unlock - taosWUnLockLatch(&pTq->pushLock); + tqError("failed to copy data for stream since out of memory, vgId:%d", vgId); + taosWUnLockLatch(&pTq->lock); return -1; } memcpy(data, pReq, len); - void* pIter = NULL; + SArray* cachedKey = taosArrayInit(0, sizeof(SItem)); + void* pIter = NULL; + while (1) { pIter = taosHashIterate(pTq->pPushMgr, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey)); if (pHandle == NULL) { - tqDebug("vgId:%d, cannot find handle %s", pTq->pVnode->config.vgId, pPushEntry->subKey); - continue; - } - - if (pPushEntry->dataRsp.reqOffset.version >= ver) { - tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", - pTq->pVnode->config.vgId, pPushEntry->dataRsp.reqOffset.version, ver); + tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId, + pPushEntry->subKey); continue; } STqExecHandle* pExec = &pHandle->execHandle; - qTaskInfo_t task = pExec->task; - - SMqDataRsp* pRsp = &pPushEntry->dataRsp; - - // prepare scan mem data - SPackedData submit = { - .msgStr = data, - .msgLen = len, - .ver = ver, - }; - qStreamSetScanMemData(task, submit); - - // here start to scan submit block to extract the subscribed data - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr()); - } - - if (pDataBlock == NULL) { - break; - } - - tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; - } - - tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d", pTq->pVnode->config.vgId, pPushEntry->subKey, - pRsp->blockNum); - if (pRsp->blockNum > 0) { - // set offset - tqOffsetResetToLog(&pRsp->rspOffset, ver); - // remove from hash - size_t kLen; - void* key = taosHashGetKey(pIter, &kLen); - void* keyCopy = taosMemoryCalloc(1, kLen + 1); - memcpy(keyCopy, key, kLen); - - taosArrayPush(cachedKeys, &keyCopy); - taosArrayPush(cachedKeyLens, &kLen); - - tqPushDataRsp(pTq, pPushEntry); - } + doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey); } - // delete entry - for (int32_t i = 0; i < taosArrayGetSize(cachedKeys); i++) { - void* key = taosArrayGetP(cachedKeys, i); - size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i); - if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) { - tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key); - } - } - taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); - taosArrayDestroy(cachedKeyLens); + doRemovePushedEntry(cachedKey, pTq); + taosArrayDestroyEx(cachedKey, freeItem); taosMemoryFree(data); } + // unlock - taosWUnLockLatch(&pTq->pushLock); + taosWUnLockLatch(&pTq->lock); } - if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; + tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks)); + + // push data for stream processing: + // 1. the vnode has already been restored. + // 2. the vnode should be the leader. + // 3. the stream is not suspended yet. + if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) { + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { + return 0; + } + if (msgType == TDMT_VND_SUBMIT) { +#if 0 void* data = taosMemoryMalloc(len); if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId); return -1; } - memcpy(data, pReq, len); - SPackedData submit = { - .msgStr = data, - .msgLen = len, - .ver = ver, - }; - tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); + memcpy(data, pReq, len); + SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver}; + tqDebug("vgId:%d tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", vgId, data, len, ver, pReq); + tqProcessSubmitReq(pTq, submit); +#endif + SPackedData submit = {0}; tqProcessSubmitReq(pTq, submit); } + if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); } @@ -344,3 +361,86 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) return 0; } + +int32_t tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, + int32_t type) { + uint64_t consumerId = pRequest->consumerId; + int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pTqHandle = pHandle; + + STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry)); + if (pPushEntry == NULL) { + tqDebug("tmq poll: consumer:0x%" PRIx64 ", vgId:%d failed to malloc, size:%d", consumerId, vgId, + (int32_t)sizeof(STqPushEntry)); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pPushEntry->pHandle = pHandle; + pPushEntry->info = pRpcMsg->info; + memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); + + if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp)); + memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp)); + } else if (type == TMQ_MSG_TYPE__POLL_RSP) { + pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp)); + memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(SMqDataRsp)); + } + + SMqRspHead* pHead = &pPushEntry->pDataRsp->head; + + pHead->consumerId = consumerId; + pHead->epoch = pRequest->epoch; + pHead->mqMsgType = type; + + taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*)); + + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", + consumerId, pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr)); + return 0; +} + +int32_t tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { + int32_t vgId = TD_VID(pTq->pVnode); + STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen); + + if (pEntry != NULL) { + uint64_t cId = (*pEntry)->pDataRsp->head.consumerId; + ASSERT(consumerId == cId); + + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId, + (*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1); + + if (rspConsumer) { // rsp the old consumer with empty block. + tqPushDataRsp(*pEntry, vgId); + } + + taosHashRemove(pTq->pPushMgr, pKey, keyLen); + } + + return 0; +} + +void recordPushedEntry(SArray* cachedKey, void* pIter) { + size_t kLen = 0; + void* key = taosHashGetKey(pIter, &kLen); + SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; + taosArrayPush(cachedKey, &item); +} + +void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys); + + for (int32_t i = 0; i < numOfKeys; i++) { + SItem* pItem = taosArrayGet(pCachedKeys, i); + if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { + tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey); + } + } + + if (numOfKeys > 0) { + tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); + } +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index bf73cca9252b80dae6bd988ce0d78b88e6eca5c7..25ab7209d2ca33c928ba5b648348bcc329189d1f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -113,7 +113,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->execHandle.pExecReader->pVnodeMeta, 0); + metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); if (metaGetTableEntryByName(&mr, req.tbName) < 0) { metaReaderClear(&mr); @@ -183,22 +183,24 @@ end: return tbSuid == realTbSuid; } -int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { +int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead, uint64_t reqId) { int32_t code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + taosThreadMutexLock(&pHandle->pWalReader->mutex); int64_t offset = *fetchOffset; while (1) { if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) { - tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", - pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return, reqId:0x%"PRIx64, + pHandle->consumerId, pHandle->epoch, vgId, offset, reqId); *fetchOffset = offset - 1; code = -1; goto END; } - tqDebug("vgId:%d, taosx get msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, offset, - TMSG_INFO((*ppCkHead)->head.msgType)); + tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64, vgId, + pHandle->consumerId, offset, TMSG_INFO((*ppCkHead)->head.msgType), reqId); if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) { code = walFetchBody(pHandle->pWalReader, ppCkHead); @@ -241,6 +243,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea offset++; } } + END: taosThreadMutexUnlock(&pHandle->pWalReader->mutex); return code; @@ -259,8 +262,6 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - /*pReader->pMsg = NULL;*/ - pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; pReader->cachedSchemaSuid = 0; @@ -287,163 +288,112 @@ void tqCloseReader(STqReader* pReader) { } // free hash taosHashCleanup(pReader->tbIdHash); + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); taosMemoryFree(pReader); } int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { - // todo set the correct vgId - tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id); if (walReadSeekVer(pReader->pWalReader, ver) < 0) { - tqError("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); return -1; - } else { - tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); - return 0; } + tqDebug("wal reader seek to ver:%"PRId64" %s", ver, id); + return 0; } -int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->msg2.msgStr != NULL; +int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { + if (walNextValidMsg(pReader) < 0) { + return -1; + } + + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pReader->pHead->head.version; + + void* data = taosMemoryMalloc(len); + if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); + return -1; + } + + memcpy(data, pBody, len); + *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; + return 0; +} +void tqNextBlock(STqReader* pReader, SFetchRet* ret) { while (1) { - if (!fromProcessedMsg) { + if (pReader->msg2.msgStr == NULL) { if (walNextValidMsg(pReader->pWalReader) < 0) { -// pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - if(pReader->pWalReader->curInvalid == 0){ - pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - }else{ - pReader->ver = walGetLastVer(pReader->pWalReader->pWal); - } - ret->offset.type = TMQ_OFFSET__LOG; - - ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; - tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); - return -1; + return; } - void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + void* pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pReader->pWalReader->pHead->head.version; - tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); } - while (tqNextDataBlock2(pReader)) { - // TODO mem free + while (tqNextDataBlock(pReader)) { memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); if (code != 0 || ret->data.info.rows == 0) { continue; } - ret->fetchType = FETCH_TYPE__DATA; - tqDebug("return data rows %d", ret->data.info.rows); - return 0; - } - if (fromProcessedMsg) { - ret->offset.type = TMQ_OFFSET__LOG; - ret->offset.version = pReader->ver; - ret->fetchType = FETCH_TYPE__SEP; - tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); - return 0; + ret->fetchType = FETCH_TYPE__DATA; + return; } } } -#if 0 -int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { - pReader->pMsg = pMsg; - -// if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; -// while (true) { -// if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1; -// tqDebug("submitnext vgId:%d, block:%p, dataLen:%d, len:%d, uid:%"PRId64, pWalReader->pWal->cfg.vgId, pReader->pBlock, pReader->msgIter.dataLen, -// pReader->msgIter.len, pReader->msgIter.uid); -// if (pReader->pBlock == NULL) break; -// } - - if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; - pReader->ver = ver; - memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); - return 0; -} -#endif - -int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { - ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0)); - +int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { pReader->msg2.msgStr = msgStr; pReader->msg2.msgLen = msgLen; pReader->msg2.ver = ver; - pReader->ver = ver; tqDebug("tq reader set msg %p %d", msgStr, msgLen); - - if (pReader->setMsg == 0) { - SDecoder decoder; - tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); - if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { - ASSERT(0); - } + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); - pReader->setMsg = 1; + tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); + return -1; } + tDecoderClear(&decoder); return 0; } -#if 0 bool tqNextDataBlock(STqReader* pReader) { - if (pReader->pMsg == NULL) return false; - while (1) { - if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) { - return false; - } - if (pReader->pBlock == NULL) { - pReader->pMsg = NULL; - return false; - } - - if (pReader->tbIdHash == NULL) { - return true; - } - void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); - /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/ - if (ret != NULL) { - /*tqDebug("find uid %" PRId64, pHandle->msgIter.uid);*/ - return true; - } - } - return false; -} -#endif - -bool tqNextDataBlock2(STqReader* pReader) { if (pReader->msg2.msgStr == NULL) { return false; } - ASSERT(pReader->setMsg == 1); - - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, - pReader->msg2.ver, pReader->nextBlk); - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - ASSERT(pSubmitTbData->uid); + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); - if (pReader->tbIdHash == NULL) return true; + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) { + return true; + } void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { + tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid); return true; + } else { + tqDebug("tq reader discard block, uid:%"PRId64", continue", pSubmitTbData->uid); } + pReader->nextBlk++; } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -452,7 +402,6 @@ bool tqNextDataBlock2(STqReader* pReader) { bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { if (pReader->msg2.msgStr == NULL) return false; - ASSERT(pReader->setMsg == 1); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { @@ -467,7 +416,6 @@ bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -503,67 +451,15 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -#if 0 -bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { - while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; - } - if (pHandle->pBlock == NULL) return false; - - void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); - if (ret == NULL) { - return true; - } - } - return false; -} - -int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { - // - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 - "), version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - } - return 0; -} -#endif - int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); - ASSERT(pReader->nextBlk < blockSz); - - tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); - + tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; - if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + if (pSubmitTbDataRet) { + *pSubmitTbDataRet = pSubmitTbData; + } + int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -663,33 +559,27 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD int32_t targetIdx = 0; int32_t sourceIdx = 0; while (targetIdx < colActual) { - ASSERT(sourceIdx < numOfCols); - + if(sourceIdx >= numOfCols){ + tqError("tqRetrieveDataBlock2 sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); + goto FAIL; + } SColData* pCol = taosArrayGet(pCols, sourceIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColVal colVal; - ASSERT(pCol->nVal == numOfRows); + if(pCol->nVal != numOfRows){ + tqError("tqRetrieveDataBlock2 pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + goto FAIL; + } if (pCol->cid < pColData->info.colId) { sourceIdx++; } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { tColDataGetValue(pCol, i, &colVal); -#if 0 - void* val = NULL; - if (IS_STR_DATA_TYPE(colVal.type)) { - val = colVal.value.pData; - } else { - val = &colVal.value.val; - } - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - goto FAIL; - } -#endif if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -707,7 +597,10 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD sourceIdx++; targetIdx++; } else { - ASSERT(0); + for (int32_t i = 0; i < pCol->nVal; i++) { + colDataSetNULL(pColData, i); + } + targetIdx++; } } } else { @@ -720,8 +613,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD for (int32_t j = 0; j < colActual; j++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); while (1) { - ASSERT(sourceIdx < pTschema->numOfCols); - SColVal colVal; tRowGet(pRow, pTschema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { @@ -730,7 +621,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else if (colVal.cid == pColData->info.colId) { if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -739,7 +630,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else { colDataSetNULL(pColData, i); } - /*val = colVal.value.pData;*/ } else { if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; @@ -749,7 +639,8 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD sourceIdx++; break; } else { - ASSERT(0); + colDataSetNULL(pColData, i); + break; } } } @@ -764,253 +655,6 @@ FAIL: return -1; } -#if 0 -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { - // TODO: cache multiple schema - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - - if (colNumNeed == 0) { - int32_t colMeta = 0; - while (colMeta < pSchemaWrapper->nCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - } - } else { - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } - - int32_t colMeta = 0; - int32_t colNeed = 0; - while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); - if (colIdSchema < colIdNeed) { - colMeta++; - } else if (colIdSchema > colIdNeed) { - colNeed++; - } else { - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - colNeed++; - } - } - } - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - int32_t colActual = blockDataGetNumOfCols(pBlock); - - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - int32_t curRow = 0; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = pReader->msgIter.numOfRows; - pBlock->info.version = pReader->pMsg->version; - pBlock->info.dataLoad = 1; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - tdSTSRowIterReset(&iter, row); - // get all wanted col of that block - for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { - goto FAIL; - } - } - curRow++; - } - return 0; - -FAIL: - blockDataFreeRes(pBlock); - return -1; -} - -int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) { - int32_t sversion = htonl(pReader->pBlock->sversion); - - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colAtMost = pSchemaWrapper->nCols; - - int32_t curRow = 0; - int32_t lastRow = 0; - - char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); - if (assigned == NULL) return -1; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - bool buildNew = false; - tdSTSRowIterReset(&iter, row); - - tqDebug("vgId:%d, row of block %d", pReader->pWalReader->pWal->cfg.vgId, curRow); - for (int32_t i = 0; i < colAtMost; i++) { - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) { - break; - } - tqDebug("vgId:%d, %d col, type %d", pReader->pWalReader->pWal->cfg.vgId, i, sVal.valType); - if (curRow == 0) { - assigned[i] = sVal.valType != TD_VTYPE_NONE; - buildNew = true; - } else { - bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE; - if (currentRowAssigned != assigned[i]) { - assigned[i] = currentRowAssigned; - buildNew = true; - } - } - } - - if (buildNew) { - if (taosArrayGetSize(blocks) > 0) { - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - lastRow = curRow; - } - SSDataBlock* pBlock = createDataBlock(); - SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (tqMaskBlock(pSW, pBlock, pSchemaWrapper, assigned) < 0) { - blockDataDestroy(pBlock); - goto FAIL; - } - SSDataBlock block = {0}; - assignOneDataBlock(&block, pBlock); - blockDataDestroy(pBlock); - - tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(block.pDataBlock)); - - taosArrayPush(blocks, &block); - taosArrayPush(schemas, &pSW); - } - - SSDataBlock* pBlock = taosArrayGetLast(blocks); - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = 0; - pBlock->info.version = pReader->pMsg->version; - - tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(blocks)); - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - tdSTSRowIterReset(&iter, row); - for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - - ASSERT(sVal.valType != TD_VTYPE_NONE); - - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { - goto FAIL; - } - tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i, - sVal.valType == TD_VTYPE_NULL); - } - curRow++; - } - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - - taosMemoryFree(assigned); - return 0; - -FAIL: - taosMemoryFree(assigned); - return -1; -} -#endif - int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); @@ -1288,7 +932,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } -int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { if (pReader->tbIdHash == NULL) { pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pReader->tbIdHash == NULL) { @@ -1297,8 +941,9 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { } } - for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); + int32_t numOfTables = taosArrayGetSize(pTableUidList); + for (int i = 0; i < numOfTables; i++) { + int64_t* pKey = (int64_t*)taosArrayGet(pTableUidList, i); taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } @@ -1314,30 +959,34 @@ int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } +// todo update the table list in wal reader int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { - void* pIter = NULL; + void* pIter = NULL; + int32_t vgId = TD_VID(pTq->pVnode); + + // update the table list for each consumer handle while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { break; } - STqHandle* pExec = (STqHandle*)pIter; - if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); + STqHandle* pTqHandle = (STqHandle*)pIter; + if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for %s", pExec->subKey); + tqError("update qualified table error for %s", pTqHandle->subKey); continue; } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); for (int32_t i = 0; i < sz; i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); + taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); } } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); SMetaReader mr = {0}; @@ -1352,35 +1001,43 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } tDecoderClear(&mr.coder); - - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pExec->execHandle.execTb.suid) { + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { tqDebug("table uid %" PRId64 " does not add to tq handle", *id); continue; } + tqDebug("table uid %" PRId64 " add to tq handle", *id); taosArrayPush(qa, id); } + metaReaderClear(&mr); if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pExec->execHandle.pExecReader, qa); + tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); } + taosArrayDestroy(qa); } else { - tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList); + tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } } } + + // update the table list handle for each stream scanner/wal reader while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); + int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for stream task %d", pTask->taskId); + tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); continue; } } } + return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c new file mode 100644 index 0000000000000000000000000000000000000000..54fcc04b62e817f9d8192c40e91a7f3f50e5f862 --- /dev/null +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); +static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList); + +// this function should be executed by stream threads. +// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure +// will not stop eventually. +int tqStreamTasksScanWal(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + int64_t st = taosGetTimestampMs(); + + while (1) { + tqInfo("vgId:%d continue check if data in wal are available", vgId); + + // check all restore tasks + bool allFull = true; + streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &allFull); + + int32_t times = 0; + + if (allFull) { + taosWLockLatch(&pMeta->lock); + pMeta->walScan -= 1; + times = pMeta->walScan; + + if (pMeta->walScan <= 0) { + taosWUnLockLatch(&pMeta->lock); + break; + } + + taosWUnLockLatch(&pMeta->lock); + tqInfo("vgId:%d scan wal for stream tasks for %d times", vgId, times); + } + } + + double el = (taosGetTimestampMs() - st) / 1000.0; + tqInfo("vgId:%d scan wal for stream tasks completed, elapsed time:%.2f sec", vgId, el); + return 0; +} + +int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { + void* pIter = NULL; + int32_t vgId = pStreamMeta->vgId; + + *pScanIdle = true; + + bool allWalChecked = true; + tqDebug("vgId:%d start to check wal to extract new submit block", vgId); + + while (1) { + pIter = taosHashIterate(pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + if (tInputQueueIsFull(pTask)) { + tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); + continue; + } + + *pScanIdle = false; + + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pOffsetStore, key); + ASSERT(pOffset != NULL); + + // seek the stored version and extract data from WAL + int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + continue; + } + + // append the data for the stream + tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); + + SPackedData packData = {0}; + code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); + if (code != TSDB_CODE_SUCCESS) { // failed, continue + continue; + } + + SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + continue; + } + + allWalChecked = false; + + tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); + code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); + if (code == TSDB_CODE_SUCCESS) { + pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); + tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, + pOffset->val.version); + } else { + // do nothing + } + + streamDataSubmitDestroy(p); + taosFreeQitem(p); + } + + if (allWalChecked) { + *pScanIdle = true; + } + return 0; +} + diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqScan.c similarity index 68% rename from source/dnode/vnode/src/tq/tqExec.c rename to source/dnode/vnode/src/tq/tqScan.c index f97c5ce93c7b46c74946d87186f405c2b93101e8..27db66f048df70ebeaa0e566a591f3ce1ffbcebd 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -18,7 +18,9 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) return -1; + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; pRetrieve->useconds = 0; @@ -31,11 +33,12 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t actualLen += sizeof(SRetrieveTableRsp); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); - return 0; + + return TSDB_CODE_SUCCESS; } static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader->pSchemaWrapper); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { return -1; } @@ -46,11 +49,13 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); + // TODO add reference to gurantee success if (metaGetTableEntryByUidCache(&mr, uid) < 0) { metaReaderClear(&mr); return -1; } + for (int32_t i = 0; i < n; i++) { char* tbName = taosStrdup(mr.me.name); taosArrayPush(pRsp->blockTbName, &tbName); @@ -60,60 +65,52 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in } int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { - const STqExecHandle* pExec = &pHandle->execHandle; + const int32_t MAX_ROWS_TO_RETURN = 4096; + int32_t vgId = TD_VID(pTq->pVnode); + int32_t code = 0; + int32_t totalRows = 0; - qTaskInfo_t task = pExec->task; + const STqExecHandle* pExec = &pHandle->execHandle; + qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return 0; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - pRsp->rspOffset = *pOffset; - return 0; - } - } + tqError("prepare scan failed, return"); + return -1; } - int32_t rowCnt = 0; while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); - if (qExecTask(task, &pDataBlock, &ts) < 0) { - tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + qStreamSetOpen(task); + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); + if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); return -1; } - tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, + pDataBlock); + // current scan should be stopped asap, since the rebalance occurs. if (pDataBlock == NULL) { break; } - tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; - - if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - rowCnt += pDataBlock->info.rows; - if (rowCnt >= 4096) break; + code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d, failed to add block to rsp msg", vgId); + return code; } - } - - if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - return -1; - } - if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; + pRsp->blockNum++; + totalRows += pDataBlock->info.rows; + if (totalRows >= MAX_ROWS_TO_RETURN) { + break; + } } - ASSERT(!(pRsp->withTbName || pRsp->withSchema)); + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", + pHandle->consumerId, vgId, pRsp->blockNum, totalRows); + qStreamExtractOffset(task, &pRsp->rspOffset); return 0; } @@ -122,18 +119,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return 0; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - pRsp->rspOffset = *pOffset; - return 0; - } - } + tqDebug("tqScanTaosx prepare scan failed, return"); + return -1; } int32_t rowCnt = 0; @@ -150,7 +137,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { continue; } @@ -179,64 +166,46 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } } - if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (qStreamExtractPrepareUid(task) != 0) { + // get meta + SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); + if (tmp->metaRspLen > 0) { + qStreamExtractOffset(task, &tmp->rspOffset); + *pMetaRsp = *tmp; + + tqDebug("tmqsnap task get meta"); + break; + } + + if (pDataBlock == NULL) { + qStreamExtractOffset(task, pOffset); + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { continue; } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } if (pRsp->blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } - - SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); - if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts); - qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META; - tqDebug("tmqsnap task exec change to get data"); - continue; - } - - *pMetaRsp = *tmp; - tqDebug("tmqsnap task exec exited, get meta"); - - tqDebug("task exec exited"); - break; - } - - qStreamExtractOffset(task, &pRsp->rspOffset); - - if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; } return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows) { STqExecHandle* pExec = &pHandle->execHandle; - /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/ - - SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); - SArray* pSchemas = taosArrayInit(0, sizeof(void*)); + SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); + SArray* pSchemas = taosArrayInit(0, sizeof(void*)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { - STqReader* pReader = pExec->pExecReader; - /*tqReaderSetDataMsg(pReader, pReq, 0);*/ - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); - while (tqNextDataBlock2(pReader)) { - /*SSDataBlock block = {0};*/ - /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ - /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ - /*}*/ - + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock(pReader)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; @@ -244,8 +213,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -266,7 +234,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (TSDB_CODE_SUCCESS != code) { continue; } - void* createReq = taosMemoryCalloc(1, len); + void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; tEncoderInit(&encoder, createReq, len); code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq); @@ -286,6 +254,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision); + totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); taosArrayPush(pRsp->blockSchema, &pSW); @@ -293,14 +262,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR } } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { - STqReader* pReader = pExec->pExecReader; - /*tqReaderSetDataMsg(pReader, pReq, 0);*/ - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { - /*SSDataBlock block = {0};*/ - /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ - /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ - /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; @@ -308,7 +272,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -329,7 +293,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (TSDB_CODE_SUCCESS != code) { continue; } - void* createReq = taosMemoryCalloc(1, len); + void* createReq = taosMemoryCalloc(1, len); SEncoder encoder = {0}; tEncoderInit(&encoder, createReq, len); code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq); @@ -345,15 +309,11 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } - /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ - /*pTq->pVnode->config.tsdbCfg.precision);*/ - /*blockDataFreeRes(&block);*/ - /*tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);*/ - /*pRsp->blockNum++;*/ for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); tqAddBlockDataToRsp(pBlock, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision); + *totalRows += pBlock->info.rows; blockDataFreeRes(pBlock); SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); taosArrayPush(pRsp->blockSchema, &pSW); @@ -363,9 +323,5 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR } taosArrayDestroy(pBlocks); taosArrayDestroy(pSchemas); -// if (pRsp->blockNum == 0) { -// return -1; -// } - return 0; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 93132293b1d085d315553279f2e28f130de1e332..62b81305b7233edd48e7dab796975b1ab69a7ad3 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -87,7 +87,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d return; } - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write into table, block num: %d", TD_VID(pVnode), pTask->id.idStr, blockSz); for (int32_t i = 0; i < blockSz; i++) { bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); @@ -382,7 +382,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -475,11 +475,9 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); - STagVal tagVal = { - .cid = pTSchema->numOfCols + step, - .type = pTagData->info.type, - }; - void* pData = colDataGetData(pTagData, rowId); + + STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type}; + void* pData = colDataGetData(pTagData, rowId); if (colDataIsNull_s(pTagData, rowId)) { continue; } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { @@ -584,6 +582,8 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* tTagNew(tagArray, 1, false, &pTag); tagArray = taosArrayDestroy(tagArray); if (pTag == NULL) { + taosMemoryFree(ctbName); + tdDestroySVCreateTbReq(pCreateTbReq); terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(ctbName); tdDestroySVCreateTbReq(pCreateTbReq); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..c09a8a030e78907d7564333cba773b1883bdacae --- /dev/null +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -0,0 +1,527 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) + +static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); + +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { + char buf[128] = {0}; + sprintf(buf, "0x%" PRIx64 "-%d", streamId, taskId); + return taosStrdup(buf); +} + +// stream_task:stream_id:task_id +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { + int32_t n = 12; + char* p = dst; + + memcpy(p, "stream_task:", n); + p += n; + + int32_t inc = tintToHex(streamId, p); + p += inc; + + *(p++) = ':'; + tintToHex(taskId, p); +} + +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { + int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); + if (code < 0) { + tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver); + return -1; + } + + if (streamSchedExec(pTask) < 0) { + tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno)); + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +void initOffsetForAllRestoreTasks(STQ* pTq) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version); + } + } +} + +void saveOffsetForAllTasks(STQ* pTq, int64_t ver) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, ver); + } + } +} + +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) { + STqOffset offset = {0}; + tqOffsetResetToLog(&offset.val, ver); + + tstrncpy(offset.subKey, pKey, tListLen(offset.subKey)); + + // keep the offset info in the offset store + tqOffsetWrite(pOffsetStore, &offset); +} + +int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + return -1; + } + + pRsp->withTbName = 0; + pRsp->withSchema = false; + return 0; +} + +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->withTbName = 1; + pRsp->withSchema = 1; + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { + if (pRsp->blockData != NULL) { + pRsp->blockData = taosArrayDestroy(pRsp->blockData); + } + + if (pRsp->blockDataLen != NULL) { + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + } + + if (pRsp->blockTbName != NULL) { + pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); + } + + if (pRsp->blockSchema != NULL) { + pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); + } + return -1; + } + + return 0; +} + +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, bool* pBlockReturned) { + uint64_t consumerId = pRequest->consumerId; + STqOffsetVal reqOffset = pRequest->reqOffset; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); + int32_t vgId = TD_VID(pTq->pVnode); + + *pBlockReturned = false; + + // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. + if (pOffset != NULL) { + *pOffsetVal = pOffset->val; + + char formatBuf[80]; + tFormatOffset(formatBuf, 80, pOffsetVal); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64, + consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); + return 0; + } else { + // no poll occurs in this vnode for this topic, let's seek to the right offset value. + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pRequest->useSnapshot) { + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", + consumerId, pHandle->subKey, vgId); + + if (pHandle->fetchMeta) { + tqOffsetResetToMeta(pOffsetVal, 0); + } else { + tqOffsetResetToData(pOffsetVal, 0, 0); + } + } else { + pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); + if (pHandle->pRef == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest); + + tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + tDeleteMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", + pHandle->subKey, consumerId, vgId, pRequest->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return -1; + } + } + + return 0; +} + +static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, STqOffsetVal* pOffset) { + char buf[80] = {0}; + uint64_t consumerId = pRequest->consumerId; + int32_t vgId = TD_VID(pTq->pVnode); + + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest); + + // lock + taosWLockLatch(&pTq->lock); + + qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + if (code == 0) { + + // till now, all data has been transferred to consumer, new data needs to push client once arrived. + if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { + code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + taosWUnLockLatch(&pTq->lock); + return code; + } + + // NOTE: this pHandle->consumerId may have been changed already. + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + } + + tFormatOffset(buf, 80, &dataRsp.rspOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 + " code:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + taosWUnLockLatch(&pTq->lock); + tDeleteMqDataRsp(&dataRsp); + + return code; +} + +static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SWalCkHead* pCkHead = NULL; + SMqMetaRsp metaRsp = {0}; + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + + if (offset->type != TMQ_OFFSET__LOG) { + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { + tDeleteSTaosxRsp(&taosxRsp); + return -1; + } + + if (metaRsp.metaRspLen > 0) { + code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); + taosMemoryFree(metaRsp.metaRsp); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 + ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts); + if (taosxRsp.blockNum > 0) { + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + return code; + }else { + *offset = taosxRsp.rspOffset; + } + } + + if (offset->type == TMQ_OFFSET__LOG) { + int64_t fetchVer = offset->version + 1; + pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); + if (pCkHead == NULL) { + tDeleteSTaosxRsp(&taosxRsp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + walSetReaderCapacity(pHandle->pWalReader, 2048); + int totalRows = 0; + while (1) { + int32_t savedEpoch = atomic_load_32(&pHandle->epoch); + if (savedEpoch > pRequest->epoch) { + tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 + ", found new consumer epoch %d, discard req epoch %d", + pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); + break; + } + + if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + SWalCont* pHead = &pCkHead->head; + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId, + pRequest->epoch, vgId, fetchVer, pHead->msgType); + + // process meta + if (pHead->msgType != TDMT_VND_SUBMIT) { + if(totalRows > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId) < 0) { + code = -1; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + code = 0; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + // process data + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), + .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), + .ver = pHead->version, + }; + + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return -1; + } + + if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } else { + fetchVer++; + } + } + } + + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return 0; +} + +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { + int32_t code = -1; + STqOffsetVal offset = {0}; + STqOffsetVal reqOffset = pRequest->reqOffset; + + // 1. reset the offset if needed + if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { + // handle the reset offset cases, according to the consumer's choice. + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + if (code != 0) { + return code; + } + + // empty block returned, quit + if (blockReturned) { + return 0; + } + } else { // use the consumer specified offset + // the offset value can not be monotonious increase?? + offset = reqOffset; + } + + // this is a normal subscribe requirement + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); + } + + // todo handle the case where re-balance occurs. + // for taosx + return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset); +} + +static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int64_t consumerId, int64_t sver, + int64_t ever) { + pMsgHead->consumerId = consumerId; + pMsgHead->epoch = epoch; + pMsgHead->mqMsgType = type; + pMsgHead->walsver = sver; + pMsgHead->walever = ever; +} + +int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId) { + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeMqMetaRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + initMqRspHead(buf, TMQ_MSG_TYPE__POLL_META_RSP, pReq->epoch, pReq->consumerId, sver, ever); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + tEncodeMqMetaRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg resp = { .info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0 }; + + tmsgSendRsp(&resp); + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", vgId, + pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); + + return 0; +} + +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, + int32_t type, int64_t sver, int64_t ever) { + int32_t len = 0; + int32_t code = 0; + + if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); + } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); + } + + if (code < 0) { + return -1; + } + + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + SMqRspHead* pHead = (SMqRspHead*)buf; + initMqRspHead(pHead, type, epoch, consumerId, sver, ever); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + + if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + tEncodeMqDataRsp(&encoder, pRsp); + } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); + } + + tEncoderClear(&encoder); + SRpcMsg rsp = { .info = *pRpcHandleInfo, .pCont = buf, .contLen = tlen, .code = 0 }; + + tmsgSendRsp(&rsp); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index dcf7286c0070572664d9ad5a947aa66ed4e32a9c..3c7edd931baf1fd2f8796368e4ac76bbb398e636 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -35,7 +35,11 @@ _err: static void tsdbCloseBICache(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->biCache; if (pCache) { + int32_t elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheEraseUnrefEntries(pCache); + elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheCleanup(pCache); @@ -506,6 +510,11 @@ static int32_t getTableDelData(STbData *pMem, STbData *pIMem, SDelFReader *pDelR SArray *aDelData) { int32_t code = 0; + if (pDelIdx) { + code = getTableDelDataFromDelIdx(pDelReader, pDelIdx, aDelData); + if (code) goto _err; + } + if (pMem) { code = getTableDelDataFromTbData(pMem, aDelData); if (code) goto _err; @@ -516,11 +525,6 @@ static int32_t getTableDelData(STbData *pMem, STbData *pIMem, SDelFReader *pDelR if (code) goto _err; } - if (pDelIdx) { - code = getTableDelDataFromDelIdx(pDelReader, pDelIdx, aDelData); - if (code) goto _err; - } - _err: return code; } @@ -590,14 +594,18 @@ typedef struct { SDataFReader **pDataFReader; TSDBROW row; + bool checkRemainingRow; SMergeTree mergeTree; SMergeTree *pMergeTree; SSttBlockLoadInfo *pLoadInfo; + int64_t lastTs; } SFSLastNextRowIter; -static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { +static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter; int32_t code = 0; + bool checkRemainingRow = true; switch (state->state) { case SFSLASTNEXTROW_FS: @@ -630,26 +638,54 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (code) goto _err; } + for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { + state->pLoadInfo[i].colIds = aCols; + state->pLoadInfo[i].numOfCols = nCols; + state->pLoadInfo[i].isLast = isLast; + } tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, - &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL); + &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); state->pMergeTree = &state->mergeTree; + state->state = SFSLASTNEXTROW_BLOCKROW; + } + case SFSLASTNEXTROW_BLOCKROW: { + if (nCols != state->pLoadInfo->numOfCols) { + for (int i = 0; i < state->pLoadInfo->numOfStt; ++i) { + state->pLoadInfo[i].numOfCols = nCols; + + state->pLoadInfo[i].checkRemainingRow = state->checkRemainingRow; + } + } bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { + if (tMergeTreeIgnoreEarlierTs(&state->mergeTree)) { + *pIgnoreEarlierTs = true; + *ppRow = NULL; + return code; + } state->state = SFSLASTNEXTROW_FILESET; goto _next_fileset; } - state->state = SFSLASTNEXTROW_BLOCKROW; - } - case SFSLASTNEXTROW_BLOCKROW: state->row = tMergeTreeGetRow(&state->mergeTree); *ppRow = &state->row; - bool hasVal = tMergeTreeNext(&state->mergeTree); + + if (TSDBROW_TS(&state->row) <= state->lastTs) { + *pIgnoreEarlierTs = true; + *ppRow = NULL; + return code; + } + + *pIgnoreEarlierTs = false; if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; } + if (!state->checkRemainingRow) { + state->checkRemainingRow = true; + } return code; + } default: ASSERT(0); break; @@ -722,11 +758,14 @@ typedef struct SFSNextRowIter { int32_t iRow; TSDBROW row; SSttBlockLoadInfo *pLoadInfo; + int64_t lastTs; } SFSNextRowIter; -static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { +static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SFSNextRowIter *state = (SFSNextRowIter *)iter; int32_t code = 0; + bool checkRemainingRow = true; switch (state->state) { case SFSNEXTROW_FS: @@ -785,7 +824,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { /* + if (!state->pBlockIdx) { + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + /* tsdbDataFReaderClose(state->pDataFReader); *state->pDataFReader = NULL; resetLastBlockLoadInfo(state->pLoadInfo);*/ @@ -812,30 +856,117 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } } case SFSNEXTROW_BLOCKDATA: + _next_datablock: if (state->iBlock >= 0) { SDataBlk block = {0}; + bool skipBlock = true; + int inputColIndex = 0; tDataBlkReset(&block); - // tBlockDataReset(&state->blockData); tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); - */ + if (block.maxKey.ts <= state->lastTs) { + *pIgnoreEarlierTs = true; + if (state->pBlockData) { + tBlockDataDestroy(state->pBlockData); + state->pBlockData = NULL; + } + + *ppRow = NULL; + return code; + } + *pIgnoreEarlierTs = false; tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; - code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); + code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, aCols, nCols); if (code) goto _err; code = tsdbReadDataBlock(*state->pDataFReader, &block, state->pBlockData); if (code) goto _err; + for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { + SColData *pColData = &state->pBlockData->aColData[colIndex]; + + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + } + + if (skipBlock) { + if (--state->iBlock < 0) { + tsdbDataFReaderClose(state->pDataFReader); + *state->pDataFReader = NULL; + // resetLastBlockLoadInfo(state->pLoadInfo); + + if (state->aBlockIdx) { + // taosArrayDestroy(state->aBlockIdx); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + } + + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + goto _next_datablock; + } + } + state->nRow = state->blockData.nRow; state->iRow = state->nRow - 1; state->state = SFSNEXTROW_BLOCKROW; + checkRemainingRow = false; } - case SFSNEXTROW_BLOCKROW: + case SFSNEXTROW_BLOCKROW: { + if (checkRemainingRow) { + bool skipBlock = true; + int inputColIndex = 0; + for (int colIndex = 0; colIndex < state->pBlockData->nColData; ++colIndex) { + SColData *pColData = &state->pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (inputColIndex < nCols && cid == aCols[inputColIndex]) { + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + + ++inputColIndex; + } + } + + if (skipBlock) { + if (--state->iBlock < 0) { + tsdbDataFReaderClose(state->pDataFReader); + *state->pDataFReader = NULL; + // resetLastBlockLoadInfo(state->pLoadInfo); + + if (state->aBlockIdx) { + // taosArrayDestroy(state->aBlockIdx); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + } + + state->state = SFSNEXTROW_FILESET; + goto _next_fileset; + } else { + goto _next_datablock; + } + } + } + if (state->iRow >= 0) { state->row = tsdbRowFromBlockData(state->pBlockData, state->iRow); *ppRow = &state->row; @@ -861,6 +992,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } return code; + } default: ASSERT(0); break; @@ -931,16 +1063,24 @@ typedef struct SMemNextRowIter { SMEMNEXTROWSTATES state; STbData *pMem; // [input] STbDataIter iter; // mem buffer skip list iterator + int64_t lastTs; // bool iterOpened; // TSDBROW *curRow; } SMemNextRowIter; -static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow) { +static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols) { SMemNextRowIter *state = (SMemNextRowIter *)iter; int32_t code = 0; + *pIgnoreEarlierTs = false; switch (state->state) { case SMEMNEXTROW_ENTER: { if (state->pMem != NULL) { + if (state->pMem->maxKey <= state->lastTs) { + *ppRow = NULL; + *pIgnoreEarlierTs = true; + return code; + } tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter); TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter); @@ -1041,13 +1181,15 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { return deleted; } -typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow); +typedef int32_t (*_next_row_fn_t)(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, int16_t *aCols, + int nCols); typedef int32_t (*_next_row_clear_fn_t)(void *iter); typedef struct { TSDBROW *pRow; bool stop; bool next; + bool ignoreEarlierTs; void *iter; _next_row_fn_t nextRowFn; _next_row_clear_fn_t nextRowClearFn; @@ -1070,7 +1212,7 @@ typedef struct { static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, - SDataFReader **pDataFReaderLast) { + SDataFReader **pDataFReaderLast, int64_t lastTs) { int code = 0; STbData *pMem = NULL; @@ -1131,6 +1273,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.uid = uid; pIter->fsLastState.pLoadInfo = pLoadInfo; pIter->fsLastState.pDataFReader = pDataFReaderLast; + pIter->fsLastState.lastTs = lastTs; pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; @@ -1141,17 +1284,19 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsState.uid = uid; pIter->fsState.pLoadInfo = pLoadInfo; pIter->fsState.pDataFReader = pDataFReader; + pIter->fsState.lastTs = lastTs; - pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, &pIter->memState, getNextRowFromMem, NULL}; - pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, &pIter->imemState, getNextRowFromMem, NULL}; - pIter->input[2] = (TsdbNextRowState){&pIter->fsLastRow, false, true, &pIter->fsLastState, getNextRowFromFSLast, - clearNextRowFromFSLast}; + pIter->input[0] = (TsdbNextRowState){&pIter->memRow, true, false, false, &pIter->memState, getNextRowFromMem, NULL}; + pIter->input[1] = (TsdbNextRowState){&pIter->imemRow, true, false, false, &pIter->imemState, getNextRowFromMem, NULL}; + pIter->input[2] = (TsdbNextRowState){ + &pIter->fsLastRow, false, true, false, &pIter->fsLastState, getNextRowFromFSLast, clearNextRowFromFSLast}; pIter->input[3] = - (TsdbNextRowState){&pIter->fsRow, false, true, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; + (TsdbNextRowState){&pIter->fsRow, false, true, false, &pIter->fsState, getNextRowFromFS, clearNextRowFromFS}; if (pMem) { pIter->memState.pMem = pMem; pIter->memState.state = SMEMNEXTROW_ENTER; + pIter->memState.lastTs = lastTs; pIter->input[0].stop = false; pIter->input[0].next = true; } @@ -1159,6 +1304,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs if (pIMem) { pIter->imemState.pMem = pIMem; pIter->imemState.state = SMEMNEXTROW_ENTER; + pIter->imemState.lastTs = lastTs; pIter->input[1].stop = false; pIter->input[1].next = true; } @@ -1186,12 +1332,14 @@ _err: } // iterate next row non deleted backward ts, version (from high to low) -static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { +static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pIgnoreEarlierTs, bool isLast, + int16_t *aCols, int nCols) { int code = 0; for (;;) { for (int i = 0; i < 4; ++i) { if (pIter->input[i].next && !pIter->input[i].stop) { - code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow); + code = pIter->input[i].nextRowFn(pIter->input[i].iter, &pIter->input[i].pRow, &pIter->input[i].ignoreEarlierTs, + isLast, aCols, nCols); if (code) goto _err; if (pIter->input[i].pRow == NULL) { @@ -1203,6 +1351,8 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow) { if (pIter->input[0].stop && pIter->input[1].stop && pIter->input[2].stop && pIter->input[3].stop) { *ppRow = NULL; + *pIgnoreEarlierTs = (pIter->input[0].ignoreEarlierTs || pIter->input[1].ignoreEarlierTs || + pIter->input[2].ignoreEarlierTs || pIter->input[3].ignoreEarlierTs); return code; } @@ -1303,6 +1453,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo int16_t noneCol = 0; bool setNoneCol = false; bool hasRow = false; + bool ignoreEarlierTs = false; SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; @@ -1315,11 +1466,11 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo CacheNextRowIter iter = {0}; nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast); + &pr->pDataFReaderLast, pr->lastTs); do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, false, NULL, 0); if (!pRow) { break; @@ -1327,11 +1478,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -1419,7 +1573,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // taosArrayDestroy(pColArray); //} else { if (!hasRow) { - taosArrayClear(pColArray); + if (ignoreEarlierTs) { + taosArrayDestroy(pColArray); + pColArray = NULL; + } else { + taosArrayClear(pColArray); + } } *ppColArray = pColArray; //} @@ -1441,23 +1600,34 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach int16_t noneCol = 0; bool setNoneCol = false; bool hasRow = false; + bool ignoreEarlierTs = false; SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; + int16_t nCols = nLastCol; int32_t code = initLastColArray(pTSchema, &pColArray); if (TSDB_CODE_SUCCESS != code) { return code; } + SArray *aColArray = taosArrayInit(nCols, sizeof(int16_t)); + if (NULL == aColArray) { + taosArrayDestroy(pColArray); + + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int i = 1; i < pTSchema->numOfCols; ++i) { + taosArrayPush(aColArray, &pTSchema->columns[i].colId); + } TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, - &pr->pDataFReaderLast); + &pr->pDataFReaderLast, pr->lastTs); do { TSDBROW *pRow = NULL; - nextRowIterGet(&iter, &pRow); + nextRowIterGet(&iter, &pRow, &ignoreEarlierTs, true, TARRAY_DATA(aColArray), TARRAY_SIZE(aColArray)); if (!pRow) { break; @@ -1465,11 +1635,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -1502,9 +1675,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } - if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { - noneCol = iCol; - setNoneCol = true; + if (!COL_VAL_IS_VALUE(pColVal)) { + if (!setNoneCol) { + noneCol = iCol; + setNoneCol = true; + } + } else { + int32_t aColIndex = taosArraySearchIdx(aColArray, &pColVal->cid, compareInt16Val, TD_EQ); + taosArrayRemove(aColArray, aColIndex); } } if (!setNoneCol) { @@ -1545,6 +1723,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } taosArraySet(pColArray, iCol, &lastCol); + int32_t aColIndex = taosArraySearchIdx(aColArray, &lastCol.colVal.cid, compareInt16Val, TD_EQ); + taosArrayRemove(aColArray, aColIndex); } else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { noneCol = iCol; setNoneCol = true; @@ -1557,12 +1737,18 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach // taosArrayDestroy(pColArray); //} else { if (!hasRow) { - taosArrayClear(pColArray); + if (ignoreEarlierTs) { + taosArrayDestroy(pColArray); + pColArray = NULL; + } else { + taosArrayClear(pColArray); + } } *ppLastArray = pColArray; //} nextRowIterClose(&iter); + taosArrayDestroy(aColArray); // taosMemoryFreeClear(pTSchema); return code; @@ -1571,6 +1757,7 @@ _err: // taosMemoryFreeClear(pTSchema); *ppLastArray = NULL; taosArrayDestroy(pColArray); + taosArrayDestroy(aColArray); return code; } @@ -1591,8 +1778,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, set handle NULL and return - if (code < 0 /* || pArray == NULL*/) { + // if table's empty or error or ignore ignore earlier ts, set handle NULL and return + if (code < 0 || pArray == NULL) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1635,8 +1822,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, set handle NULL and return - if (code < 0 /* || pLastArray == NULL*/) { + // if table's empty or error or ignore ignore earlier ts, set handle NULL and return + if (code < 0 || pLastArray == NULL) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1682,6 +1869,15 @@ size_t tsdbCacheGetUsage(SVnode *pVnode) { return usage; } +int32_t tsdbCacheGetElems(SVnode *pVnode) { + int32_t elems = 0; + if (pVnode->pTsdb != NULL) { + elems = taosLRUCacheGetElems(pVnode->pTsdb->lruCache); + } + + return elems; +} + static void getBICacheKey(int32_t fid, int64_t commitID, char *key, int *len) { struct { int32_t fid; @@ -1750,6 +1946,7 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa taosThreadMutexUnlock(&pTsdb->biMutex); } + tsdbTrace("bi cache:%p, ref", pCache); *handle = h; return code; @@ -1759,6 +1956,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; taosLRUCacheRelease(pCache, h, false); + tsdbTrace("bi cache:%p, release", pCache); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 5528b6313cfac083d35ed13849bd823b8f85cae3..d6e819d84ad3d8a6131dcf15f09afdc8ff4b7f64 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id if (suid != 0) { p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1); if (p->pSchema == NULL) { - taosMemoryFree(p); tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } @@ -135,7 +134,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id // all queried tables have been dropped already, return immediately. if (p->pSchema == NULL) { - taosMemoryFree(p); tsdbWarn("all queried tables has been dropped, try next group, %s", idstr); return TSDB_CODE_PAR_TABLE_NOT_EXIST; } @@ -199,6 +197,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->idstr = taosStrdup(idstr); taosThreadMutexInit(&p->readerMutex, NULL); + p->lastTs = INT64_MIN; + *pReader = p; return TSDB_CODE_SUCCESS; } @@ -330,6 +330,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 // retrieve the only one last row of all tables in the uid list. if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) { + int64_t st = taosGetTimestampUs(); + int64_t totalLastTs = INT64_MAX; for (int32_t i = 0; i < pr->numOfTables; ++i) { STableKeyInfo* pKeyInfo = &pr->pTableList[i]; @@ -347,6 +349,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } { + bool hasNotNullRow = true; + int64_t singleTableLastTs = INT64_MAX; for (int32_t k = 0; k < pr->numOfCols; ++k) { int32_t slotId = slotIds[k]; @@ -357,15 +361,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 hasRes = true; p->ts = pCol->ts; p->colVal = pCol->colVal; - - // only set value for last row query - if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) { - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - } + singleTableLastTs = pCol->ts; } } else { SLastCol* p = taosArrayGet(pLastCols, slotId); @@ -373,11 +369,17 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (pColVal->ts > p->ts) { if (!COL_VAL_IS_VALUE(&pColVal->colVal) && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { + if (!COL_VAL_IS_VALUE(&p->colVal)) { + hasNotNullRow = false; + } continue; } hasRes = true; p->ts = pColVal->ts; + if (pColVal->ts < singleTableLastTs && HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST)) { + singleTableLastTs = pColVal->ts; + } if (!IS_VAR_DATA_TYPE(pColVal->colVal.type)) { p->colVal = pColVal->colVal; @@ -394,6 +396,22 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } } + + if (hasNotNullRow) { + if (INT64_MAX == totalLastTs || (INT64_MAX != singleTableLastTs && totalLastTs < singleTableLastTs)) { + totalLastTs = singleTableLastTs; + } + double cost = (taosGetTimestampUs() - st) / 1000.0; + if (cost > tsCacheLazyLoadThreshold) { + pr->lastTs = totalLastTs; + } + } + } + + if (taosArrayGetSize(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); } tsdbCacheRelease(lruCache, h); @@ -402,7 +420,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (hasRes) { saveOneRow(pLastCols, pResBlock, pr, slotIds, pRes, pr->idstr); } - } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) { for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) { STableKeyInfo* pKeyInfo = &pr->pTableList[i]; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d34af9acae68b964879cd7ba24491f0ff878e439..d0ff403bf70a96f114b0722bfb13f7ce035edfb4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -282,6 +282,40 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } +int64_t tsdbCountTbDataRows(STbData *pTbData) { + SMemSkipListNode *pNode = pTbData->sl.pHead; + int64_t rowsNum = 0; + + while (NULL != pNode) { + pNode = SL_GET_NODE_FORWARD(pNode, 0); + if (pNode == pTbData->sl.pTail) { + return rowsNum; + } + + rowsNum++; + } + + return rowsNum; +} + +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { + taosRLockLatch(&pMemTable->latch); + for (int32_t i = 0; i < pMemTable->nBucket; ++i) { + STbData *pTbData = pMemTable->aBucket[i]; + while (pTbData) { + void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + if (p == NULL) { + pTbData = pTbData->next; + continue; + } + + *rowsNum += tsdbCountTbDataRows(pTbData); + pTbData = pTbData->next; + } + } + taosRUnLockLatch(&pMemTable->latch); +} + static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; @@ -787,4 +821,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index d9d60442ffbfd36dfa9b132b4d5f9c714558de7a..fa8870835c5fc6b914663a2d76c9202e1410e52f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -29,9 +29,11 @@ struct SLDataIter { STimeWindow timeWindow; SVersionRange verRange; SSttBlockLoadInfo *pBlockLoadInfo; + bool ignoreEarlierTs; }; -SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfSttTrigger) { +SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, + int32_t numOfSttTrigger) { SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -162,7 +164,8 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; - tsdbDebug("last block index list:%d, %d, rowIndex:%d %s", pInfo->blockIndex[0], pInfo->blockIndex[1], pIter->iRow, idStr); + tsdbDebug("last block index list:%d, %d, rowIndex:%d %s", pInfo->blockIndex[0], pInfo->blockIndex[1], pIter->iRow, + idStr); return &pInfo->blockData[pInfo->currentLoadBlockIndex]; _exit: @@ -263,7 +266,7 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, - const char *idStr) { + const char *idStr, bool strictTimeRange) { int32_t code = TSDB_CODE_SUCCESS; *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); @@ -340,6 +343,17 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t if ((*pIter)->iSttBlk != -1) { (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; + + if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) || + (!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) { + (*pIter)->pSttBlk = NULL; + } + + if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) || + (!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) { + (*pIter)->pSttBlk = NULL; + (*pIter)->ignoreEarlierTs = true; + } } return code; @@ -421,7 +435,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { pBlockData->aUid != NULL) { i = binarySearchForStartRowIndex((uint64_t *)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); if (i == -1) { - tsdbDebug("failed to find the data in pBlockData, uid:%"PRIu64" , %s", pIter->uid, idStr); + tsdbDebug("failed to find the data in pBlockData, uid:%" PRIu64 " , %s", pIter->uid, idStr); pIter->iRow = -1; return; } @@ -490,9 +504,34 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->iRow += step; while (1) { + bool skipBlock = false; + findNextValidRow(pIter, idStr); - if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { + if (pIter->pBlockLoadInfo->checkRemainingRow) { + skipBlock = true; + int16_t *aCols = pIter->pBlockLoadInfo->colIds; + int nCols = pIter->pBlockLoadInfo->numOfCols; + bool isLast = pIter->pBlockLoadInfo->isLast; + for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) { + for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) { + SColData *pColData = &pBlockData->aColData[colIndex]; + int16_t cid = pColData->cid; + + if (cid == aCols[inputColIndex]) { + if (isLast && (pColData->flag & HAS_VALUE)) { + skipBlock = false; + break; + } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) { + skipBlock = false; + break; + } + } + } + } + } + + if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { tLDataIterNextBlock(pIter, idStr); if (pIter->pSttBlk == NULL) { // no more data goto _exit; @@ -508,7 +547,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { } // set start row index - pIter->iRow = pIter->backward? pBlockData->nRow-1:0; + pIter->iRow = pIter->backward ? pBlockData->nRow - 1 : 0; } } @@ -551,7 +590,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr) { + bool destroyLoadInfo, const char *idStr, bool strictTimeRange) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -569,11 +608,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; + pMTree->ignoreEarlierTs = false; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter *pIter = NULL; code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, - &pMTree->pLoadInfo[i], pMTree->idStr); + &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -583,6 +623,9 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead taosArrayPush(pMTree->pIterList, &pIter); tMergeTreeAddIter(pMTree, pIter); } else { + if (!pMTree->ignoreEarlierTs) { + pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; + } tLDataIterClose(pIter); } } @@ -596,6 +639,8 @@ _end: void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } +bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; } + bool tMergeTreeNext(SMergeTree *pMTree) { int32_t code = TSDB_CODE_SUCCESS; if (pMTree->pIter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 31153f0832bb83f3b6c68acd0957b4b5171f61da..362dc51ad5a9efcf9752669729662d1973b613a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -15,6 +15,7 @@ #include "osDef.h" #include "tsdb.h" +#include "tsimplehash.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -24,6 +25,11 @@ typedef enum { EXTERNAL_ROWS_NEXT = 0x3, } EContentData; +typedef enum { + READ_MODE_COUNT_ONLY = 0x1, + READ_MODE_ALL, +} EReadMode; + typedef struct { STbDataIter* iter; int32_t index; @@ -167,6 +173,8 @@ struct STsdbReader { uint64_t suid; int16_t order; bool freeBlock; + EReadMode readMode; + uint64_t rowsNum; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -176,14 +184,15 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + // STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); @@ -390,8 +399,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { + taosHashCleanup(pTableMap); return NULL; } + pUidList->currentIndex = 0; for (int32_t j = 0; j < numOfTables; ++j) { @@ -425,7 +436,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf return pTableMap; } -static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { +static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts, int32_t step) { STableBlockScanInfo** p = NULL; while ((p = taosHashIterate(pTableMap, p)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; @@ -444,6 +455,7 @@ static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); pInfo->lastKey = ts; + pInfo->lastKeyInStt = ts + step; } } @@ -819,6 +831,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, // this block belongs to a table that is not queried. STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); if (pScanInfo == NULL) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); return terrno; } @@ -919,10 +932,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN pBlockNum->numOfBlocks += 1; } - if (pScanInfo->pBlockList != NULL) { - if (taosArrayGetSize(pScanInfo->pBlockList) > 0) { - numOfQTable += 1; - } + if (taosArrayGetSize(pScanInfo->pBlockList) > 0) { + numOfQTable += 1; } } @@ -948,14 +959,18 @@ static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int pDumpInfo->lastKey = maxKey + step; } -static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, - SBlockLoadSuppInfo* pSup) { +static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal, + SBlockLoadSuppInfo* pSup) { if (IS_VAR_DATA_TYPE(pColVal->type)) { if (!COL_VAL_IS_VALUE(pColVal)) { colDataSetNULL(pColInfoData, rowIndex); } else { varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData); - ASSERT(pColVal->value.nData <= pColInfoData->info.bytes); + if (pColVal->value.nData > pColInfoData->info.bytes) { + tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData, + pColInfoData->info.bytes); + return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + } if (pColVal->value.nData > 0) { // pData may be null, if nData is 0 memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData); } @@ -965,6 +980,8 @@ static void doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_ } else { colDataSetVal(pColInfoData, rowIndex, (const char*)&pColVal->value, !COL_VAL_IS_VALUE(pColVal)); } + + return TSDB_CODE_SUCCESS; } static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { @@ -1166,6 +1183,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; + int32_t code = TSDB_CODE_SUCCESS; SColVal cv = {0}; int64_t st = taosGetTimestampUs(); @@ -1243,7 +1261,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { } else { // varchar/nchar type for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) { tColDataGetValue(pData, j, &cv); - doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (code) { + return code; + } } } } @@ -1765,7 +1786,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* setComposedBlockFlag(pReader, true); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64 + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); @@ -1775,23 +1796,29 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* } static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, int64_t key, - SFileBlockDumpInfo* pDumpInfo) { + SFileBlockDumpInfo* pDumpInfo, bool* copied) { // opt version // 1. it is not a border point // 2. the direct next point is not an duplicated timestamp + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; bool asc = (pReader->order == TSDB_ORDER_ASC); if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1; int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; if (nextKey != key) { // merge is not needed - doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + if (code) { + return code; + } pDumpInfo->rowIndex += step; - return true; + *copied = true; } } - return false; + return code; } static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, @@ -1818,20 +1845,35 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, - STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) { + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, + bool* copied) { + int32_t code = TSDB_CODE_SUCCESS; + + *copied = false; + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->verRange); if (hasVal) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 != ts) { - doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); - return true; + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; } } else { - doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); - return true; + code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + if (code) { + return code; + } + + *copied = true; + return code; } - return false; + return code; } static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { @@ -1839,8 +1881,8 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ return pReader->pSchema; } - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); - if (pReader->pSchema == NULL) { + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) { tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); } @@ -1848,37 +1890,38 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ } static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + int32_t code = 0; + // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); - } - - if (pReader->pSchema && sversion == pReader->pSchema->version) { - return pReader->pSchema; - } - - if (pReader->pMemSchema == NULL) { - int32_t code = - metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; - } else { - return pReader->pMemSchema; } } - if (pReader->pMemSchema->version == sversion) { - return pReader->pMemSchema; + if (pReader->pSchema && sversion == pReader->pSchema->version) { + return pReader->pSchema; + } + + void** p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion)); + if (p != NULL) { + return *(STSchema**)p; } - taosMemoryFreeClear(pReader->pMemSchema); - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - if (code != TSDB_CODE_SUCCESS || pReader->pMemSchema == NULL) { + STSchema* ptr = NULL; + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } else { - return pReader->pMemSchema; + code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ptr, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + return ptr; } } @@ -1932,8 +1975,8 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { - init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + init = true; // todo check if pReader->pSchema is null or not + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1943,10 +1986,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1963,7 +2006,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1977,7 +2020,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + if (pSchema == NULL) { + return terrno; + } + + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1991,10 +2038,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2004,10 +2051,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2021,11 +2068,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + + return code; } static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, @@ -2033,25 +2081,31 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - - SRow* pTSRow = NULL; - SRowMerger merge = {0}; - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + if (code) { + return code; + } + + if (copied) { pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2059,13 +2113,17 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } } } else { // not merge block data - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2082,10 +2140,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); + + if (code != TSDB_CODE_SUCCESS) { + return code; + } } return TSDB_CODE_SUCCESS; @@ -2113,7 +2175,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2121,7 +2183,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); @@ -2130,7 +2192,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); @@ -2164,8 +2226,17 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return code; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return code; + } int64_t minKey = 0; if (ASCENDING_TRAVERSE(pReader->order)) { @@ -2212,7 +2283,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2223,10 +2294,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2237,15 +2308,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return code; - } - - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2264,10 +2330,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tsdbRowMerge(&merge, pRow); + tsdbRowMergerAdd(&merge, pRow, pSchema); } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2281,8 +2347,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2296,11 +2361,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, piRow, pSchema); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2315,10 +2380,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2329,7 +2394,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2337,7 +2402,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2352,7 +2417,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); @@ -2471,7 +2536,6 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan initMemDataIterator(pScanInfo, pReader); pLBlockReader->uid = pScanInfo->uid; - int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1; STimeWindow w = pLBlockReader->window; if (ASCENDING_TRAVERSE(pLBlockReader->order)) { w.skey = pScanInfo->lastKeyInStt; @@ -2483,7 +2547,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan pScanInfo->uid, pReader->idStr); int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, - pLBlockReader->pInfo, false, pReader->idStr); + pLBlockReader->pInfo, false, pReader->idStr, false); if (code != TSDB_CODE_SUCCESS) { return false; } @@ -2508,7 +2572,13 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { + bool copied = false; + int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); + if (code) { + return code; + } + + if (copied) { pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; } else { @@ -2517,7 +2587,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2528,11 +2598,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + return code; } } @@ -2649,7 +2719,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && pBlock->nRow <= pReader->capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { - copyBlockDataToSDataBlock(pReader); + code = copyBlockDataToSDataBlock(pReader); + if (code) { + goto _end; + } // record the last key value pBlockScanInfo->lastKey = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; @@ -2696,7 +2769,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); + if (code) { + goto _end; + } // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { @@ -2715,8 +2791,8 @@ _end: updateComposedBlockInfo(pReader, el, pBlockScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -2922,6 +2998,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; STableUidList* pUidList = &pStatus->uidList; + int32_t code = TSDB_CODE_SUCCESS; if (taosHashGetSize(pStatus->pTableMap) == 0) { return TSDB_CODE_SUCCESS; @@ -2952,7 +3029,11 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -2962,8 +3043,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3032,7 +3113,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { break; } - buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); + if (code) { + return code; + } + if (pResBlock->info.rows >= pReader->capacity) { break; } @@ -3042,8 +3127,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { updateComposedBlockInfo(pReader, el, pScanInfo); if (pResBlock->info.rows > 0) { - tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64 + ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -3069,6 +3154,148 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } +static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { + int64_t st = taosGetTimestampUs(); + LRUHandle* handle = NULL; + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } + + SBlockIdx* pBlockIdx = NULL; + int32_t i = 0; + for (int32_t i = 0; i < num; ++i) { + pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); + if (pBlockIdx->suid != pReader->suid) { + continue; + } + + STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + if (p == NULL) { + continue; + } + + STableBlockScanInfo* pScanInfo = *p; + tMapDataReset(&pScanInfo->mapData); + tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); + + SDataBlk block = {0}; + for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { + tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); + pReader->rowsNum += block.nRow; + } + } + +_end: + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + +static int32_t doSumSttBlockRows(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; + + for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file + pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + if (size >= 1) { + SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != pReader->suid) { + // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + continue; + } + for (int32_t i = 0; i < size; ++i) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + pReader->rowsNum += p->nRow; + } + } else { + for (int32_t i = 0; i < size; ++i) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + uint64_t s = p->suid; + if (s < pReader->suid) { + continue; + } + + if (s == pReader->suid) { + pReader->rowsNum += p->nRow; + } else if (s > pReader->suid) { + break; + } + } + } + } + } + + return code; +} + +static int32_t readRowsCountFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + code = doSumFileBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doSumSttBlockRows(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + +static int32_t readRowsCountFromMem(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; + if (pReader->pReadSnap->pMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); + } + + if (pReader->pReadSnap->pIMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum); + } + + pReader->rowsNum += memNum + imemNum; + + return code; +} + static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -3300,8 +3527,8 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; int64_t endVer = 0; - if (pCond->endVersion == - -1) { // user not specified end version, set current maximum version of vnode as the endVersion + if (pCond->endVersion == -1) { + // user not specified end version, set current maximum version of vnode as the endVersion endVer = pVnode->state.applied; } else { endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; @@ -3481,7 +3708,7 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe tsdbRowMergerAdd(pMerger, pRow, pTSchema); } else { // column format - tsdbRowMerge(pMerger, pRow); + tsdbRowMergerAdd(pMerger, pRow, NULL); } } @@ -3497,7 +3724,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMerge(pMerger, &fRow); + tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3575,7 +3802,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(pMerger, &fRow1); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, @@ -3627,11 +3854,8 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - - code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema; + code = tsdbRowMergerInit(&merge, ps, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3643,12 +3867,12 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMerge(&merge, pNextRow); + tsdbRowMergerAdd(&merge, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); @@ -3672,13 +3896,20 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p SRow** pTSRow) { SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } - if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } - int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); + if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem + int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3689,7 +3920,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, pRow); + tsdbRowMergerAdd(&merge, pRow, pSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3697,9 +3928,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3710,7 +3939,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3784,11 +4013,13 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; - - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + if (pSchema == NULL) { + return terrno; + } SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -3806,7 +4037,10 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); tRowGet(pTSRow, pSchema, j, &colVal); - doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); + if (code) { + return code; + } i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { @@ -3836,6 +4070,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; + int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3858,7 +4093,10 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); - doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); + if (code) { + return code; + } j += 1; } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data @@ -3882,6 +4120,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { SSDataBlock* pBlock = pReader->pResBlock; + int32_t code = TSDB_CODE_SUCCESS; do { // SRow* pTSRow = NULL; @@ -3893,13 +4132,20 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } if (row.type == TSDBROW_ROW_FMT) { - doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); if (freeTSRow) { taosMemoryFree(row.pTSRow); } + + if (code) { + return code; + } } else { - doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); + if (code) { + break; + } } // no data in buffer, return immediately @@ -3912,7 +4158,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e } } while (1); - return TSDB_CODE_SUCCESS; + return code; } // TODO refactor: with createDataBlockScanInfo @@ -3929,8 +4175,13 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n if (code) { return code; } - pReader->status.uidList.tableUidList = - (uint64_t*)taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + + char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num); + if (p1 == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pReader->status.uidList.tableUidList = (uint64_t*)p1; } taosHashClear(pReader->status.pTableMap); @@ -3986,6 +4237,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; + } else if (READ_MODE_COUNT_ONLY == pReader->readMode) { + // DO NOTHING } else { code = initForFirstBlockInFile(pReader, pBlockIter); } @@ -3997,14 +4250,15 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { return code; } +static void freeSchemaFunc(void* param) { + void* p = *(void**)param; + taosMemoryFree(p); +} + // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) { STimeWindow window = pCond->twindows; - if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { - pCond->twindows.skey += 1; - pCond->twindows.ekey -= 1; - } int32_t capacity = pVnode->config.tsdbCfg.maxRows; if (pResBlock != NULL) { @@ -4027,11 +4281,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL // update the SQueryTableDataCond to create inner reader int32_t order = pCond->order; if (order == TSDB_ORDER_ASC) { - pCond->twindows.ekey = window.skey; + pCond->twindows.ekey = window.skey - 1; pCond->twindows.skey = INT64_MIN; pCond->order = TSDB_ORDER_DESC; } else { - pCond->twindows.skey = window.ekey; + pCond->twindows.skey = window.ekey + 1; pCond->twindows.ekey = INT64_MAX; pCond->order = TSDB_ORDER_ASC; } @@ -4043,11 +4297,11 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } if (order == TSDB_ORDER_ASC) { - pCond->twindows.skey = window.ekey; + pCond->twindows.skey = window.ekey + 1; pCond->twindows.ekey = INT64_MAX; } else { pCond->twindows.skey = INT64_MIN; - pCond->twindows.ekey = window.ekey; + pCond->twindows.ekey = window.ekey - 1; } pCond->order = order; @@ -4073,6 +4327,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } } + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); + if (pReader->pSchemaMap == NULL) { + tsdbError("failed init schema hash for reader %s", pReader->idStr); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc); if (pReader->pSchema != NULL) { code = updateBlockSMAInfo(pReader->pSchema, &pReader->suppInfo); if (code != TSDB_CODE_SUCCESS) { @@ -4091,6 +4353,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->suspended = true; + if (countOnly) { + pReader->readMode = READ_MODE_COUNT_ONLY; + } + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4115,7 +4381,7 @@ void tsdbReaderClose(STsdbReader* pReader) { p->status.uidList.tableUidList = NULL; p->pReadSnap = NULL; p->pSchema = NULL; - p->pMemSchema = NULL; + p->pSchemaMap = NULL; p = pReader->innerReader[1]; @@ -4123,7 +4389,7 @@ void tsdbReaderClose(STsdbReader* pReader) { p->status.uidList.tableUidList = NULL; p->pReadSnap = NULL; p->pSchema = NULL; - p->pMemSchema = NULL; + p->pSchemaMap = NULL; tsdbReaderClose(pReader->innerReader[0]); tsdbReaderClose(pReader->innerReader[1]); @@ -4203,10 +4469,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); - if (pReader->pMemSchema != pReader->pSchema) { - taosMemoryFree(pReader->pMemSchema); - } - + tSimpleHashCleanup(pReader->pSchemaMap); taosMemoryFreeClear(pReader); } @@ -4220,12 +4483,8 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { if (pStatus->loadFromFile) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); if (pBlockInfo != NULL) { - pBlockScanInfo = - *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { - code = TSDB_CODE_INVALID_PARA; - tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, - taosHashGetSize(pReader->status.pTableMap), pReader->idStr); goto _err; } } else { @@ -4370,14 +4629,14 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { pPrevReader->status.pTableMap = pReader->status.pTableMap; pPrevReader->status.uidList = pReader->status.uidList; pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pSchemaMap = pReader->pSchemaMap; pPrevReader->pReadSnap = pReader->pReadSnap; pNextReader->capacity = 1; pNextReader->status.pTableMap = pReader->status.pTableMap; pNextReader->status.uidList = pReader->status.uidList; pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pSchemaMap = pReader->pSchemaMap; pNextReader->pReadSnap = pReader->pReadSnap; code = doOpenReaderImpl(pPrevReader); @@ -4398,43 +4657,82 @@ _err: return code; } -static bool doTsdbNextDataBlock(STsdbReader* pReader) { +static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->pResBlock; + + if (pReader->status.loadFromFile == false) { + return false; + } + + code = readRowsCountFromFiles(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromMem(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + pBlock->info.rows = pReader->rowsNum; + pBlock->info.id.uid = 0; + pBlock->info.dataLoad = 0; + + pReader->rowsNum = 0; + + return pBlock->info.rows > 0; +} + +static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; blockDataCleanup(pBlock); + *hasNext = false; + SReaderStatus* pStatus = &pReader->status; if (taosHashGetSize(pStatus->pTableMap) == 0) { - return false; + return code; + } + + if (READ_MODE_COUNT_ONLY == pReader->readMode) { + return tsdbReadRowsCountOnly(pReader); } if (pStatus->loadFromFile) { - int32_t code = buildBlockFromFiles(pReader); + code = buildBlockFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { - return false; + return code; } - if (pBlock->info.rows > 0) { - return true; - } else { + if (pBlock->info.rows <= 0) { resetTableListIndex(&pReader->status); - buildBlockFromBufferSequentially(pReader); - return pBlock->info.rows > 0; + code = buildBlockFromBufferSequentially(pReader); } } else { // no data in files, let's try the buffer - buildBlockFromBufferSequentially(pReader); - return pBlock->info.rows > 0; + code = buildBlockFromBufferSequentially(pReader); } + + *hasNext = pBlock->info.rows > 0; + + return code; } -bool tsdbNextDataBlock(STsdbReader* pReader) { +int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { + int32_t code = TSDB_CODE_SUCCESS; + + *hasNext = false; + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) { - return false; + return code; } SReaderStatus* pStatus = &pReader->status; - int32_t code = tsdbAcquireReader(pReader); + code = tsdbAcquireReader(pReader); qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); if (pReader->suspended) { @@ -4442,23 +4740,29 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { } if (pReader->innerReader[0] != NULL && pReader->step == 0) { - bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); + code = doTsdbNextDataBlock(pReader->innerReader[0], hasNext); + if (code) { + tsdbReleaseReader(pReader); + return code; + } + pReader->step = EXTERNAL_ROWS_PREV; - if (ret) { + if (*hasNext) { pStatus = &pReader->innerReader[0]->status; if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } } if (pReader->step == EXTERNAL_ROWS_PREV) { // prepare for the main scan - int32_t code = doOpenReaderImpl(pReader); - resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey); + code = doOpenReaderImpl(pReader); + int32_t step = 1; + resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->window.ekey, step); if (code != TSDB_CODE_SUCCESS) { return code; @@ -4467,41 +4771,52 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { pReader->step = EXTERNAL_ROWS_MAIN; } - bool ret = doTsdbNextDataBlock(pReader); - if (ret) { + code = doTsdbNextDataBlock(pReader, hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + + if (*hasNext) { if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { // prepare for the next row scan - int32_t code = doOpenReaderImpl(pReader->innerReader[1]); - resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey); + int32_t step = -1; + code = doOpenReaderImpl(pReader->innerReader[1]); + resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->window.ekey, step); if (code != TSDB_CODE_SUCCESS) { return code; } - ret = doTsdbNextDataBlock(pReader->innerReader[1]); + code = doTsdbNextDataBlock(pReader->innerReader[1], hasNext); + if (code != TSDB_CODE_SUCCESS) { + tsdbReleaseReader(pReader); + return code; + } + pReader->step = EXTERNAL_ROWS_NEXT; - if (ret) { + if (*hasNext) { pStatus = &pReader->innerReader[1]->status; if (pStatus->composedDataBlock) { qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); } - return ret; + return code; } } qTrace("tsdb/read: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); - return false; + return code; } static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { @@ -4509,6 +4824,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); taosArrayInsert(pSup->pColAgg, 0, pTsAgg); + size++; while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); @@ -4521,10 +4837,21 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); + i += 1; + size++; } j += 1; } } + + while (j < numOfCols) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + taosArrayInsert(pSup->pColAgg, i, &nullColAgg); + i += 1; + } + j++; + } } int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { @@ -4550,6 +4877,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, return TSDB_CODE_SUCCESS; } + int64_t st = taosGetTimestampUs(); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); @@ -4602,8 +4931,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { - // ASSERT(pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID); - pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; + pResBlock->pBlockAgg[pSup->slotId[j]] = NULL; + *allHave = false; j += 1; } } @@ -4611,6 +4940,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, *pBlockSMA = pResBlock->pBlockAgg; pReader->cost.smaDataLoad += 1; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; + pReader->cost.smaLoadTime += elapsedTime; + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } @@ -4629,20 +4961,27 @@ STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, co static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; + int32_t code = TSDB_CODE_SUCCESS; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { return NULL; } - int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + tBlockDataDestroy(&pStatus->fileBlockData); + terrno = code; + return NULL; + } + + code = copyBlockDataToSDataBlock(pReader); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } - copyBlockDataToSDataBlock(pReader); return pReader->pResBlock; } @@ -4678,15 +5017,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { - tsdbDebug("tsdb reader reset return %p", pReader->pReadSnap); - + tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr); tsdbReleaseReader(pReader); - return TSDB_CODE_SUCCESS; } - SReaderStatus* pStatus = &pReader->status; - + SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; pReader->order = pCond->order; @@ -4707,8 +5043,10 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { resetDataBlockIterator(pBlockIter, pReader->order); resetTableListIndex(&pReader->status); - int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1; - resetAllDataBlockScanInfo(pStatus->pTableMap, ts); + bool asc = ASCENDING_TRAVERSE(pReader->order); + int32_t step = asc ? 1 : -1; + int64_t ts = asc ? pReader->window.skey - 1 : pReader->window.ekey + 1; + resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step); int32_t code = 0; @@ -4723,7 +5061,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); tsdbReleaseReader(pReader); - return code; } } @@ -4767,7 +5104,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->defMinRows = pc->minRows; pTableBlockInfo->defMaxRows = pc->maxRows; - int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBuckets); + int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets); pTableBlockInfo->numOfFiles += 1; @@ -4866,8 +5203,6 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { - int32_t sversion = 1; - SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); int32_t code = metaGetTableEntryByUidCache(&mr, uid); @@ -4879,6 +5214,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 *suid = 0; + // only child table and ordinary table is allowed, super table is not allowed. if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; @@ -4888,9 +5224,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); return terrno; } - sversion = mr.me.stbEntry.schemaRow.version; - } else if (mr.me.type == TSDB_NORMAL_TABLE) { - sversion = mr.me.ntbEntry.schemaRow.version; + } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing } else { terrno = TSDB_CODE_INVALID_PARA; metaReaderClear(&mr); @@ -4898,9 +5232,10 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 } metaReaderClear(&mr); - *pSchema = metaGetTbTSchema(pVnode->pMeta, uid, sversion, 1); - return TSDB_CODE_SUCCESS; + // get the newest table schema version + code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + return code; } int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { @@ -4992,3 +5327,9 @@ void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proacti } tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); } + +// if failed, do nothing +void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) { + taosMemoryFreeClear(pReader->idStr); + pReader->idStr = taosStrdup(idstr); +} diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 1a98134d70c486259a659cb9771152333835b8c6..aed863d194a98a7ab00f9a599da4cce27b88ef67 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -198,7 +198,7 @@ static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { ASSERT(pReader->bData.nRow); int32_t aBufN[5] = {0}; - code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN); + code = tCmprBlockData(&pReader->bData, NO_COMPRESSION, NULL, NULL, pReader->aBuf, aBufN); if (code) goto _exit; int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; @@ -276,7 +276,7 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* pReader, uint8_t** pp code = tsdbSnapReadNextRow(pReader, &pRowInfo); TSDB_CHECK_CODE(code, lino, _exit); - if (pReader->bData.nRow >= 4096) break; + if (pReader->bData.nRow >= 81920) break; } while (pRowInfo); ASSERT(pReader->bData.nRow > 0); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c323ae1532a3f8fe97fdcddaae020723bda88c94..8e778da877af5affc40e81e06c3ba7048f7ae439 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -638,13 +638,17 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { // SRowMerger ====================================================== -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 0; + if (NULL == pResTSchema) { + pResTSchema = pTSchema; + } + pMerger->pTSchema = pResTSchema; pMerger->version = key.version; @@ -712,6 +716,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) STColumn *pTColumn; int32_t iCol, jCol = 1; + if (NULL == pTSchema) { + pTSchema = pMerger->pTSchema; + } ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { @@ -727,23 +734,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if (key.version > pMerger->version) { -#if 0 - if (!COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; - - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); - } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } -#endif if (!COL_VAL_IS_NONE(pColVal)) { if (IS_VAR_DATA_TYPE(pColVal->type)) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -758,7 +748,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pTColVal->flag = 0; } else { tFree(pTColVal->value.pData); - pTColVal->value.pData = NULL; taosArraySet(pMerger->pArray, iCol, pColVal); } } else { @@ -789,7 +778,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } - +/* int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -840,7 +829,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema _exit: return code; } - +*/ void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -851,7 +840,7 @@ void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } - +/* int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -916,7 +905,7 @@ int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { _exit: return code; } - +*/ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } @@ -1270,6 +1259,11 @@ void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + pBlockData->nColData = 0; + taosMemoryFreeClear(pBlockData->aColData); } void tBlockDataClear(SBlockData *pBlockData) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c326c8bfacbd897fc15527edaef5cd8e77c40f5f..65f32b0a857e31cfec611e5b723861456817b87c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -57,6 +57,28 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { return 0; } +const char* vnodeRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "voter"; + case TAOS_SYNC_ROLE_LEARNER: + return "learner"; + default: + return "unknown"; + } +} + +const ESyncRole vnodeStrToRole(char* str) { + if(strcmp(str, "voter") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "learner") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + int vnodeEncodeConfig(const void *pObj, SJson *pJson) { const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; @@ -117,6 +139,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1; @@ -128,9 +151,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; - vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); - for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { + vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); + for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; if (info == NULL) return -1; @@ -138,6 +161,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeRole", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); @@ -235,6 +259,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum, code); + if (code < 0) return -1; tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); if (code < 0) return -1; @@ -251,10 +277,13 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(nodeInfo); - if (arraySize != pCfg->syncCfg.replicaNum) return -1; + if(pCfg->syncCfg.totalReplicaNum == 0 && pCfg->syncCfg.replicaNum > 0){ + pCfg->syncCfg.totalReplicaNum = pCfg->syncCfg.replicaNum; + } + if (arraySize != pCfg->syncCfg.totalReplicaNum) return -1; - vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); + vDebug("vgId:%d, decode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); for (int i = 0; i < arraySize; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; @@ -266,6 +295,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; + char role[10] = {0}; + code = tjsonGetStringValue(info, "nodeRole", role); + if (code < 0) return -1; + if(strlen(role) != 0){ + pNode->nodeRole = vnodeStrToRole(role); + } + else{ + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + } vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 0779388ba9ca6da06ff0bfe3ce65219904118f0e..847125018c5f15baa660dc2f53677e17e18e0d7f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -143,26 +143,14 @@ _exit: return code; } -void vnodeUpdCommitSched(SVnode *pVnode) { - int64_t randNum = taosRand(); - pVnode->commitSched.commitMs = taosGetMonoTimestampMs(); - pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs); -} - -int vnodeShouldCommit(SVnode *pVnode) { - SVCommitSched *pSched = &pVnode->commitSched; - int64_t nowMs = taosGetMonoTimestampMs(); +int vnodeShouldCommit(SVnode *pVnode, bool atExit) { bool diskAvail = osDataSpaceAvailable(); bool needCommit = false; taosThreadMutexLock(&pVnode->mutex); - if (!pVnode->inUse || !diskAvail) { - goto _out; + if (pVnode->inUse && diskAvail) { + needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) || (pVnode->inUse->size > 0 && atExit); } - needCommit = - (((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || - (pVnode->inUse->size > 0 && pSched->commitMs + pSched->maxWaitMs < nowMs)); -_out: taosThreadMutexUnlock(&pVnode->mutex); return needCommit; } @@ -433,8 +421,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { vInfo("vgId:%d, start to commit, commitId:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pInfo->info.state.commitID, pInfo->info.state.committed, pInfo->info.state.commitTerm); - vnodeUpdCommitSched(pVnode); - // persist wal before starting if (walPersist(pVnode->pWal) < 0) { vError("vgId:%d, failed to persist wal since %s", TD_VID(pVnode), terrstr()); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index c7d155be0d50600e419940b03297d2bad6d29528..7d41edfdd95913eb45dd7581e8f105252b949845 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -76,20 +76,42 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p } SSyncCfg *pCfg = &info.config.syncCfg; - pCfg->myIndex = pReq->selfIndex; - pCfg->replicaNum = pReq->replica; + + pCfg->replicaNum = 0; + pCfg->totalReplicaNum = 0; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); - vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); for (int i = 0; i < pReq->replica; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->replicaNum++; + } + if(pReq->selfIndex != -1){ + pCfg->myIndex = pReq->selfIndex; + } + for (int i = pCfg->replicaNum; i < pReq->replica + pReq->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + pNode->nodeId = pReq->learnerReplicas[pCfg->totalReplicaNum].id; + pNode->nodePort = pReq->learnerReplicas[pCfg->totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[pCfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn)); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->totalReplicaNum++; + } + pCfg->totalReplicaNum += pReq->replica; + if(pReq->learnerSelfIndex != -1){ + pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex; } + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", + pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex); + info.config.syncCfg = *pCfg; ret = vnodeSaveInfo(dir, &info); if (ret < 0) { @@ -181,6 +203,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod SSyncCfg *pCfg = &info.config.syncCfg; pCfg->myIndex = 0; pCfg->replicaNum = 1; + pCfg->totalReplicaNum = 1; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); vInfo("vgId:%d, alter vnode replicas to 1", pReq->srcVgId); @@ -248,7 +271,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // save vnode info on dnode ep changed bool updated = false; SSyncCfg *pCfg = &info.config.syncCfg; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -286,8 +309,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - vnodeUpdCommitSched(pVnode); - int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool @@ -406,6 +427,14 @@ void vnodeClose(SVnode *pVnode) { // start the sync timer after the queue is ready int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); } +int32_t vnodeIsCatchUp(SVnode *pVnode){ + return syncIsCatchUp(pVnode->sync); +} + +ESyncRole vnodeGetRole(SVnode *pVnode){ + return syncGetRole(pVnode->sync); +} + void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c017266839a872711b5262ee6671d17d3a3a76b6..303d2a9ca449de504a582c8472951a1bf6c6c683 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -382,6 +382,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->syncRestore = state.restored; pLoad->syncCanRead = state.canRead; pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); + pLoad->numOfCachedTables = tsdbCacheGetElems(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->totalStorage = (int64_t)3 * 1073741824; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 71458acce2cdab30365c452d23efd5841bb9424a..052e4ab2c18e6dad651cf591962bcb4dd49a0a38 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -335,6 +335,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * // commit json if (!rollback) { + pWriter->info.state.committed = pWriter->ever; pVnode->config = pWriter->info.config; pVnode->state = (SVState){.committed = pWriter->info.state.committed, .applied = pWriter->info.state.committed, diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fb8d230ebab727f12f69d88c375cd10517b01155..dd576392f0c93b6ce5f2cff2d7d2ed8e9ed3c488 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -306,13 +306,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp void *pReq; int32_t len; int32_t ret; - /* - if (!pVnode->inUse) { - terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL; - vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); - return -1; - } - */ + if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); @@ -326,8 +320,8 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); ASSERT(pVnode->state.applied + 1 == version); - pVnode->state.applied = version; - pVnode->state.applyTerm = pMsg->info.conn.applyTerm; + atomic_store_64(&pVnode->state.applied, version); + atomic_store_64(&pVnode->state.applyTerm, pMsg->info.conn.applyTerm); if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; @@ -395,6 +389,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } break; + case TDMT_VND_TMQ_SEEK_TO_OFFSET: + if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; + } + break; case TDMT_VND_TMQ_ADD_CHECKINFO: if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; @@ -406,7 +405,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } break; case TDMT_STREAM_TASK_DEPLOY: { - if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } } break; @@ -453,13 +452,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp walApplyVer(pVnode->pWal, version); - /*vInfo("vgId:%d, push msg begin", pVnode->config.vgId);*/ if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } - /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ // commit if need if (needCommit) { @@ -516,7 +513,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || - pMsg->msgType == TDMT_VND_BATCH_META) && + pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; @@ -547,12 +544,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetBatchMeta(pVnode, pMsg); case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_VG_WALINFO: + return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); -#if 1 case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); -#endif case TDMT_STREAM_TASK_CHECK: return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: @@ -1457,7 +1454,8 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { } static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - vInfo("vgId:%d, alter replica confim msg is processed", TD_VID(pVnode)); + vInfo("vgId:%d, vnode management handle msgType:alter-confirm, alter replica confim msg is processed", + TD_VID(pVnode)); pRsp->msgType = TDMT_VND_ALTER_CONFIRM_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; @@ -1477,10 +1475,11 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void } vInfo("vgId:%d, start to alter vnode config, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 - " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d", + " cacheLast:%d cacheLastSize:%d days:%d keep0:%d keep1:%d keep2:%d fsync:%d level:%d walRetentionPeriod:%d " + "walRetentionSize:%d", TD_VID(pVnode), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, - req.walFsyncPeriod, req.walLevel); + req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize); if (pVnode->config.cacheLastSize != req.cacheLastSize) { pVnode->config.cacheLastSize = req.cacheLastSize; @@ -1510,13 +1509,21 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void if (pVnode->config.walCfg.fsyncPeriod != req.walFsyncPeriod) { pVnode->config.walCfg.fsyncPeriod = req.walFsyncPeriod; - walChanged = true; } if (pVnode->config.walCfg.level != req.walLevel) { pVnode->config.walCfg.level = req.walLevel; + walChanged = true; + } + if (pVnode->config.walCfg.retentionPeriod != req.walRetentionPeriod) { + pVnode->config.walCfg.retentionPeriod = req.walRetentionPeriod; + walChanged = true; + } + + if (pVnode->config.walCfg.retentionSize != req.walRetentionSize) { + pVnode->config.walCfg.retentionSize = req.walRetentionSize; walChanged = true; } @@ -1541,6 +1548,14 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void } } + if (req.sttTrigger != -1 && req.sttTrigger != pVnode->config.sttTrigger) { + pVnode->config.sttTrigger = req.sttTrigger; + } + + if (req.minRows != -1 && req.minRows != pVnode->config.tsdbCfg.minRows) { + pVnode->config.tsdbCfg.minRows = req.minRows; + } + if (walChanged) { walAlter(pVnode->pWal, &pVnode->config.walCfg); } @@ -1656,7 +1671,7 @@ _err: } static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SDropIndexReq req = {0}; - pRsp->msgType = TDMT_VND_CREATE_INDEX_RSP; + pRsp->msgType = TDMT_VND_DROP_INDEX_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; pRsp->contLen = 0; @@ -1665,6 +1680,7 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *p terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { pRsp->code = terrno; return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index e71b03d2af86c69ac6437094dfbeffe4a6da4faa..ba73c5fd4cfd1023bab861218a526898b342e072 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -112,9 +112,6 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak pVnode->blocked = true; pVnode->blockSec = taosGetTimestampSec(); pVnode->blockSeq = seq; -#if 0 - pVnode->blockInfo = pMsg->info; -#endif } taosThreadMutexUnlock(&pVnode->lock); @@ -129,8 +126,8 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak return code; } -void vnodeProposeCommitOnNeed(SVnode *pVnode) { - if (!vnodeShouldCommit(pVnode)) { +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { + if (!vnodeShouldCommit(pVnode, atExit)) { return; } @@ -145,18 +142,18 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode) { rpcMsg.pCont = pHead; rpcMsg.info.noResp = 1; + vInfo("vgId:%d, propose vnode commit", pVnode->config.vgId); bool isWeak = false; - if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { - vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); - goto _out; - } - - vInfo("vgId:%d, proposed vnode commit", pVnode->config.vgId); -_out: - vnodeUpdCommitSched(pVnode); - rpcFreeCont(rpcMsg.pCont); - rpcMsg.pCont = NULL; + if (!atExit) { + if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { + vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); + } + rpcFreeCont(rpcMsg.pCont); + rpcMsg.pCont = NULL; + } else { + tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg); + } } #if BATCH_ENABLE @@ -236,7 +233,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -288,7 +286,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -378,7 +377,7 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { return -1; } - int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + int32_t code = tmsgPutToQueue(msgcb, SYNC_RD_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; @@ -433,7 +432,23 @@ static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsm } static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); + if (pMsg->code == 0) { + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); + } + + const STraceId *trace = &pMsg->info.traceId; + SVnode *pVnode = pFsm->data; + vnodePostBlockMsg(pVnode, pMsg); + + SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; + if (rsp.info.handle != NULL) { + tmsgSendRsp(&rsp); + } + + vGTrace("vgId:%d, msg:%p is freed, code:0x%x index:%" PRId64, TD_VID(pVnode), pMsg, rsp.code, pMeta->index); + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return 0; } static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { @@ -443,6 +458,11 @@ static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const return 0; } +static SyncIndex vnodeSyncAppliedIndex(const SSyncFSM *pFSM) { + SVnode *pVnode = pFSM->data; + return atomic_load_64(&pVnode->state.applied); +} + static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", weak:%d, code:%d, state:%d %s, type:%s", @@ -505,24 +525,32 @@ static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *p return code; } -static void vnodeRestoreFinish(const SSyncFSM *pFsm) { +static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { SVnode *pVnode = pFsm->data; + SyncIndex appliedIdx = -1; do { - int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE); - if (itemSize == 0) { - vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); + appliedIdx = vnodeSyncAppliedIndex(pFsm); + ASSERT(appliedIdx <= commitIdx); + if (appliedIdx == commitIdx) { + vInfo("vgId:%d, no items to be applied, restore finish", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize); + vInfo("vgId:%d, restore not finish since %" PRId64 " items to be applied. commit-index:%" PRId64 + ", applied-index:%" PRId64, + pVnode->config.vgId, commitIdx - appliedIdx, commitIdx, appliedIdx); taosMsleep(10); } } while (true); - walApplyVer(pVnode->pWal, pVnode->state.applied); + ASSERT(commitIdx == vnodeSyncAppliedIndex(pFsm)); + walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; - vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); + vInfo("vgId:%d, sync restore finished, start to restore stream tasks by replay wal", pVnode->config.vgId); + + // start to restore all stream tasks + tqStartStreamTasks(pVnode->pTq); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { @@ -538,6 +566,19 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pVnode->lock); } +static void vnodeBecomeLearner(const SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vInfo("vgId:%d, become learner", pVnode->config.vgId); + + taosThreadMutexLock(&pVnode->lock); + if (pVnode->blocked) { + pVnode->blocked = false; + vDebug("vgId:%d, become learner and post block", pVnode->config.vgId); + tsem_post(&pVnode->syncSem); + } + taosThreadMutexUnlock(&pVnode->lock); +} + static void vnodeBecomeLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become leader", pVnode->config.vgId); @@ -569,6 +610,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; pFsm->FpCommitCb = vnodeSyncCommitMsg; + pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex; pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg; pFsm->FpRollBackCb = vnodeSyncRollBackMsg; pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshotInfo; @@ -578,6 +620,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpApplyQueueItems = vnodeApplyQueueItems; pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; + pFsm->FpBecomeLearnerCb = vnodeBecomeLearner; pFsm->FpReConfigCb = NULL; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; @@ -610,7 +653,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncCfg *pCfg = &syncInfo.syncCfg; vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index 632034d6b60ef0454ae53ba799534934608b9b07..bc56ff4c7f766862f2f70057d09af56d2a9f8895 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,9 +8,9 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util transport qcom + PRIVATE os util transport qcom nodes ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5d76ab6f2537e7ed2a6413bee42e95b739b9186b..13a7328887ddba92a8244ec8c05d3a5ac4577780 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -66,8 +66,8 @@ typedef enum { } CTG_CACHE_ITEM; #define CTG_CI_FLAG_LEVEL_GLOBAL (1) -#define CTG_CI_FLAG_LEVEL_CLUSTER (1<<1) -#define CTG_CI_FLAG_LEVEL_DB (1<<2) +#define CTG_CI_FLAG_LEVEL_CLUSTER (1 << 1) +#define CTG_CI_FLAG_LEVEL_DB (1 << 2) enum { CTG_READ = 1, @@ -133,6 +133,19 @@ typedef struct SCtgCacheStat { uint64_t cacheNHit[CTG_CI_MAX_VALUE]; } SCtgCacheStat; +typedef struct SCtgAuthReq { + SRequestConnInfo* pConn; + SUserAuthInfo* pRawReq; + SGetUserAuthRsp authInfo; + AUTH_TYPE singleType; + bool onlyCache; +} SCtgAuthReq; + +typedef struct SCtgAuthRsp { + SUserAuthRes* pRawRes; + bool metaNotExists; +} SCtgAuthRsp; + typedef struct SCtgTbCacheInfo { bool inCache; uint64_t dbId; @@ -225,13 +238,13 @@ typedef struct SCtgVgCache { } SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads - uint64_t dbId; - int8_t deleted; - SCtgVgCache vgCache; - SHashObj* tbCache; // key:tbname, value:SCtgTbCache - SHashObj* stbCache; // key:suid, value:char* - uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads + uint64_t dbId; + int8_t deleted; + SCtgVgCache vgCache; + SHashObj* tbCache; // key:tbname, value:SCtgTbCache + SHashObj* stbCache; // key:suid, value:char* + uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; } SCtgDBCache; typedef struct SCtgRentSlot { @@ -249,12 +262,8 @@ typedef struct SCtgRentMgmt { } SCtgRentMgmt; typedef struct SCtgUserAuth { - int32_t version; - SRWLatch lock; - bool superUser; - SHashObj* createdDbs; - SHashObj* readDbs; - SHashObj* writeDbs; + SRWLatch lock; + SGetUserAuthRsp userAuth; } SCtgUserAuth; typedef struct SCatalog { @@ -511,7 +520,7 @@ typedef struct SCtgCacheItemInfo { uint32_t persistSize; } SCtgCacheItemInfo; -#define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) +#define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) @@ -521,87 +530,89 @@ typedef struct SCtgCacheItemInfo { #define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 -#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 -#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 +#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 +#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 +#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 #define CTG_DB_NUM_RESET(_item) dbCache->dbCacheNum[_item] = 0 -#define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) -#define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) -#define CTG_STAT_NUM_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) -#define CTG_STAT_NUM_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) -#define CTG_STAT_HIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) -#define CTG_STAT_HIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) +#define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) +#define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) +#define CTG_STAT_NUM_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) +#define CTG_STAT_NUM_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) +#define CTG_STAT_HIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) +#define CTG_STAT_HIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) #define CTG_STAT_NHIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNHit[item], n)) #define CTG_STAT_NHIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNHit[item], n)) -#define CTG_CACHE_NUM_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNum[item], n)) -#define CTG_CACHE_NUM_DEC(item, n) (CTG_STAT_DEC(pCtg->cacheStat.cacheNum[item], n)) -#define CTG_CACHE_HIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheHit[item], n)) +#define CTG_CACHE_NUM_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNum[item], n)) +#define CTG_CACHE_NUM_DEC(item, n) (CTG_STAT_DEC(pCtg->cacheStat.cacheNum[item], n)) +#define CTG_CACHE_HIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheHit[item], n)) #define CTG_CACHE_NHIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNHit[item], n)) -#define CTG_META_NUM_INC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_STABLE_META); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_CTABLE_META); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_NTABLE_META); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_SYSTABLE_META); \ - break; \ - default: \ - CTG_DB_NUM_INC(CTG_CI_OTHERTABLE_META); \ - break; \ - } \ -} while (0) - -#define CTG_META_NUM_DEC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_STABLE_META); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_CTABLE_META); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_NTABLE_META); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_SYSTABLE_META); \ - break; \ - default: \ - CTG_DB_NUM_DEC(CTG_CI_OTHERTABLE_META); \ - break; \ - } \ -} while (0) - -#define CTG_META_HIT_INC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_STABLE_META, 1); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_CTABLE_META, 1); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_NTABLE_META, 1); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_SYSTABLE_META, 1); \ - break; \ - default: \ - CTG_CACHE_HIT_INC(CTG_CI_OTHERTABLE_META, 1); \ - break; \ - } \ -} while (0) +#define CTG_META_NUM_INC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_STABLE_META); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_CTABLE_META); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_NTABLE_META); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_SYSTABLE_META); \ + break; \ + default: \ + CTG_DB_NUM_INC(CTG_CI_OTHERTABLE_META); \ + break; \ + } \ + } while (0) -#define CTG_META_NHIT_INC() CTG_CACHE_NHIT_INC(CTG_CI_OTHERTABLE_META, 1) +#define CTG_META_NUM_DEC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_STABLE_META); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_CTABLE_META); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_NTABLE_META); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_SYSTABLE_META); \ + break; \ + default: \ + CTG_DB_NUM_DEC(CTG_CI_OTHERTABLE_META); \ + break; \ + } \ + } while (0) +#define CTG_META_HIT_INC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_STABLE_META, 1); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_CTABLE_META, 1); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_NTABLE_META, 1); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_SYSTABLE_META, 1); \ + break; \ + default: \ + CTG_CACHE_HIT_INC(CTG_CI_OTHERTABLE_META, 1); \ + break; \ + } \ + } while (0) + +#define CTG_META_NHIT_INC() CTG_CACHE_NHIT_INC(CTG_CI_OTHERTABLE_META, 1) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -661,7 +672,6 @@ typedef struct SCtgCacheItemInfo { #define ctgTaskDebug(param, ...) qDebug("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) #define ctgTaskTrace(param, ...) qTrace("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) - #define CTG_LOCK_DEBUG(...) \ do { \ if (gCTGDebug.lockEnable) { \ @@ -683,38 +693,38 @@ typedef struct SCtgCacheItemInfo { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define CTG_LOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ - } \ +#define CTG_LOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ + } else { \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ + } \ } while (0) -#define CTG_UNLOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ - } \ +#define CTG_UNLOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ + } else { \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ + } \ } while (0) #define CTG_ERR_RET(c) \ @@ -808,7 +818,7 @@ int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32 int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType, uint64_t* suid, char* stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool* inCache, bool* pass); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pReq, bool* inCache, SCtgAuthRsp* pRes); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid, @@ -881,7 +891,8 @@ void ctgFreeHandleImpl(SCatalog* pCtg); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); -int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, + int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); @@ -902,20 +913,24 @@ void ctgFreeQNode(SCtgQNode* node); void ctgClearHandle(SCatalog* pCtg); void ctgFreeTbCacheImpl(SCtgTbCache* pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); -int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); +int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, + bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); -int32_t ctgdGetOneHandle(SCatalog **pHandle); +int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); -int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb); -int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName); -void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache); -void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache); -void ctgGetGlobalCacheStat(SCtgCacheStat *pStat); - -extern SCatalogMgmt gCtgMgmt; -extern SCtgDebug gCTGDebug; -extern SCtgAsyncFps gCtgAsyncFps[]; +int32_t ctgAcquireVgMetaFromCache(SCatalog* pCtg, const char* dbFName, const char* tbName, SCtgDBCache** pDb, + SCtgTbCache** pTb); +int32_t ctgCopyTbMeta(SCatalog* pCtg, SCtgTbMetaCtx* ctx, SCtgDBCache** pDb, SCtgTbCache** pTb, STableMeta** pTableMeta, + char* dbFName); +void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); +void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); +int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); + +extern SCatalogMgmt gCtgMgmt; +extern SCtgDebug gCTGDebug; +extern SCtgAsyncFps gCtgAsyncFps[]; extern SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE]; #ifdef __cplusplus diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 8a56b0889b679b750e7077662dbbc4361e478310..bddc6c01a7cc8bf782db98987651ff1546707ce9 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -319,14 +319,13 @@ _return: CTG_RET(code); } -int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass, bool* exists) { +int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, SUserAuthRes* pRes, bool* exists) { bool inCache = false; int32_t code = 0; + SCtgAuthRsp rsp = {0}; + rsp.pRawRes = pRes; - *pass = false; - - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pReq, &inCache, &rsp)); if (inCache) { if (exists) { @@ -339,30 +338,22 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, co return TSDB_CODE_SUCCESS; } - SGetUserAuthRsp authRsp = {0}; - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, user, &authRsp, NULL)); - - if (authRsp.superAuth) { - *pass = true; - goto _return; - } - - if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - goto _return; - } + SCtgAuthReq req = {0}; + req.pRawReq = pReq; + req.pConn = pConn; + req.onlyCache = exists ? true : false; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL)); - if (CTG_AUTH_READ(type) && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { - *pass = true; - } else if (CTG_AUTH_WRITE(type) && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { - *pass = true; + CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp)); + if (rsp.metaNotExists && exists) { + *exists = false; } _return: - ctgUpdateUserEnqueue(pCtg, &authRsp, false); + ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); - return TSDB_CODE_SUCCESS; + CTG_RET(code); } int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { @@ -1373,7 +1364,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ void* key = taosHashGetKey(pAuth, &len); strncpy((*users)[i].user, key, len); (*users)[i].user[len] = 0; - (*users)[i].version = pAuth->version; + (*users)[i].version = pAuth->userAuth.version; ++i; if (i >= *num) { taosHashCancelIterate(pCtg->userCache, pAuth); @@ -1457,32 +1448,30 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { +int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass) { + if (NULL == pCtg || NULL == pConn || NULL == pAuth || NULL == pRes) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, NULL)); + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, pAuth, pRes, NULL)); _return: CTG_API_LEAVE(code); } -int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass, bool* exists) { +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { + if (NULL == pCtg || NULL == pAuth || NULL == pRes || NULL == exists) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists)); + CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, pAuth, pRes, exists)); _return: diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6c5666193753aea698559937c660022a5650bbbb..f2a354997da4c199d58bfe141f29d4f48270f65e 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1550,45 +1550,20 @@ _return: int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; - SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - bool pass = false; SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - if (pOut->superAuth) { - pass = true; - goto _return; - } + ctgUpdateUserEnqueue(pCtg, pOut, true); + taosMemoryFreeClear(pTask->msgCtx.out); - if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - goto _return; - } + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); - if (CTG_AUTH_READ(ctx->user.type) && pOut->readDbs && - taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - } else if (CTG_AUTH_WRITE(ctx->user.type) && pOut->writeDbs && - taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - } + return TSDB_CODE_SUCCESS; _return: - if (TSDB_CODE_SUCCESS == code) { - pTask->res = taosMemoryCalloc(1, sizeof(bool)); - if (NULL == pTask->res) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *(bool*)pTask->res = pass; - } - } - - ctgUpdateUserEnqueue(pCtg, pOut, false); - taosMemoryFreeClear(pTask->msgCtx.out); - ctgHandleTaskEnd(pTask, code); CTG_RET(code); @@ -2081,31 +2056,39 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) { } int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { + int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; bool inCache = false; - bool pass = false; + SCtgAuthRsp rsp = {0}; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass)); + rsp.pRawRes = taosMemoryCalloc(1, sizeof(SUserAuthRes)); + if (NULL == rsp.pRawRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, &inCache, &rsp)); if (inCache) { - pTask->res = taosMemoryCalloc(1, sizeof(bool)); - if (NULL == pTask->res) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } - *(bool*)pTask->res = pass; + pTask->res = rsp.pRawRes; CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); + taosMemoryFreeClear(rsp.pRawRes); + if (rsp.metaNotExists) { + CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName)); + } else { + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); + } + return TSDB_CODE_SUCCESS; } @@ -2155,6 +2138,20 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } + +int32_t ctgGetUserCb(SCtgTask* pTask) { + int32_t code = 0; + + CTG_ERR_JRET(pTask->subRes.code); + + CTG_RET(ctgLaunchGetUserTask(pTask)); + +_return: + + CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); +} + + int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { SCtgDbVgCtx* ctx = pTask->taskCtx; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index a13a58bec8b1a23efd47427ad0caf07405c87ca7..92a5956fc4044e86dc1b8ae8dfc7fdaf2a552ac1 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -52,7 +52,6 @@ SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER, }; - int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); @@ -263,10 +262,11 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { +int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, + SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - bool vgInCache = false; + bool vgInCache = false; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -331,7 +331,6 @@ _return: return TSDB_CODE_SUCCESS; } - /* int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; @@ -385,9 +384,10 @@ _return: } */ -int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { +int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, + SCtgTbCache **pTb) { SCtgTbCache *pCache = NULL; - char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); goto _return; @@ -484,10 +484,11 @@ int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32 return TSDB_CODE_SUCCESS; } -int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName) { +int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, + char *dbFName) { SCtgDBCache *dbCache = *pDb; SCtgTbCache *tbCache = *pTb; - STableMeta *tbMeta = tbCache->pMeta; + STableMeta *tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; @@ -516,12 +517,12 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt memcpy(*pTableMeta, tbMeta, metaSize); - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); taosHashRelease(dbCache->tbCache, tbCache); *pTb = NULL; - + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); @@ -553,7 +554,6 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt return TSDB_CODE_SUCCESS; } - int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; @@ -623,17 +623,17 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, // PROCESS FOR CHILD TABLE - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); if (tbCache) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); taosHashRelease(dbCache->tbCache, tbCache); } - + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } @@ -703,55 +703,40 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog *pCtg, char *user, char *dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { - char *p = strchr(dbFName, '.'); - if (p) { - ++p; - } else { - p = dbFName; - } - - if (IS_SYS_DBNAME(p)) { +int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { + if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; - *pass = true; - ctgDebug("sysdb %s, pass", dbFName); + pRes->pRawRes->pass = true; + ctgDebug("sysdb %s, pass", pReq->tbName.dbname); return TSDB_CODE_SUCCESS; } - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); if (NULL == pUser) { - ctgDebug("user not in cache, user:%s", user); + ctgDebug("user not in cache, user:%s", pReq->user); goto _return; } *inCache = true; - ctgDebug("Got user from cache, user:%s", user); + ctgDebug("Got user from cache, user:%s", pReq->user); CTG_CACHE_HIT_INC(CTG_CI_USER, 1); - if (pUser->superUser) { - *pass = true; - return TSDB_CODE_SUCCESS; - } + SCtgAuthReq req = {0}; + req.pRawReq = pReq; + req.onlyCache = true; CTG_LOCK(CTG_READ, &pUser->lock); - if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - CTG_UNLOCK(CTG_READ, &pUser->lock); - return TSDB_CODE_SUCCESS; - } - - if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && CTG_AUTH_READ(type)) { - *pass = true; - } + memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); + int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes); + CTG_UNLOCK(CTG_READ, &pUser->lock); + CTG_ERR_JRET(code); - if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && CTG_AUTH_WRITE(type)) { - *pass = true; + if (pRes->metaNotExists) { + goto _return; } - CTG_UNLOCK(CTG_READ, &pUser->lock); - - return TSDB_CODE_SUCCESS; + CTG_RET(code); _return: @@ -1639,7 +1624,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa } CTG_DB_NUM_INC(CTG_CI_TBL_SMA); - + *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); @@ -1725,9 +1710,9 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } -int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { - int32_t* key = (int32_t*)lp; - SVgroupInfo* pVg = (SVgroupInfo*)rp; +int32_t ctgVgInfoIdComp(void const *lp, void const *rp) { + int32_t *key = (int32_t *)lp; + SVgroupInfo *pVg = (SVgroupInfo *)rp; if (*key < pVg->vgId) { return -1; @@ -1738,7 +1723,6 @@ int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { return 0; } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1805,10 +1789,10 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { dbCache = NULL; - //if (!IS_SYS_DBNAME(dbFName)) { - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + // if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); //} _return: @@ -2054,11 +2038,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { if (NULL == pUser) { SCtgUserAuth userAuth = {0}; - userAuth.version = msg->userAuth.version; - userAuth.superUser = msg->userAuth.superAuth; - userAuth.createdDbs = msg->userAuth.createdDbs; - userAuth.readDbs = msg->userAuth.readDbs; - userAuth.writeDbs = msg->userAuth.writeDbs; + memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); @@ -2072,22 +2052,32 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - pUser->version = msg->userAuth.version; - CTG_LOCK(CTG_WRITE, &pUser->lock); - taosHashCleanup(pUser->createdDbs); - pUser->createdDbs = msg->userAuth.createdDbs; + taosHashCleanup(pUser->userAuth.createdDbs); + pUser->userAuth.createdDbs = msg->userAuth.createdDbs; msg->userAuth.createdDbs = NULL; - taosHashCleanup(pUser->readDbs); - pUser->readDbs = msg->userAuth.readDbs; + taosHashCleanup(pUser->userAuth.readDbs); + pUser->userAuth.readDbs = msg->userAuth.readDbs; msg->userAuth.readDbs = NULL; - taosHashCleanup(pUser->writeDbs); - pUser->writeDbs = msg->userAuth.writeDbs; + taosHashCleanup(pUser->userAuth.writeDbs); + pUser->userAuth.writeDbs = msg->userAuth.writeDbs; msg->userAuth.writeDbs = NULL; + taosHashCleanup(pUser->userAuth.readTbs); + pUser->userAuth.readTbs = msg->userAuth.readTbs; + msg->userAuth.readTbs = NULL; + + taosHashCleanup(pUser->userAuth.writeTbs); + pUser->userAuth.writeTbs = msg->userAuth.writeTbs; + msg->userAuth.writeTbs = NULL; + + taosHashCleanup(pUser->userAuth.useDbs); + pUser->userAuth.useDbs = msg->userAuth.useDbs; + msg->userAuth.useDbs = NULL; + CTG_UNLOCK(CTG_WRITE, &pUser->lock); _return: @@ -2095,6 +2085,9 @@ _return: taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); + taosHashCleanup(msg->userAuth.readTbs); + taosHashCleanup(msg->userAuth.writeTbs); + taosHashCleanup(msg->userAuth.useDbs); taosMemoryFreeClear(msg); @@ -2287,6 +2280,9 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) { taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); + taosHashCleanup(msg->userAuth.readTbs); + taosHashCleanup(msg->userAuth.writeTbs); + taosHashCleanup(msg->userAuth.useDbs); taosMemoryFreeClear(op->data); break; } @@ -2526,7 +2522,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArrayPush(ctx->pResList, &(SMetaRes){0}); CTG_META_NHIT_INC(); - + continue; } @@ -2544,7 +2540,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe STableMeta *tbMeta = pCache->pMeta; CTG_META_HIT_INC(tbMeta->tableType); - + SCtgTbMetaCtx nctx = {0}; nctx.flag = flag; nctx.tbInfo.inCache = true; @@ -2626,7 +2622,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArrayPush(ctx->pResList, &(SMetaRes){0}); taosMemoryFreeClear(pTableMeta); - + CTG_META_NHIT_INC(); continue; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b5d587d257a8d084246efa86ed72e77a723de98c..12ff8a7b388a994e8fd0249a557e993768934342 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -21,6 +21,8 @@ extern SCatalogMgmt gCtgMgmt; SCtgDebug gCTGDebug = {0}; +#if 0 + void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { taosMemoryFree(param); @@ -224,6 +226,7 @@ _return: CTG_RET(code); } +#endif int32_t ctgdEnableDebug(char *option, bool enable) { if (0 == strcasecmp(option, "lock")) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 57645e8b879ffb332c48575387abff0ed77aec35..bb45b9bd953466c76c0400d8c06ccd42095ed402 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -174,9 +174,12 @@ void ctgFreeSMetaData(SMetaData* pData) { } void ctgFreeSCtgUserAuth(SCtgUserAuth* userCache) { - taosHashCleanup(userCache->createdDbs); - taosHashCleanup(userCache->readDbs); - taosHashCleanup(userCache->writeDbs); + taosHashCleanup(userCache->userAuth.createdDbs); + taosHashCleanup(userCache->userAuth.readDbs); + taosHashCleanup(userCache->userAuth.writeDbs); + taosHashCleanup(userCache->userAuth.readTbs); + taosHashCleanup(userCache->userAuth.writeTbs); + taosHashCleanup(userCache->userAuth.useDbs); } void ctgFreeMetaRent(SCtgRentMgmt* mgmt) { @@ -419,6 +422,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { taosHashCleanup(pOut->createdDbs); taosHashCleanup(pOut->readDbs); taosHashCleanup(pOut->writeDbs); + taosHashCleanup(pOut->readTbs); + taosHashCleanup(pOut->writeTbs); + taosHashCleanup(pOut->useDbs); taosMemoryFreeClear(pCtx->out); break; } @@ -865,19 +871,19 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); -/* - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } + /* + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } -*/ + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + */ if (NULL == vgInfo) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, @@ -902,7 +908,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* CTG_ERR_RET(ctgMakeVgArray(dbInfo)); - int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -982,43 +988,43 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* CTG_RET(code); } -int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId) { - int32_t code = 0; - CTG_ERR_RET(ctgMakeVgArray(dbInfo)); +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, + int32_t* vgId) { + int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); - int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); - if (vgNum <= 0) { - ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } - SVgroupInfo* vgInfo = NULL; - char tbFullName[TSDB_TABLE_FNAME_LEN]; - snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); - int32_t offset = strlen(tbFullName); - - for (int32_t i = 0; i < tbNum; ++i) { - snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); - uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, - dbInfo->hashPrefix, dbInfo->hashSuffix); + SVgroupInfo* vgInfo = NULL; + char tbFullName[TSDB_TABLE_FNAME_LEN]; + snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); + int32_t offset = strlen(tbFullName); + + for (int32_t i = 0; i < tbNum; ++i) { + snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, + dbInfo->hashPrefix, dbInfo->hashSuffix); - vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); - if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - (int32_t)taosArrayGetSize(dbInfo->vgArray)); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, + (int32_t)taosArrayGetSize(dbInfo->vgArray)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - vgId[i] = vgInfo->vgId; + vgId[i] = vgInfo->vgId; - ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId); - } + ctgDebug("Got tb %s vgId:%d", tbFullName, vgInfo->vgId); + } - CTG_RET(code); + CTG_RET(code); } - int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t*)key1 < ((SSTableVersion*)key2)->suid) { return -1; @@ -1063,26 +1069,25 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { return TSDB_CODE_SUCCESS; } - + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); if (NULL == dbInfo->vgArray) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { taosArrayPush(dbInfo->vgArray, pIter); pIter = taosHashIterate(dbInfo->vgHash, pIter); } - + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); } return TSDB_CODE_SUCCESS; } - int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { CTG_ERR_RET(ctgMakeVgArray(src)); @@ -1326,6 +1331,144 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*) static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } +int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { + int32_t code = 0; + STableMeta* pMeta = NULL; + SGetUserAuthRsp* pInfo = &req->authInfo; + SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&req->pRawReq->tbName, tbFullName); + char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName)); + if (pCond) { + if (strlen(pCond) > 1) { + CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + } + + res->pRawRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + res->pRawRes->pass = false; + + // CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + // if (NULL == pMeta) { + // if (req->onlyCache) { + // res->metaNotExists = true; + // ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + // return TSDB_CODE_SUCCESS; + // } + + // CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta)); + // } + + // if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { + // res->pRawRes->pass = false; + // goto _return; + // } + + // if (TSDB_CHILD_TABLE == pMeta->tableType) { + // res->pRawRes->pass = true; + + // /* + // char stbName[TSDB_TABLE_NAME_LEN] = {0}; + // CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName)); + // if (0 == stbName[0]) { + // if (req->onlyCache) { + // res->notExists = true; + // return TSDB_CODE_SUCCESS; + // } + + // CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0)); + // } + // */ + // } + +_return: + + taosMemoryFree(pMeta); + + CTG_RET(code); +} + +int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { + int32_t code = 0; + SUserAuthInfo* pReq = req->pRawReq; + SUserAuthRes* pRes = res->pRawRes; + SGetUserAuthRsp* pInfo = &req->authInfo; + + pRes->pass = false; + pRes->pCond = NULL; + + if (!pInfo->enable) { + pRes->pass = false; + return TSDB_CODE_SUCCESS; + } + + if (pInfo->superAuth) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(&pReq->tbName, dbFName); + + if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + switch (pReq->type) { + case AUTH_TYPE_READ: { + if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + req->singleType = AUTH_TYPE_READ; + CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); + if (pRes->pass) { + return TSDB_CODE_SUCCESS; + } + } + + if (pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + case AUTH_TYPE_WRITE: { + if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + req->singleType = AUTH_TYPE_WRITE; + CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); + if (pRes->pass) { + return TSDB_CODE_SUCCESS; + } + } + + if (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + case AUTH_TYPE_READ_OR_WRITE: { + if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) || + (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) || + (pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName)))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + default: + break; + } + + return TSDB_CODE_SUCCESS; +} + +#if 0 static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) { if (NULL == pSrc) { return TSDB_CODE_SUCCESS; @@ -1417,21 +1560,22 @@ void catalogFreeMetaData(SMetaData* pData) { taosMemoryFreeClear(pData->pSvrVer); taosMemoryFree(pData); } +#endif void ctgGetClusterCacheStat(SCatalog* pCtg) { for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) { if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) { continue; } - + pCtg->cacheStat.cacheNum[i] = 0; } - - SCtgDBCache *dbCache = NULL; - void *pIter = taosHashIterate(pCtg->dbCache, NULL); + + SCtgDBCache* dbCache = NULL; + void* pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { - dbCache = (SCtgDBCache *)pIter; - + dbCache = (SCtgDBCache*)pIter; + for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) { if (0 == (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_DB)) { continue; @@ -1444,33 +1588,33 @@ void ctgGetClusterCacheStat(SCatalog* pCtg) { } } -void ctgSummaryClusterCacheStat(SCatalog *pCtg) { +void ctgSummaryClusterCacheStat(SCatalog* pCtg) { for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) { if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) { continue; } - + gCtgMgmt.statInfo.cache.cacheNum[i] += pCtg->cacheStat.cacheNum[i]; gCtgMgmt.statInfo.cache.cacheHit[i] += pCtg->cacheStat.cacheHit[i]; gCtgMgmt.statInfo.cache.cacheNHit[i] += pCtg->cacheStat.cacheNHit[i]; } } -void ctgGetGlobalCacheStat(SCtgCacheStat *pStat) { +void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) { for (int32_t i = 0; i < CTG_CI_MAX_VALUE; ++i) { if (gCtgStatItem[i].flag & CTG_CI_FLAG_LEVEL_GLOBAL) { continue; } - + gCtgMgmt.statInfo.cache.cacheNum[i] = 0; gCtgMgmt.statInfo.cache.cacheHit[i] = 0; gCtgMgmt.statInfo.cache.cacheNHit[i] = 0; } - - SCatalog *pCtg = NULL; - void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + + SCatalog* pCtg = NULL; + void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { - pCtg = *(SCatalog **)pIter; + pCtg = *(SCatalog**)pIter; if (pCtg) { ctgGetClusterCacheStat(pCtg); diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt index dbbb24ce0b5e8b3ac37b212d88b49d2766bf78ca..de4d08835cf0ebd149e36f252330dfb73940f8df 100644 --- a/source/libs/catalog/test/CMakeLists.txt +++ b/source/libs/catalog/test/CMakeLists.txt @@ -9,7 +9,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( catalogTest - PUBLIC os util common catalog transport gtest qcom taos_static + PUBLIC os util common nodes catalog transport gtest qcom taos_static ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 5e543384ac2747005ddb40602ca778253ea1eacd..2cba433e849f89b161b7b3e2c3e58d9694aacdb0 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -27,8 +27,8 @@ #ifdef WINDOWS #define TD_USE_WINSOCK #endif -#include "catalogInt.h" #include "catalog.h" +#include "catalogInt.h" #include "os.h" #include "stub.h" #include "taos.h" @@ -48,7 +48,7 @@ void ctgTestSetRspCTableMeta(); void ctgTestSetRspSTableMeta(); void ctgTestSetRspMultiSTableMeta(); -extern int32_t clientConnRefPool; +extern int32_t clientConnRefPool; enum { CTGT_RSP_VGINFO = 1, @@ -298,7 +298,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + SUseDbRsp usedbRsp = {0}; strcpy(usedbRsp.db, ctgTestDbname); usedbRsp.vgVersion = ctgTestVgVersion; @@ -343,7 +343,7 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestTablename); @@ -384,13 +384,13 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST; } void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); @@ -438,7 +438,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); @@ -450,7 +450,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid; - metaRsp.tuid = ctgTestSuid+1; + metaRsp.tuid = ctgTestSuid + 1; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -486,7 +486,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableMetaRsp metaRsp = {0}; @@ -536,16 +536,14 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp tFreeSTableMetaRsp(&metaRsp); } - void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + pRsp->code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; pRsp->contLen = 0; pRsp->pCont = NULL; } - void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); @@ -553,6 +551,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p strcpy(userRsp.user, ctgTestUsername); userRsp.version = 1; userRsp.superAuth = 1; + userRsp.enable = 1; int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp); void *pReq = rpcMallocCont(contLen); @@ -565,7 +564,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableCfgRsp tblRsp = {0}; @@ -600,7 +599,7 @@ void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableIndexRsp tblRsp = {0}; @@ -611,10 +610,10 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg STableIndexInfo info = {0}; for (int32_t i = 0; i < ctgTestIndexNum; ++i) { info.interval = 1 + i; - info.expr = (char*)taosMemoryCalloc(1, 10); + info.expr = (char *)taosMemoryCalloc(1, 10); taosArrayPush(tblRsp.pIndex, &info); } - + int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp); void *pReq = rpcMallocCont(contLen); tSerializeSTableIndexRsp(pReq, contLen, &tblRsp); @@ -628,12 +627,12 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; SDbCfgRsp dbRsp = {0}; dbRsp.numOfVgroups = ctgTestVgNum; - + int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp); void *pReq = rpcMallocCont(contLen); tSerializeSDbCfgRsp(pReq, contLen, &dbRsp); @@ -651,7 +650,7 @@ void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * for (int32_t i = 0; i < ctgTestQnodeNum; ++i) { SQueryNodeLoad nodeLoad = {0}; nodeLoad.addr.nodeId = i; - + (void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad); } @@ -672,12 +671,17 @@ void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pR SRetrieveFuncRsp funcRsp = {0}; funcRsp.numOfFuncs = 1; funcRsp.pFuncInfos = taosArrayInit(1, sizeof(SFuncInfo)); + funcRsp.pFuncExtraInfos = taosArrayInit(1, sizeof(SFuncExtraInfo)); SFuncInfo funcInfo = {0}; strcpy(funcInfo.name, "func1"); funcInfo.funcType = ctgTestFuncType; - - (void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo); + (void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo); + SFuncExtraInfo extraInfo = {0}; + extraInfo.funcVersion = 0; + extraInfo.funcCreatedTime = taosGetTimestampMs(); + (void)taosArrayPush(funcRsp.pFuncExtraInfos, &extraInfo); + int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSRetrieveFuncRsp(pReq, rspLen, &funcRsp); @@ -694,7 +698,7 @@ void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRs SServerVerRsp verRsp = {0}; strcpy(verRsp.ver, "1.0"); - + int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSServerVerRsp(pReq, rspLen, &verRsp); @@ -713,9 +717,9 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p epSet.numOfEps = 1; tstrncpy(epSet.eps[0].fqdn, "localhost", TSDB_FQDN_LEN); epSet.eps[0].port = 6030; - + (void)taosArrayPush(dRsp.dnodeList, &epSet); - + int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSDnodeListRsp(pReq, rspLen, &dRsp); @@ -727,8 +731,6 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p tFreeSDnodeListRsp(&dRsp); } - - void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (pMsg->msgType) { case TDMT_MND_USE_DB: @@ -745,7 +747,7 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp); break; case TDMT_MND_QNODE_LIST: - ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp); + ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp); break; case TDMT_MND_RETRIEVE_FUNC: ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp); @@ -763,7 +765,6 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) return; } - void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (ctgTestRspFunc[ctgTestRspIdx]) { case CTGT_RSP_VGINFO: @@ -1058,7 +1059,7 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { void *ctgTestGetDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SArray *vgList = NULL; int32_t n = 0; @@ -1204,7 +1205,6 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } - void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { TAOS_ROW row; int num_fields = taos_num_fields(result); @@ -1220,23 +1220,22 @@ void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { } } -void ctgTestExecQuery(TAOS * taos, char* sql, bool fetch, int32_t *rows) { +void ctgTestExecQuery(TAOS *taos, char *sql, bool fetch, int32_t *rows) { TAOS_RES *result = taos_query(taos, sql); - int code = taos_errno(result); + int code = taos_errno(result); ASSERT_EQ(code, 0); if (fetch) { ctgTestFetchRows(result, rows); } - + taos_free_result(result); } - TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; SVgroupInfo vgInfo = {0}; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -1372,7 +1371,7 @@ TEST(tableMeta, normalTable) { TEST(tableMeta, childTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1484,7 +1483,7 @@ TEST(tableMeta, childTableCase) { TEST(tableMeta, superTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1626,7 +1625,7 @@ TEST(tableMeta, superTableCase) { TEST(tableMeta, rmStbMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1696,7 +1695,7 @@ TEST(tableMeta, rmStbMeta) { TEST(tableMeta, updateStbMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1785,7 +1784,7 @@ TEST(tableMeta, updateStbMeta) { TEST(getIndexInfo, notExists) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1809,13 +1808,13 @@ TEST(getIndexInfo, notExists) { SIndexInfo info; code = catalogGetIndexMeta(pCtg, mockPointer, "index1", &info); ASSERT_TRUE(code != 0); - + catalogDestroy(); } TEST(refreshGetMeta, normal2normal) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1894,7 +1893,7 @@ TEST(refreshGetMeta, normal2normal) { TEST(refreshGetMeta, normal2notexist) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1964,7 +1963,7 @@ TEST(refreshGetMeta, normal2notexist) { TEST(refreshGetMeta, normal2child) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2045,10 +2044,9 @@ TEST(refreshGetMeta, normal2child) { ctgTestCurrentSTableName = NULL; } - TEST(refreshGetMeta, stable2child) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2134,7 +2132,7 @@ TEST(refreshGetMeta, stable2child) { TEST(refreshGetMeta, stable2stable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2220,7 +2218,7 @@ TEST(refreshGetMeta, stable2stable) { TEST(refreshGetMeta, child2stable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2306,7 +2304,7 @@ TEST(refreshGetMeta, child2stable) { TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2343,7 +2341,7 @@ TEST(tableDistVgroup, normalTable) { TEST(tableDistVgroup, childTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2381,7 +2379,7 @@ TEST(tableDistVgroup, childTableCase) { TEST(tableDistVgroup, superTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2430,7 +2428,7 @@ TEST(tableDistVgroup, superTableCase) { TEST(dbVgroup, getSetDbVgroupCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2520,7 +2518,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { TEST(multiThread, getSetRmSameDbVgroup) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2572,7 +2570,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { TEST(multiThread, getSetRmDiffDbVgroup) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2624,7 +2622,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2675,7 +2673,7 @@ TEST(multiThread, ctableMeta) { TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2754,7 +2752,7 @@ TEST(rentTest, allRent) { TEST(apiTest, catalogRefreshDBVgInfo_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2781,7 +2779,7 @@ TEST(apiTest, catalogRefreshDBVgInfo_test) { TEST(apiTest, catalogChkAuth_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2800,15 +2798,19 @@ TEST(apiTest, catalogChkAuth_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - bool pass = false; + SUserAuthInfo authInfo = {0}; + SUserAuthRes authRes = {0}; + strcpy(authInfo.user, ctgTestUsername); + toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); + authInfo.type = AUTH_TYPE_READ; bool exists = false; - code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); ASSERT_EQ(code, 0); ASSERT_EQ(exists, false); - - code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass); + + code = catalogChkAuth(pCtg, mockPointer, &authInfo, &authRes); ASSERT_EQ(code, 0); - ASSERT_EQ(pass, true); + ASSERT_EQ(authRes.pass, true); while (true) { uint64_t n = 0; @@ -2820,9 +2822,9 @@ TEST(apiTest, catalogChkAuth_test) { } } - code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); ASSERT_EQ(code, 0); - ASSERT_EQ(pass, true); + ASSERT_EQ(authRes.pass, true); ASSERT_EQ(exists, true); catalogDestroy(); @@ -2830,7 +2832,7 @@ TEST(apiTest, catalogChkAuth_test) { TEST(apiTest, catalogRefreshGetTableCfg_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2868,7 +2870,7 @@ TEST(apiTest, catalogRefreshGetTableCfg_test) { TEST(apiTest, catalogGetTableIndex_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2904,7 +2906,7 @@ TEST(apiTest, catalogGetTableIndex_test) { TEST(apiTest, catalogGetDBCfg_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2937,7 +2939,7 @@ TEST(apiTest, catalogGetDBCfg_test) { TEST(apiTest, catalogGetQnodeList_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2962,17 +2964,16 @@ TEST(apiTest, catalogGetQnodeList_test) { ASSERT_EQ(taosArrayGetSize(qnodeList), ctgTestQnodeNum); for (int32_t i = 0; i < ctgTestQnodeNum; ++i) { - SQueryNodeLoad * pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i); + SQueryNodeLoad *pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i); ASSERT_EQ(pLoad->addr.nodeId, i); } catalogDestroy(); } - TEST(apiTest, catalogGetUdfInfo_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2999,10 +3000,9 @@ TEST(apiTest, catalogGetUdfInfo_test) { catalogDestroy(); } - TEST(apiTest, catalogGetServerVersion_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3021,7 +3021,7 @@ TEST(apiTest, catalogGetServerVersion_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - char* ver = NULL; + char *ver = NULL; code = catalogGetServerVersion(pCtg, mockPointer, &ver); ASSERT_EQ(code, 0); ASSERT_TRUE(0 == strcmp(ver, "1.0")); @@ -3031,7 +3031,7 @@ TEST(apiTest, catalogGetServerVersion_test) { TEST(apiTest, catalogUpdateTableIndex_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3061,10 +3061,9 @@ TEST(apiTest, catalogUpdateTableIndex_test) { catalogDestroy(); } - TEST(apiTest, catalogGetDnodeList_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3083,7 +3082,7 @@ TEST(apiTest, catalogGetDnodeList_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SArray* pList = NULL; + SArray *pList = NULL; code = catalogGetDnodeList(pCtg, mockPointer, &pList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize(pList), 1); @@ -3095,23 +3094,23 @@ TEST(apiTest, catalogGetDnodeList_test) { #ifdef INTEGRATION_TEST TEST(intTest, autoCreateTableTest) { - struct SCatalog *pCtg = NULL; + struct SCatalog *pCtg = NULL; - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_TRUE(NULL != taos); + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_TRUE(NULL != taos); ctgdEnableDebug("api", true); ctgdEnableDebug("meta", true); ctgdEnableDebug("cache", true); ctgdEnableDebug("lock", true); - + ctgTestExecQuery(taos, "drop database if exists db1", false, NULL); ctgTestExecQuery(taos, "create database db1", false, NULL); ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL); ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL); ctgdGetOneHandle(&pCtg); - + while (true) { uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { @@ -3137,15 +3136,14 @@ TEST(intTest, autoCreateTableTest) { ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL); ctgdEnableDebug("stopUpdate", false); - + ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL); - + taos_close(taos); } #endif - int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index c862a75ed3adaaff3ecb759db9c796f51b2473ce..58c43829cfcc8a4978345542d288e96c99793f35 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -307,7 +307,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) { // vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | // v4_status | cacheload | tsma | - if (nFields != 13) { + if (nFields != 14) { return false; } diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 220e9c0b6ced01edce3a36a4297f2c76151a4092..57a771b275b219b776dbee34193e2ba644e73a1f 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,7 +29,6 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; - TdThreadMutex mutex; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f99c7de93d95fa99de90a13eb2af8d527b876e0a..2e92f9e396eb2a0b76acfa1066c3829c22e71071 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -107,6 +107,8 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t uint64_t tableListGetSize(const STableListInfo* pTableList); uint64_t tableListGetSuid(const STableListInfo* pTableList); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); +int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0b55eb4a457fbad294478e5b490fb1600903333c..85424fd7de24021b4506966413d87f9abde6f159 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -115,24 +115,21 @@ typedef struct STaskIdInfo { uint64_t subplanId; uint64_t templateId; char* str; + int32_t vgId; } STaskIdInfo; enum { STREAM_RECOVER_STEP__NONE = 0, STREAM_RECOVER_STEP__PREPARE1, STREAM_RECOVER_STEP__PREPARE2, - STREAM_RECOVER_STEP__SCAN, + STREAM_RECOVER_STEP__SCAN1, + STREAM_RECOVER_STEP__SCAN2, }; typedef struct { - // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - // const SSubmitReq* pReq; - + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -141,10 +138,9 @@ typedef struct { SQueryTableDataCond tableCond; int64_t fillHistoryVer1; int64_t fillHistoryVer2; - - // int8_t triggerSaved; - // int64_t deleteMarkSaved; - SStreamState* pState; + SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; typedef struct { @@ -166,26 +162,25 @@ typedef struct STaskStopInfo { } STaskStopInfo; struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; - int32_t qbufQuota; // total available buffer (in KB) during execution query - - int64_t version; // used for stream to record wal version, why not move to sschemainfo + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query + int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; - STableListInfo* pTableInfoList; // this is a table list - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SSubplan* pSubplan; struct SOperatorInfo* pRoot; SLocalFetch localFetch; SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; + SRWLatch lock; // secure the access of STableListInfo }; enum { @@ -193,7 +188,6 @@ enum { OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, OP_EXEC_DONE = 0x9, - OP_EXEC_RECV = 0x11, }; typedef struct SOperatorFpSet { @@ -324,6 +318,8 @@ typedef struct STableScanBase { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SLimitInfo limitInfo; + // there are more than one table list exists in one task, if only one vnode exists. + STableListInfo* pTableListInfo; } STableScanBase; typedef struct STableScanInfo { @@ -337,6 +333,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; bool hasGroupByTag; + bool countOnly; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -363,11 +360,12 @@ typedef struct STableMergeScanInfo { } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - SColMatchInfo matchInfo; - int32_t curPos; - SReadHandle readHandle; + SColumnInfo* pCols; + SSDataBlock* pRes; + SColMatchInfo matchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableListInfo; } STagScanInfo; typedef enum EStreamScanMode { @@ -429,7 +427,6 @@ typedef struct STimeWindowAggSupp { } STimeWindowAggSupp; typedef struct SStreamScanInfo { - uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; @@ -483,16 +480,11 @@ typedef struct SStreamScanInfo { } SStreamScanInfo; typedef struct { - // int8_t subType; - // bool withMeta; - // int64_t suid; - // int64_t snapVersion; - // void *metaInfo; - // void *dataInfo; - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; + SVnode* vnode; + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + SSnapContext* sContext; + STableListInfo* pTableListInfo; } SStreamRawScanInfo; typedef struct STableCountScanSupp { @@ -505,16 +497,6 @@ typedef struct STableCountScanSupp { char stbNameFilter[TSDB_TABLE_NAME_LEN]; } STableCountScanSupp; -typedef struct STableCountScanOperatorInfo { - SReadHandle readHandle; - SSDataBlock* pRes; - - STableCountScanSupp supp; - - int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and/or stable_name -} STableCountScanOperatorInfo; - typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; @@ -557,6 +539,7 @@ typedef struct SStreamIntervalOperatorInfo { STimeWindowAggSupp twAggSup; bool invertible; bool ignoreExpiredData; + bool ignoreExpiredDataSaved; SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; @@ -572,6 +555,7 @@ typedef struct SStreamIntervalOperatorInfo { uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; + int64_t dataVersion; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -618,8 +602,10 @@ typedef struct SStreamSessionAggOperatorInfo { SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; + bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pStUpdated; + int64_t dataVersion; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -635,8 +621,10 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; + bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pSeUpdated; + int64_t dataVersion; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -688,6 +676,9 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) +SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, + char* dbFName); + SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); int32_t optrDummyOpenFn(SOperatorInfo* pOperator); @@ -732,8 +723,9 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int3 STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); -int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); +int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); extern void doDestroyExchangeOperatorInfo(void* param); @@ -749,16 +741,16 @@ void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, - bool isIntervalQuery, SAggSupporter* pSup); + bool isIntervalQuery, SAggSupporter* pSup, bool keepGroup); // operator creater functions // clang-format off SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); @@ -774,7 +766,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); @@ -788,9 +780,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); @@ -833,9 +825,13 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); void doDestroyTask(SExecTaskInfo* pTaskInfo); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); + +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); + +int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, @@ -849,7 +845,6 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index fad4059515f386ca94ce2c97985184ff4b95f744..5fd75f9b99d004be68b369172668697bfa6fa9a6 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -30,6 +30,7 @@ extern "C" { struct SSDataBlock; typedef struct SFillColInfo { + int32_t numOfFillExpr; SExprInfo* pExpr; bool notFillCol; // denote if this column needs fill operation SVariant fillVal; @@ -92,8 +93,8 @@ typedef struct SResultRowData { typedef struct SStreamFillLinearInfo { TSKEY nextEnd; - SArray* pDeltaVal; // double. value for Fill(linear). - SArray* pNextDeltaVal; // double. value for Fill(linear). + SArray* pEndPoints; + SArray* pNextEndPoints; int64_t winIndex; bool hasNext; } SStreamFillLinearInfo; diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index cff568aebc0c26fb2e9c22c7a027fa9edb4dd81a..d51a24bb43253e09d6ab21e6a1785d6ef6b5e5ff 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -44,7 +44,8 @@ typedef struct SSortSource { void* param; bool onlyRef; }; - + int64_t fetchUs; + int64_t fetchNum; } SSortSource; typedef struct SMsortComparParam { diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c new file mode 100644 index 0000000000000000000000000000000000000000..ec8060348d4b5dd465be4e6356c73994aa54df7c --- /dev/null +++ b/source/libs/executor/src/aggregateoperator.c @@ -0,0 +1,573 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef struct { + bool hasAgg; + int32_t numOfRows; + int32_t startOffset; +} SFunctionCtxStatus; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + bool groupKeyOptimized; +} SAggOperatorInfo; + +static void destroyAggOperatorInfo(void* param); +static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); + +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock); +static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock); + +static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator); +static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx); +static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator); + +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey); + +static int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size); + +static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); + +static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus); +static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus); + +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, + SExecTaskInfo* pTaskInfo) { + SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(&pOperator->resultInfo, 4096); + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); + int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + int32_t numOfScalarExpr = 0; + SExprInfo* pScalarExprInfo = NULL; + if (pAggNode->pExprs != NULL) { + pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); + } + + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; + pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; + pInfo->groupId = UINT64_MAX; + + setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, + optrDefaultBufFn, NULL); + + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pTableScanInfo = downstream->info; + pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp; + pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup; + } + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + if (pInfo != NULL) { + destroyAggOperatorInfo(pInfo); + } + + if (pOperator != NULL) { + cleanupExprSupp(&pOperator->exprSupp); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyAggOperatorInfo(void* param) { + SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + + cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarExprSup); + cleanupGroupResInfo(&pInfo->groupResInfo); + taosMemoryFreeClear(param); +} + +// this is a blocking operator +int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { + if (OPTR_IS_OPENED(pOperator)) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SAggOperatorInfo* pAggInfo = pOperator->info; + + SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + int64_t st = taosGetTimestampUs(); + + int32_t order = TSDB_ORDER_ASC; + int32_t scanFlag = MAIN_SCAN; + + bool hasValidBlock = false; + + while (1) { + bool blockAllocated = false; + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + if (!hasValidBlock) { + createDataBlockForEmptyInput(pOperator, &pBlock); + if (pBlock == NULL) { + break; + } + blockAllocated = true; + } else { + break; + } + } + hasValidBlock = true; + + int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); + if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + + // there is an scalar expression that needs to be calculated before apply the group aggregation. + if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { + SExprSupp* pSup1 = &pAggInfo->scalarExprSup; + code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); + if (code != TSDB_CODE_SUCCESS) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + } + + // the pDataBlock are always the same one, no need to call this again + setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); + setInputDataBlock(pSup, pBlock, order, scanFlag, true); + code = doAggregateImpl(pOperator, pSup->pCtx); + if (code != 0) { + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + T_LONG_JMP(pTaskInfo->env, code); + } + + destroyDataBlockForEmptyInput(blockAllocated, &pBlock); + } + + // the downstream operator may return with error code, so let's check the code before generating results. + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); + OPTR_SET_OPENED(pOperator); + + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + return pTaskInfo->code; +} + +SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + setOperatorCompleted(pOperator); + return NULL; + } + + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + while (1) { + doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + + if (!hasRemainResults(&pAggInfo->groupResInfo)) { + setOperatorCompleted(pOperator); + break; + } + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + + size_t rows = blockDataGetNumOfRows(pInfo->pRes); + pOperator->resultInfo.totalRows += rows; + + return (rows == 0) ? NULL : pInfo->pRes; +} + +int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) { + for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) { + if (functionNeedToExecute(&pCtx[k])) { + // todo add a dummy funtion to avoid process check + if (pCtx[k].fpSet.process == NULL) { + continue; + } + + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); + return code; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { + if (!tsCountAlwaysReturnValue) { + return TSDB_CODE_SUCCESS; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + if (pAggInfo->groupKeyOptimized) { + return TSDB_CODE_SUCCESS; + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || + (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && + ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { + return TSDB_CODE_SUCCESS; + } + + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + bool hasCountFunc = false; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; + if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || + (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { + hasCountFunc = true; + break; + } + } + + if (!hasCountFunc) { + return TSDB_CODE_SUCCESS; + } + + SSDataBlock* pBlock = createDataBlock(); + pBlock->info.rows = 1; + pBlock->info.capacity = 0; + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.hasNull = true; + colInfo.info.type = TSDB_DATA_TYPE_NULL; + colInfo.info.bytes = 1; + + SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; + for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { + SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; + if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { + int32_t slotId = pFuncParam->pCol->slotId; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + if (slotId >= numOfCols) { + taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); + for (int32_t k = numOfCols; k < slotId + 1; ++k) { + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + } + } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { + // do nothing + } + } + } + + blockDataEnsureCapacity(pBlock, pBlock->info.rows); + *ppBlock = pBlock; + + return TSDB_CODE_SUCCESS; +} + +void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) { + if (!blockAllocated) { + return; + } + + blockDataDestroy(*ppBlock); + *ppBlock = NULL; +} + +void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { + SAggOperatorInfo* pAggInfo = pOperator->info; + if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) { + return; + } + + doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId); + + // record the current active group id + pAggInfo->groupId = groupId; +} + +void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { + // for simple group by query without interval, all the tables belong to one group result. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SAggOperatorInfo* pAggInfo = pOperator->info; + + SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; + int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; + + SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, + sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true); + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pResultRow->pageId == -1) { + int32_t ret = addNewResultRowBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); + if (ret != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + } + + setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); +} + +// a new buffer page for each table. Needs to opt this design +int32_t addNewResultRowBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { + if (pWindowRes->pageId != -1) { + return 0; + } + + SFilePage* pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SArray* list = getDataBufPagesIdList(pResultBuf); + + if (taosArrayGetSize(list) == 0) { + pData = getNewBufPage(pResultBuf, &pageId); + pData->num = sizeof(SFilePage); + } else { + SPageInfo* pi = getLastPageInfo(list); + pData = getBufPage(pResultBuf, getPageId(pi)); + if (pData == NULL) { + qError("failed to get buffer, code:%s", tstrerror(terrno)); + return terrno; + } + + pageId = getPageId(pi); + + if (pData->num + size > getBufPageSize(pResultBuf)) { + // release current page first, and prepare the next one + releaseBufPageInfo(pResultBuf, pi); + + pData = getNewBufPage(pResultBuf, &pageId); + if (pData != NULL) { + pData->num = sizeof(SFilePage); + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->offset = (int32_t)pData->num; + + pData->num += size; + } + + return 0; +} + +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey) { + int32_t code = 0; + // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + + pAggSup->currentPageId = -1; + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); + pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); + pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash); + + if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + uint32_t defaultPgsz = 0; + uint32_t defaultBufsz = 0; + getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); + + if (!osTempSpaceAvailable()) { + code = TSDB_CODE_NO_AVAIL_DISK; + qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey); + return code; + } + + code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); + if (code != TSDB_CODE_SUCCESS) { + qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey); + return code; + } + + return code; +} + +void cleanupAggSup(SAggSupporter* pAggSup) { + taosMemoryFreeClear(pAggSup->keyBuf); + tSimpleHashCleanup(pAggSup->pResultRowHashTable); + destroyDiskbasedBuf(pAggSup->pResultBuf); +} + +int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, + const char* pkey, void* pState) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + if (pState) { + pSup->pCtx[i].saveHandle.pBuf = NULL; + pSup->pCtx[i].saveHandle.pState = pState; + pSup->pCtx[i].exprIdx = i; + } else { + pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; + } + } + + return TSDB_CODE_SUCCESS; +} + +void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, + int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) { + for (int32_t k = 0; k < numOfOutput; ++k) { + // keep it temporarily + SFunctionCtxStatus status = {0}; + functionCtxSave(&pCtx[k], &status); + + pCtx[k].input.startRowIndex = offset; + pCtx[k].input.numOfRows = forwardStep; + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) { + pCtx[k].input.colDataSMAIsSet = false; + } + + if (pCtx[k].isPseudoFunc) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); + + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + + SColumnInfoData idata = {0}; + idata.info.type = TSDB_DATA_TYPE_BIGINT; + idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + idata.pData = p; + + SScalarParam out = {.columnData = &idata}; + SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; + pCtx[k].sfp.process(&tw, 1, &out); + pEntryInfo->numOfRes = 1; + } else { + int32_t code = TSDB_CODE_SUCCESS; + if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { + code = pCtx[k].fpSet.process(&pCtx[k]); + + if (code != TSDB_CODE_SUCCESS) { + qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + T_LONG_JMP(taskInfo->env, code); + } + } + + // restore it + functionCtxRestore(&pCtx[k], &status); + } + } +} + +void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { + pStatus->hasAgg = pCtx->input.colDataSMAIsSet; + pStatus->numOfRows = pCtx->input.numOfRows; + pStatus->startOffset = pCtx->input.startRowIndex; +} + +void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { + pCtx->input.colDataSMAIsSet = pStatus->hasAgg; + pCtx->input.numOfRows = pStatus->numOfRows; + pCtx->input.startRowIndex = pStatus->startOffset; +} diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index d42b348fd8c911b3915ad381f1270a290796f5aa..61f024ebb70bcbbf2b2127b41d3ac5f22ec74693 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -37,6 +37,7 @@ typedef struct SCacheRowsScanInfo { SSDataBlock* pBufferredRes; SArray* pUidList; int32_t indexOfBufferedRes; + STableListInfo* pTableList; } SCacheRowsScanInfo; static SSDataBlock* doScanCache(SOperatorInfo* pOperator); @@ -44,18 +45,20 @@ static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo); -#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) +#define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_SUCCESS; + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } + pInfo->pTableList = pTableListInfo; pInfo->readHandle = *readHandle; SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc; @@ -75,22 +78,21 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe goto _error; } - STableListInfo* pTableList = pTaskInfo->pTableInfoList; - - int32_t totalTables = tableListGetSize(pTableList); + int32_t totalTables = tableListGetSize(pTableListInfo); int32_t capacity = 0; pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); // partition by tbname - if (oneTableForEachGroup(pTableList) || (totalTables == 1)) { + if (oneTableForEachGroup(pTableListInfo) || (totalTables == 1)) { pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull); - STableKeyInfo* pList = tableListGetInfo(pTableList, 0); + STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); - uint64_t suid = tableListGetSuid(pTableList); + uint64_t suid = tableListGetSuid(pTableListInfo); code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str); + taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -113,7 +115,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = @@ -122,7 +125,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->cost.openCost = 0; return pOperator; - _error: +_error: pTaskInfo->code = code; destroyCacheScanOperator(pInfo); taosMemoryFree(pOperator); @@ -135,8 +138,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } SCacheRowsScanInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableListInfo* pTableList = pTaskInfo->pTableInfoList; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableListInfo* pTableList = pInfo->pTableList; uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); @@ -193,8 +196,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pRes->info.rows = 1; SExprSupp* pSup = &pInfo->pseudoExprSup; - int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, - pRes->info.rows, GET_TASKID(pTaskInfo), NULL); + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, + pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -216,7 +219,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } STableKeyInfo* pList = NULL; - int32_t num = 0; + int32_t num = 0; int32_t code = tableListGetGroupList(pTableList, pInfo->currentGroupIndex, &pList, &num); if (code != TSDB_CODE_SUCCESS) { @@ -250,11 +253,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = pKeyInfo->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { - ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); - pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); - code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo), NULL); + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -281,6 +282,7 @@ void destroyCacheScanOperator(void* param) { taosMemoryFree(pInfo->pSlotIds); taosArrayDestroy(pInfo->pUidList); taosArrayDestroy(pInfo->matchInfo.pList); + tableListDestroy(pInfo->pTableList); if (pInfo->pLastrowReader != NULL) { pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); @@ -323,7 +325,7 @@ int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pC return TSDB_CODE_SUCCESS; } - size_t size = taosArrayGetSize(pColMatchInfo->pList); + size_t size = taosArrayGetSize(pColMatchInfo->pList); SArray* pMatchInfo = taosArrayInit(size, sizeof(SColMatchItem)); for (int32_t i = 0; i < size; ++i) { diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index dc3bf83a914eca1a93304b1487b7acdeca59c1ba..d693faf7f14e90399c4c2393f48c8da26ac1ff39 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -240,6 +240,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { + taosMemoryFree(pParam); code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index e9c46843c02cbe9c736dda90a0cf66fa22a780b4..90d740bebd8658ced2346292f527e9ecf1ade2cf 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -408,6 +408,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { + taosMemoryFree(pParam); terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 7fbde07f596989c3a27ab5551c2b0215c13d4a59..b3cb57325ba5dee1d2d2f3b0d07304a1d1ea5ae2 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -23,7 +23,6 @@ SDataSinkStat gDataSinkStat = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) { gDataSinkManager.cfg = *cfg; - taosThreadMutexInit(&gDataSinkManager.mutex, NULL); return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index e5089ab4a9b6a6486d86bebbf82bf54c4abd543e..c855a104b2cccdb7ed28fe36e5e9c740ec6f5bd9 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -70,7 +70,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } for (int32_t i = 0; i < totalSources; ++i) { @@ -212,6 +212,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { return NULL; } + doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); + if (blockDataGetNumOfRows(pBlock) == 0) { + continue; + } + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -303,6 +308,11 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + code = filterInitFromNode((SNode*)pExNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -512,8 +522,6 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo blockDataDestroy(pBlock); } - // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. - blockDataUpdateTsWindow(pRes, 0); return TSDB_CODE_SUCCESS; } @@ -568,7 +576,7 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } return TSDB_CODE_SUCCESS; @@ -621,7 +629,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0ac458af495c8f4c3a3f4fd1ce7d982e5459aaf4..f61fd1ae0159c8219a7070d56f9874c10190f329 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -17,6 +17,7 @@ #include "functionMgt.h" #include "index.h" #include "os.h" +#include "query.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" @@ -26,15 +27,21 @@ #include "executorimpl.h" #include "tcompression.h" +typedef struct STableListIdInfo { + uint64_t suid; + uint64_t uid; + int32_t tableType; +} STableListIdInfo; + // If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups struct STableListInfo { - bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - uint64_t suid; + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + STableListIdInfo idInfo; // this maybe the super table or ordinary table }; typedef struct tagFilterAssist { @@ -469,7 +476,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis } // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -570,6 +577,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis memcpy(pStart, data, len); pStart += len; } else if (IS_VAR_DATA_TYPE(pValue->info.type)) { + if (varDataTLen(data) > pValue->info.bytes) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto end; + } memcpy(pStart, data, varDataTLen(data)); pStart += varDataTLen(data); } else { @@ -820,6 +831,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables); if (code != TSDB_CODE_SUCCESS) { terrno = code; + taosMemoryFree(pResBlock); return NULL; } @@ -947,7 +959,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -960,12 +972,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); + qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); terrno = code; goto end; } @@ -1019,15 +1031,17 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; - pListInfo->suid = pScanNode->suid; + pListInfo->idInfo.suid = pScanNode->suid; + pListInfo->idInfo.tableType = pScanNode->tableType; + SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { + pListInfo->idInfo.uid = pScanNode->uid; if (metaIsTableExist(metaHandle, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -1057,10 +1071,10 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; - SIdxFltStatus status = SFLT_NOT_INDEX; + status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake - // qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); + qWarn("failed to get tableIds from index, suid:%" PRIu64, pScanNode->uid); code = TDB_CODE_SUCCESS; } else { qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); @@ -1788,7 +1802,9 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; } +uint64_t tableListGetSuid(const STableListInfo* pTableList) { + return pTableList->idInfo.suid; +} STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -1798,6 +1814,27 @@ STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) return taosArrayGet(pTableList->pTableList, index); } +int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex) { + int32_t numOfTables = taosArrayGetSize(pTableList->pTableList); + if (startIndex >= numOfTables) { + return -1; + } + + for (int32_t i = startIndex; i < numOfTables; ++i) { + STableKeyInfo* p = taosArrayGet(pTableList->pTableList, i); + if (p->uid == uid) { + return i; + } + } + return -1; +} + +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type) { + *psuid = pTableList->idInfo.suid; + *uid = pTableList->idInfo.uid; + *type = pTableList->idInfo.tableType; +} + uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); ASSERT(pTableList->map != NULL && slot != NULL); @@ -2040,7 +2077,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags void printDataBlock(SSDataBlock* pBlock, const char* flag) { if (!pBlock || pBlock->info.rows == 0) { - qDebug("===stream===printDataBlock: Block is Null or Empty"); + qDebug("===stream===%s: Block is Null or Empty", flag); return; } char* pBuf = NULL; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 04d54a95aed736f9d299bac3d5ea33c457dd14d5..1732ec04a77f54d10ab57a31aae663ea5e47e9d2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,7 @@ */ #include "executor.h" +#include #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" @@ -29,8 +30,8 @@ static void cleanupRefPool() { taosCloseRef(ref); } -static void initRefPool() { - exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); +static void initRefPool() { + exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); atexit(cleanupRefPool); } @@ -127,12 +128,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("stream set total blocks:%d, task id:%s" PRIx64, (int32_t)numOfBlocks, id); - ASSERT(pInfo->validBlockIndex == 0); - ASSERT(taosArrayGetSize(pInfo->pBlockLists) == 0); + qDebug("s-task set source blocks:%d %s", (int32_t)numOfBlocks, id); + ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { - // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); @@ -159,6 +158,28 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +void doSetTaskId(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pStreamScanInfo = pOperator->info; + STableScanInfo* pScanInfo = pStreamScanInfo->pTableScanOp->info; + if (pScanInfo->base.dataReader != NULL) { + tsdbReaderSetId(pScanInfo->base.dataReader, pTaskInfo->id.str); + } + } else { + doSetTaskId(pOperator->pDownstream[0]); + } +} + +void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { + SExecTaskInfo* pTaskInfo = tinfo; + pTaskInfo->id.queryId = queryId; + buildTaskId(taskId, queryId, pTaskInfo->id.str); + + // set the idstr for tsdbReader + doSetTaskId(pTaskInfo->pRoot); +} + int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; @@ -176,6 +197,13 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { return code; } +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId) { + SExecTaskInfo* pTaskInfo = tinfo; + *dataVer = pTaskInfo->streamInfo.dataVersion; + *ckId = pTaskInfo->streamInfo.checkPointId; +} + + int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; @@ -218,37 +246,35 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, return code; } -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) { - if (msg == NULL) { - // create raw scan - - SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, + uint64_t id) { + if (msg == NULL) { // create raw scan + SExecTaskInfo* pTaskInfo = doCreateExecTaskInfo(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, ""); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - - pTaskInfo->cost.created = taosGetTimestampUs(); - pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; - pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo); + pTaskInfo->pRoot = createRawScanOperatorInfo(pReaderHandle, pTaskInfo); if (NULL == pTaskInfo->pRoot) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pTaskInfo); return NULL; } + + qDebug("create raw scan task info completed, vgId:%d, %s", vgId, GET_TASKID(pTaskInfo)); return pTaskInfo; } struct SSubplan* pPlan = NULL; - int32_t code = qStringToSubplan(msg, &pPlan); + + int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, 0, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE); + code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE); if (code != TSDB_CODE_SUCCESS) { nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); @@ -268,19 +294,14 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } } - if (pSchema) { - *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); - } return pTaskInfo; } -qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t vgId) { if (msg == NULL) { return NULL; } - /*qDebugL("stream task string %s", (const char*)msg);*/ - struct SSubplan* pPlan = NULL; int32_t code = qStringToSubplan(msg, &pPlan); if (code != TSDB_CODE_SUCCESS) { @@ -289,7 +310,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, 0, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); @@ -307,6 +328,13 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + + uint64_t suid = 0; + uint64_t uid = 0; + int32_t type = 0; + tableListGetSourceTableInfo(pTableScanInfo->base.pTableListInfo, &suid, &uid, &type); + // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); @@ -321,9 +349,21 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S tDecoderClear(&mr.coder); - // TODO handle ntb case - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { + if (mr.me.type == TSDB_SUPER_TABLE) { continue; + } else { + if (type == TSDB_SUPER_TABLE) { + // this new created child table does not belong to the scanned super table. + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != suid) { + continue; + } + } else { // ordinary table + // In case that the scanned target table is an ordinary table. When replay the WAL during restore the vnode, we + // should check all newly created ordinary table to make sure that this table isn't the destination table. + if (mr.me.uid != uid) { + continue; + } + } } if (pScanInfo->pTagCond != NULL) { @@ -348,27 +388,23 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); + int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); + qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id - SOperatorInfo* pInfo = pTaskInfo->pRoot; - while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - pInfo = pInfo->pDownstream[0]; - } - - int32_t code = 0; + SOperatorInfo* pInfo = extractOperatorInTree(pTaskInfo->pRoot, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); SStreamScanInfo* pScanInfo = pInfo->info; + if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); int32_t numOfQualifiedTables = taosArrayGetSize(qa); - - qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables); - + qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); @@ -387,7 +423,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + STableListInfo* pTableListInfo = ((STableScanInfo*)pScanInfo->pTableScanOp->info)->base.pTableListInfo; + taosWLockLatch(&pTaskInfo->lock); for (int32_t i = 0; i < numOfQualifiedTables; ++i) { uint64_t* uid = taosArrayGet(qa, i); @@ -402,35 +439,26 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(keyBuf); taosArrayDestroy(qa); + taosWUnLockLatch(&pTaskInfo->lock); return code; } } } -#if 0 - bool exists = false; - for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) { - STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k); - if (pKeyInfo->uid == keyInfo.uid) { - qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str); - exists = true; - } - } - - if (!exists) { -#endif - tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId); } + taosWUnLockLatch(&pTaskInfo->lock); if (keyBuf != NULL) { taosMemoryFree(keyBuf); } taosArrayDestroy(qa); } else { // remove the table id in current list - qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); + qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); + taosWLockLatch(&pTaskInfo->lock); code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); + taosWUnLockLatch(&pTaskInfo->lock); } return code; @@ -463,16 +491,14 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { - assert(pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - taosThreadOnce(&initPoolOnce, initRefPool); - qDebug("start to create subplan task, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); + qDebug("start to create task, TID:0x%" PRIx64 " QID:0x%" PRIx64 ", vgId:%d", taskId, pSubplan->id.queryId, vgId); - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); + int32_t code = createExecTaskInfo(pSubplan, pTask, readHandle, taskId, vgId, sql, model); if (code != TSDB_CODE_SUCCESS) { - qError("failed to createExecTaskInfoImpl, code: %s", tstrerror(code)); + qError("failed to createExecTaskInfo, code: %s", tstrerror(code)); goto _error; } @@ -485,16 +511,14 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, if (handle) { void* pSinkParam = NULL; - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, (*pTask), readHandle); if (code != TSDB_CODE_SUCCESS) { qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str); goto _error; } + // pSinkParam has been freed during create sinker. code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pSinkParam); - } } qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); @@ -715,7 +739,6 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - if (pTaskInfo == NULL) { return TSDB_CODE_QRY_INVALID_QHANDLE; } @@ -723,12 +746,28 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); setTaskKilled(pTaskInfo, rspCode); - qStopTaskOperators(pTaskInfo); return TSDB_CODE_SUCCESS; } +int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED); + + while (qTaskIsExecuting(pTaskInfo)) { + taosMsleep(10); + } + + pTaskInfo->code = rspCode; + return TSDB_CODE_SUCCESS; +} + bool qTaskIsExecuting(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; if (NULL == pTaskInfo) { @@ -868,7 +907,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; - + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { @@ -884,6 +924,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || @@ -897,6 +939,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; + pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData; + pInfo->ignoreExpiredData = false; } // iterate operator tree @@ -924,35 +968,23 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/ - /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/ - pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/ - /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/ + pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/ - /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/ - pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/ - /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/ + pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE);*/ - /*ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX);*/ - pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; - /*ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE ||*/ - /*pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);*/ + pInfo->ignoreExpiredData = pInfo->ignoreExpiredDataSaved; qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } @@ -970,6 +1002,7 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { } return 0; } + bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; return pTaskInfo->streamInfo.recoverScanFinished; @@ -992,21 +1025,12 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) { SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); return &pTaskInfo->streamInfo.metaRsp; } -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) { +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - return pTaskInfo->streamInfo.prepareStatus.uid; -} - -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); - return 0; + memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal)); } int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { @@ -1040,53 +1064,52 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -#if 0 -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - ASSERT(pTaskInfo->streamInfo.pReq == NULL); - pTaskInfo->streamInfo.pReq = pReq; - pTaskInfo->streamInfo.scanVer = scanVer; - return 0; -} -#endif - int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT((pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE )&& (pTaskInfo->streamInfo.submit.msgStr == NULL)); + if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) { + qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } qDebug("set the submit block for future scan"); pTaskInfo->streamInfo.submit = submit; return 0; } +void qStreamSetOpen(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + pOperator->status = OP_NOT_OPENED; +} + int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - pTaskInfo->streamInfo.prepareStatus = *pOffset; - pTaskInfo->streamInfo.returned = 0; + const char* id = GET_TASKID(pTaskInfo); - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { + // if pOffset equal to current offset, means continue consume + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } if (subType == TOPIC_SUB_TYPE__COLUMN) { - pOperator->status = OP_OPENED; - - // TODO add more check - if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; + pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); + if (pOperator == NULL) { + return -1; } - SStreamScanInfo* pInfo = pOperator->info; + STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; + STableScanBase* pScanBaseInfo = &pScanInfo->base; + STableListInfo* pTableListInfo = pScanBaseInfo->pTableListInfo; + if (pOffset->type == TMQ_OFFSET__LOG) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; + tsdbReaderClose(pScanBaseInfo->dataReader); + pScanBaseInfo->dataReader = NULL; + // let's seek to the next version in wal file - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { + qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1094,125 +1117,151 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // those data are from the snapshot in tsdb, besides the data in the wal file. int64_t uid = pOffset->uid; int64_t ts = pOffset->ts; + int32_t index = 0; + + // this value may be changed if new tables are created + taosRLockLatch(&pTaskInfo->lock); + int32_t numOfTables = tableListGetSize(pTableListInfo); if (uid == 0) { - if (tableListGetSize(pTaskInfo->pTableInfoList) != 0) { - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, 0); + if (numOfTables != 0) { + STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; + pScanInfo->currentTable = 0; } else { + taosRUnLockLatch(&pTaskInfo->lock); + qError("no table in table list, %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } } - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); - -#ifndef NDEBUG - qDebug("switch to next table %" PRId64 " (cursor %d), %" PRId64 " rows returned", uid, - pTableScanInfo->currentTable, pInfo->pTableScanOp->resultInfo.totalRows); + qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% " PRId64 " rows returned", uid, ts, + pInfo->pTableScanOp->resultInfo.totalRows); pInfo->pTableScanOp->resultInfo.totalRows = 0; -#endif - bool found = false; - for (int32_t i = 0; i < numOfTables; i++) { - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i); - if (pTableInfo->uid == uid) { - found = true; - pTableScanInfo->currentTable = i; - break; - } + // start from current accessed position + // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start + // position, let's find it from the beginning. + index = tableListFind(pTableListInfo, uid, 0); + taosRUnLockLatch(&pTaskInfo->lock); + + if (index >= 0) { + pScanInfo->currentTable = index; + } else { + qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, + numOfTables, pScanInfo->currentTable, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; } - // TODO after dropping table, table may not found - ASSERT(found); + STableKeyInfo keyInfo = {.uid = uid}; + int64_t oldSkey = pScanBaseInfo->cond.twindows.skey; - if (pTableScanInfo->base.dataReader == NULL) { - STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); + // let's start from the next ts that returned to consumer. + pScanBaseInfo->cond.twindows.skey = ts + 1; + pScanInfo->scanTimes = 0; - if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || - pTableScanInfo->base.dataReader == NULL) { - ASSERT(0); + if (pScanBaseInfo->dataReader == NULL) { + int32_t code = tsdbReaderOpen(pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, + pScanInfo->pResBlock, &pScanBaseInfo->dataReader, id, false); + if (code != TSDB_CODE_SUCCESS) { + qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id); + terrno = code; + return -1; } - } - STableKeyInfo tki = {.uid = uid}; - tsdbSetTableList(pTableScanInfo->base.dataReader, &tki, 1); - int64_t oldSkey = pTableScanInfo->base.cond.twindows.skey; - pTableScanInfo->base.cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); - pTableScanInfo->base.cond.twindows.skey = oldSkey; - pTableScanInfo->scanTimes = 0; + qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + } else { + tsdbSetTableList(pScanBaseInfo->dataReader, &keyInfo, 1); + tsdbReaderReset(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); + qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); + } - qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, - ts, pTableScanInfo->currentTable, numOfTables); + // restore the key value + pScanBaseInfo->cond.twindows.skey = oldSkey; } else { - ASSERT(0); - } - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - SStreamRawScanInfo* pInfo = pOperator->info; - SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid); + qError("invalid pOffset->type:%d, %s", pOffset->type, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } - SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); - tsdbReaderClose(pInfo->dataReader); - pInfo->dataReader = NULL; + } else { // subType == TOPIC_SUB_TYPE__TABLE/TOPIC_SUB_TYPE__DB - cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); - tableListClear(pTaskInfo->pTableInfoList); + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + SStreamRawScanInfo* pInfo = pOperator->info; + SSnapContext* sContext = pInfo->sContext; - if (mtInfo.uid == 0) { - return 0; // no data - } + SOperatorInfo* p = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, id); + STableListInfo* pTableListInfo = ((SStreamRawScanInfo*)(p->info))->pTableListInfo; - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); - pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; + if (setForSnapShot(sContext, pOffset->uid) != 0) { + qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } - if (pTaskInfo->pTableInfoList == NULL) { - pTaskInfo->pTableInfoList = tableListCreate(); - } + SMetaTableInfo mtInfo = getUidfromSnapShot(sContext); + tsdbReaderClose(pInfo->dataReader); + pInfo->dataReader = NULL; - tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0); + cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); + tableListClear(pTableListInfo); - STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); - int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); - ASSERT(size == 1); + if (mtInfo.uid == 0) { + goto end; // no data + } - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); + pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; - cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); - strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); - tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); - pTaskInfo->streamInfo.schema = mtInfo.schema; + tableListAddTableInfo(pTableListInfo, mtInfo.uid, 0); - qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64, mtInfo.uid, pOffset->ts); - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { - SStreamRawScanInfo* pInfo = pOperator->info; - SSnapContext* sContext = pInfo->sContext; - if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); - return -1; + STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); + int32_t size = tableListGetSize(pTableListInfo); + + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, + false); + + cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); + strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); + tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); + pTaskInfo->streamInfo.schema = mtInfo.schema; + + qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64 " %s", mtInfo.uid, pOffset->ts, id); + } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { + SStreamRawScanInfo* pInfo = pOperator->info; + SSnapContext* sContext = pInfo->sContext; + if (setForSnapShot(sContext, pOffset->uid) != 0) { + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return -1; + } + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts, + id); + } else if (pOffset->type == TMQ_OFFSET__LOG) { + SStreamRawScanInfo* pInfo = pOperator->info; + tsdbReaderClose(pInfo->dataReader); + pInfo->dataReader = NULL; + qDebug("tmqsnap qStreamPrepareScan snapshot log, %s", id); } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64, pOffset->uid, pOffset->ts); - } else if (pOffset->type == TMQ_OFFSET__LOG) { - SStreamRawScanInfo* pInfo = pOperator->info; - tsdbReaderClose(pInfo->dataReader); - pInfo->dataReader = NULL; - qDebug("tmqsnap qStreamPrepareScan snapshot log"); } + +end: + pTaskInfo->streamInfo.currentOffset = *pOffset; + return 0; } void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); + if (pMsg->info.ahandle == NULL) { + qError("pMsg->info.ahandle is NULL"); + return; + } SDataBuf buf = {.len = pMsg->contLen, .pData = NULL}; @@ -1231,3 +1280,21 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { destroySendMsgInfo(pSendInfo); } +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = tinfo; + SArray* plist = getTableListInfo(pTaskInfo); + + // only extract table in the first elements + STableListInfo* pTableListInfo = taosArrayGetP(plist, 0); + + SArray* pUidList = taosArrayInit(10, sizeof(uint64_t)); + + int32_t numOfTables = tableListGetSize(pTableListInfo); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); + taosArrayPush(pUidList, &pKeyInfo->uid); + } + + taosArrayDestroy(plist); + return pUidList; +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5a2cdffa9295ebd7db036054b2638c91f5ae10ff..7594079cfb03fbe7b8a3a06b21aa6b8b625deb23 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -34,9 +34,7 @@ #include "ttypes.h" #include "vnode.h" -#define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) -#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) - +#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #if 0 @@ -73,35 +71,22 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define realloc u_realloc #endif -#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - STableQueryInfo* current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - bool groupKeyOptimized; -} SAggOperatorInfo; +#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); -static void destroyAggOperatorInfo(void* param); -static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); -static void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); -static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, - const char* pKey); +static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); +static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); + static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep, int32_t status); static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo); +static SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; @@ -195,8 +180,11 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i */ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, - bool isIntervalQuery, SAggSupporter* pSup) { + bool isIntervalQuery, SAggSupporter* pSup, bool keepGroup) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); + if (!keepGroup) { + *(uint64_t*)pSup->keyBuf = calcGroupId(pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + } SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); @@ -263,57 +251,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR return pResult; } -// a new buffer page for each table. Needs to opt this design -static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pResultBuf, uint32_t size) { - if (pWindowRes->pageId != -1) { - return 0; - } - - SFilePage* pData = NULL; - - // in the first scan, new space needed for results - int32_t pageId = -1; - SArray* list = getDataBufPagesIdList(pResultBuf); - - if (taosArrayGetSize(list) == 0) { - pData = getNewBufPage(pResultBuf, &pageId); - pData->num = sizeof(SFilePage); - } else { - SPageInfo* pi = getLastPageInfo(list); - pData = getBufPage(pResultBuf, getPageId(pi)); - if (pData == NULL) { - qError("failed to get buffer, code:%s", tstrerror(terrno)); - return terrno; - } - - pageId = getPageId(pi); - - if (pData->num + size > getBufPageSize(pResultBuf)) { - // release current page first, and prepare the next one - releaseBufPageInfo(pResultBuf, pi); - - pData = getNewBufPage(pResultBuf, &pageId); - if (pData != NULL) { - pData->num = sizeof(SFilePage); - } - } - } - - if (pData == NULL) { - return -1; - } - - // set the number of rows in current disk page - if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer - pWindowRes->pageId = pageId; - pWindowRes->offset = (int32_t)pData->num; - - pData->num += size; - } - - return 0; -} - // query_range_start, query_range_end, window_duration, window_start, window_end void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; @@ -329,72 +266,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataSetInt64(pColData, 4, &pQueryWindow->ekey); } -typedef struct { - bool hasAgg; - int32_t numOfRows; - int32_t startOffset; -} SFunctionCtxStatus; - -static void functionCtxSave(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { - pStatus->hasAgg = pCtx->input.colDataSMAIsSet; - pStatus->numOfRows = pCtx->input.numOfRows; - pStatus->startOffset = pCtx->input.startRowIndex; -} - -static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { - pCtx->input.colDataSMAIsSet = pStatus->hasAgg; - pCtx->input.numOfRows = pStatus->numOfRows; - pCtx->input.startRowIndex = pStatus->startOffset; -} - -void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, - int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) { - for (int32_t k = 0; k < numOfOutput; ++k) { - // keep it temporarily - SFunctionCtxStatus status = {0}; - functionCtxSave(&pCtx[k], &status); - - pCtx[k].input.startRowIndex = offset; - pCtx[k].input.numOfRows = forwardStep; - - // not a whole block involved in query processing, statistics data can not be used - // NOTE: the original value of isSet have been changed here - if (pCtx[k].input.colDataSMAIsSet && forwardStep < numOfTotal) { - pCtx[k].input.colDataSMAIsSet = false; - } - - if (pCtx[k].isPseudoFunc) { - SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); - - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); - - SColumnInfoData idata = {0}; - idata.info.type = TSDB_DATA_TYPE_BIGINT; - idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - idata.pData = p; - - SScalarParam out = {.columnData = &idata}; - SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; - pCtx[k].sfp.process(&tw, 1, &out); - pEntryInfo->numOfRes = 1; - } else { - int32_t code = TSDB_CODE_SUCCESS; - if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { - code = pCtx[k].fpSet.process(&pCtx[k]); - - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); - taskInfo->code = code; - T_LONG_JMP(taskInfo->env, code); - } - } - - // restore it - functionCtxRestore(&pCtx[k], &status); - } - } -} - static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { SqlFunctionCtx* pCtx = pExprSup->pCtx; for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { @@ -509,25 +380,6 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int return code; } -static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) { - for (int32_t k = 0; k < pOperator->exprSupp.numOfExprs; ++k) { - if (functionNeedToExecute(&pCtx[k])) { - // todo add a dummy funtion to avoid process check - if (pCtx[k].fpSet.process == NULL) { - continue; - } - - int32_t code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); - return code; - } - } - } - - return TSDB_CODE_SUCCESS; -} - bool functionNeedToExecute(SqlFunctionCtx* pCtx) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -630,7 +482,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } } -bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; } +bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } @@ -651,149 +503,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, - uint32_t* status) { - *status = BLK_DATA_NOT_LOAD; - - pBlock->pDataBlock = NULL; - pBlock->pBlockAgg = NULL; - - // int64_t groupId = pRuntimeEnv->current->groupIndex; - // bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - - STaskCostInfo* pCost = &pTaskInfo->cost; - -// pCost->totalBlocks += 1; -// pCost->totalRows += pBlock->info.rows; -#if 0 - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || - (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pTaskInfo, &pBlock->info))) { - (*status) = BLK_DATA_DATA_LOAD; - } - - // check if this data block is required to load - if ((*status) != BLK_DATA_DATA_LOAD) { - bool needFilter = true; - - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (pQueryAttr->pointInterpQuery) { - needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset); - } else { - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate - doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, - pTableScanInfo->rowEntryInfoOffset, pTableScanInfo->numOfOutput, - pRuntimeEnv->current->groupIndex); - } - - if (needFilter) { - (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); - } else { - (*status) = BLK_DATA_DATA_LOAD; - } - } - - SDataBlockInfo* pBlockInfo = &pBlock->info; -// *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); - - if ((*status) == BLK_DATA_NOT_LOAD || (*status) == BLK_DATA_FILTEROUT) { - //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, -// pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; - } else if ((*status) == BLK_DATA_SMA_LOAD) { - // this function never returns error? - pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); - - if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); - pCost->totalCheckedRows += pBlock->info.rows; - } - } else { - assert((*status) == BLK_DATA_DATA_LOAD); - - // load the data block statistics to perform further filter - pCost->loadBlockStatis += 1; -// tsdbRetrieveDatablockSMA(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); - - if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { - { // set previous window - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - SResultRow* pResult = NULL; - - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); - TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; - - STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); - if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, - pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, - pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); - } - } - } - bool load = false; - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pTableScanInfo->pCtx[i].functionId; - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { -// load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min), -// (char*)&(pBlock->pBlockAgg[i].max)); - if (!load) { // current block has been discard due to filter applied - pCost->skipBlocks += 1; - //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - (*status) = BLK_DATA_FILTEROUT; - return TSDB_CODE_SUCCESS; - } - } - } - } - - // current block has been discard due to filter applied -// if (!doFilterByBlockSMA(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { -// pCost->skipBlocks += 1; -// qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, -// pBlockInfo->window.ekey, pBlockInfo->rows); -// (*status) = BLK_DATA_FILTEROUT; -// return TSDB_CODE_SUCCESS; -// } - - pCost->totalCheckedRows += pBlockInfo->rows; - pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); -// if (pBlock->pDataBlock == NULL) { -// return terrno; -// } - -// if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); -// } - -// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { -// filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); -// } - } -#endif - return TSDB_CODE_SUCCESS; -} - void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; @@ -1024,43 +733,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } } -void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { - // for simple group by query without interval, all the tables belong to one group result. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SAggOperatorInfo* pAggInfo = pOperator->info; - - SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo; - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - int32_t* rowEntryInfoOffset = pOperator->exprSupp.rowEntryInfoOffset; - - SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, - sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); - /* - * not assign result buffer yet, add new result buffer - * all group belong to one result set, and each group result has different group id so set the id to be one - */ - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize); - if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, terrno); - } - } - - setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); -} - -static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId) { - SAggOperatorInfo* pAggInfo = pOperator->info; - if (pAggInfo->groupId != UINT64_MAX && pAggInfo->groupId == groupId) { - return; - } - - doSetTableGroupOutputBuf(pOperator, numOfOutput, groupId); - - // record the current active group id - pAggInfo->groupId = groupId; -} - void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -1198,10 +870,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", - (pRow->numOfRows+pBlock->info.rows), - pBlock->info.capacity, GET_TASKID(pTaskInfo)); - // todo set the pOperator->resultInfo size + qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s", + (pRow->numOfRows + pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo)); + // todo set the pOperator->resultInfo size } pGroupResInfo->index += 1; @@ -1211,7 +882,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS pBlock->info.rows += pRow->numOfRows; } - qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, + qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.id.groupId); pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); @@ -1278,161 +949,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -// static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, -// STableQueryInfo* pTableQueryInfo) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// -// STimeWindow tw = *win; -// getNextTimeWindow(pQueryAttr, &tw); -// -// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || -// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { -// -// // load the data block and check data remaining in current data block -// // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// tw = *win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); -// -// // set the abort info -// pQueryAttr->pos = startPos; -// -// // reset the query start timestamp -// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pQueryAttr->window.skey = pTableQueryInfo->win.skey; -// TSKEY key = pTableQueryInfo->win.skey; -// -// pWindowResInfo->prevSKey = tw.skey; -// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index -// -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, -// lastKey:%" PRId64, -// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, -// pQueryAttr->current->lastKey); -// -// return key; -// } else { // do nothing -// pQueryAttr->window.skey = tw.skey; -// pWindowResInfo->prevSKey = tw.skey; -// pTableQueryInfo->lastKey = tw.skey; -// -// return tw.skey; -// } -// -// return true; -// } - -// static bool skipTimeInterval(STaskRuntimeEnv *pRuntimeEnv, TSKEY* start) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// // if queried with value filter, do NOT forward query start position -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || -// pRuntimeEnv->pFillInfo != NULL) { -// return true; -// } -// -// /* -// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for -// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset -// value is -// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points -// */ -// STimeWindow w = TSWINDOW_INITIALIZER; -// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); -// -// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; -// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, -// &w); pWindowResInfo->prevSKey = w.skey; -// } -// } else { -// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); -// pWindowResInfo->prevSKey = w.skey; -// } -// -// // the first time window -// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); -// -// while (pQueryAttr->limit.offset > 0) { -// STimeWindow tw = win; -// -// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// pWindowResInfo->prevSKey = win.skey; -// -// // current time window is aligned with blockInfo.window.ekey -// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; -// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { -// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; -// } -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } -// -// // current window does not ended in current data block, try next data block -// getNextTimeWindow(pQueryAttr, &tw); -// -// /* -// * If the next time window still starts from current data block, -// * load the primary timestamp column first, and then find the start position for the next queried time window. -// * Note that only the primary timestamp column is required. -// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually -// required -// * time window resides in current data block. -// */ -// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { -// -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { -// pQueryAttr->limit.offset -= 1; -// } -// -// if (pQueryAttr->limit.offset == 0) { -// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); -// return true; -// } else { -// tw = win; -// int32_t startPos = -// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); -// // set the abort info -// pQueryAttr->pos = startPos; -// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; -// pWindowResInfo->prevSKey = tw.skey; -// win = tw; -// } -// } else { -// break; // offset is not 0, and next time window begins or ends in the next block. -// } -// } -// } -// -// // check for error -// if (terrno != TSDB_CODE_SUCCESS) { -// T_LONG_JMP(pRuntimeEnv->env, terrno); -// } -// -// return true; -// } - int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); if (p->pDownstream == NULL) { @@ -1478,191 +994,25 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan } } -static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { - if (!tsCountAlwaysReturnValue) { - return TSDB_CODE_SUCCESS; - } - - SAggOperatorInfo* pAggInfo = pOperator->info; - if (pAggInfo->groupKeyOptimized) { - return TSDB_CODE_SUCCESS; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || - (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && - ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { - return TSDB_CODE_SUCCESS; - } - - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - bool hasCountFunc = false; - - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; - if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || - (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { - hasCountFunc = true; - break; - } - } - - if (!hasCountFunc) { - return TSDB_CODE_SUCCESS; - } - - SSDataBlock* pBlock = createDataBlock(); - pBlock->info.rows = 1; - pBlock->info.capacity = 0; - - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.hasNull = true; - colInfo.info.type = TSDB_DATA_TYPE_NULL; - colInfo.info.bytes = 1; - - SExprInfo* pOneExpr = &pOperator->exprSupp.pExprInfo[i]; - for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { - SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; - if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { - int32_t slotId = pFuncParam->pCol->slotId; - int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - if (slotId >= numOfCols) { - taosArrayEnsureCap(pBlock->pDataBlock, slotId + 1); - for (int32_t k = numOfCols; k < slotId + 1; ++k) { - taosArrayPush(pBlock->pDataBlock, &colInfo); - } - } - } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - // do nothing - } - } - } - - blockDataEnsureCapacity(pBlock, pBlock->info.rows); - *ppBlock = pBlock; - - return TSDB_CODE_SUCCESS; -} - -static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) { - if (!blockAllocated) { - return; - } - - blockDataDestroy(*ppBlock); - *ppBlock = NULL; -} - -// this is a blocking operator -static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SAggOperatorInfo* pAggInfo = pOperator->info; - - SExprSupp* pSup = &pOperator->exprSupp; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - int64_t st = taosGetTimestampUs(); - - int32_t order = TSDB_ORDER_ASC; - int32_t scanFlag = MAIN_SCAN; - - bool hasValidBlock = false; - bool blockAllocated = false; - - while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - if (!hasValidBlock) { - createDataBlockForEmptyInput(pOperator, &pBlock); - if (pBlock == NULL) { - break; - } - blockAllocated = true; - } else { - break; - } - } - hasValidBlock = true; - - int32_t code = getTableScanInfo(pOperator, &order, &scanFlag, false); - if (code != TSDB_CODE_SUCCESS) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - - // there is an scalar expression that needs to be calculated before apply the group aggregation. - if (pAggInfo->scalarExprSup.pExprInfo != NULL && !blockAllocated) { - SExprSupp* pSup1 = &pAggInfo->scalarExprSup; - code = projectApplyFunctions(pSup1->pExprInfo, pBlock, pBlock, pSup1->pCtx, pSup1->numOfExprs, NULL); - if (code != TSDB_CODE_SUCCESS) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - } - - // the pDataBlock are always the same one, no need to call this again - setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId); - setInputDataBlock(pSup, pBlock, order, scanFlag, true); - code = doAggregateImpl(pOperator, pSup->pCtx); - if (code != 0) { - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - T_LONG_JMP(pTaskInfo->env, code); - } - - destroyDataBlockForEmptyInput(blockAllocated, &pBlock); - } - - // the downstream operator may return with error code, so let's check the code before generating results. - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); - } - - initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, 0); - OPTR_SET_OPENED(pOperator); - - pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return pTaskInfo->code; -} - -static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { - SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { - setOperatorCompleted(pOperator); +// QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { + if (pOperator == NULL) { + qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return NULL; } - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - while (1) { - doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); - - if (!hasRemainResults(&pAggInfo->groupResInfo)) { - setOperatorCompleted(pOperator); - break; + if (pOperator->operatorType == type) { + return pOperator; + } else { + if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { + qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return NULL; } - if (pInfo->pRes->info.rows > 0) { - break; - } + return extractOperatorInTree(pOperator->pDownstream[0], type, id); } - - size_t rows = blockDataGetNumOfRows(pInfo->pRes); - pOperator->resultInfo.totalRows += rows; - - return (rows == 0) ? NULL : pInfo->pRes; } void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { @@ -1729,70 +1079,6 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul return 0; } -int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, - const char* pKey) { - int32_t code = 0; - // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - - pAggSup->currentPageId = -1; - pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = tSimpleHashInit(100, taosFastHash); - - if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - uint32_t defaultPgsz = 0; - uint32_t defaultBufsz = 0; - getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz); - - if (!osTempSpaceAvailable()) { - code = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey); - return code; - } - - code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); - if (code != TSDB_CODE_SUCCESS) { - qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey); - return code; - } - - return code; -} - -void cleanupAggSup(SAggSupporter* pAggSup) { - taosMemoryFreeClear(pAggSup->keyBuf); - tSimpleHashCleanup(pAggSup->pResultRowHashTable); - destroyDiskbasedBuf(pAggSup->pResultBuf); -} - -int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState) { - int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - if (pState) { - pSup->pCtx[i].saveHandle.pBuf = NULL; - pSup->pCtx[i].saveHandle.pState = pState; - pSup->pCtx[i].exprIdx = i; - } else { - pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf; - } - } - - return TSDB_CODE_SUCCESS; -} - void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows) { if (numOfRows == 0) { numOfRows = 4096; @@ -1811,7 +1097,7 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) { initResultRowInfo(&pInfo->resultRowInfo); } -void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -1863,101 +1149,10 @@ void cleanupExprSupp(SExprSupp* pSupp) { taosMemoryFree(pSupp->rowEntryInfoOffset); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, - SExecTaskInfo* pTaskInfo) { - SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc); - initBasicInfo(&pInfo->binfo, pResBlock); - - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); - - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); - int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, - pTaskInfo->streamInfo.pState); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - int32_t numOfScalarExpr = 0; - SExprInfo* pScalarExprInfo = NULL; - if (pAggNode->pExprs != NULL) { - pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); - } - - code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; - pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; - pInfo->groupId = UINT64_MAX; - - setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, - pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, - optrDefaultBufFn, NULL); - - if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { - STableScanInfo* pTableScanInfo = downstream->info; - pTableScanInfo->base.pdInfo.pExprSup = &pOperator->exprSupp; - pTableScanInfo->base.pdInfo.pAggSup = &pInfo->aggSup; - } - - code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - return pOperator; - -_error: - if (pInfo != NULL) { - destroyAggOperatorInfo(pInfo); - } - - if (pOperator != NULL) { - cleanupExprSupp(&pOperator->exprSupp); - } - - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -static void freeItem(void* pItem) { - void** p = pItem; - if (*p != NULL) { - taosMemoryFreeClear(*p); - } -} - -void destroyAggOperatorInfo(void* param) { - SAggOperatorInfo* pInfo = (SAggOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - - cleanupAggSup(&pInfo->aggSup); - cleanupExprSupp(&pInfo->scalarExprSup); - cleanupGroupResInfo(&pInfo->groupResInfo); - taosMemoryFreeClear(param); -} - -static char* buildTaskId(uint64_t taskId, uint64_t queryId) { - char* p = taosMemoryMalloc(64); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { + char* p = dst; int32_t offset = 6; memcpy(p, "TID:0x", offset); @@ -1968,11 +1163,10 @@ static char* buildTaskId(uint64_t taskId, uint64_t queryId) { offset += tintToHex(queryId, &p[offset]); p[offset] = 0; - - return p; } -static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) { +SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, + char* dbFName) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1980,20 +1174,22 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT } setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); pTaskInfo->execModel = model; - pTaskInfo->pTableInfoList = tableListCreate(); pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + taosInitRWLatch(&pTaskInfo->lock); + pTaskInfo->id.vgId = vgId; pTaskInfo->id.queryId = queryId; - pTaskInfo->id.str = buildTaskId(taskId, queryId); + + pTaskInfo->id.str = taosMemoryMalloc(64); + buildTaskId(taskId, queryId, pTaskInfo->id.str); return pTaskInfo; } -SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); - int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; if (pHandle == NULL) { @@ -2100,9 +1296,8 @@ bool groupbyTbname(SNodeList* pGroupList) { SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, SNode* pTagIndexCond, const char* pUser) { - int32_t type = nodeType(pPhyNode); - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - const char* idstr = GET_TASKID(pTaskInfo); + int32_t type = nodeType(pPhyNode); + const char* idstr = GET_TASKID(pTaskInfo); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { SOperatorInfo* pOperator = NULL; @@ -2115,11 +1310,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTableScanNode->groupSort = true; } - int32_t code = + STableListInfo* pTableListInfo = tableListCreate(); + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); return NULL; } @@ -2127,10 +1324,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); if (code) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } - pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; return NULL; @@ -2140,11 +1338,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); return NULL; } @@ -2152,12 +1352,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); if (code) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } - pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } @@ -2168,29 +1370,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); return NULL; } - -#ifndef NDEBUG - int32_t sz = tableListGetSize(pTableListInfo); - qDebug("create stream task, total:%d", sz); - - for (int32_t i = 0; i < sz; i++) { - STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); - qDebug("add table uid:%" PRIu64 ", gid:%" PRIu64, pKeyInfo->uid, pKeyInfo->groupId); - } -#endif } pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); + pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); @@ -2199,18 +1394,19 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - - int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, - pTagIndexCond, pTaskInfo); + STableListInfo* pTableListInfo = tableListCreate(); + int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, + pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; qError("failed to getTableList, code: %s", tstrerror(code)); return NULL; } - pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo); + pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); @@ -2231,9 +1427,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0); } - pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTaskInfo); + pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); @@ -2248,7 +1445,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTaskInfo); + pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); } else { @@ -2256,7 +1453,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - if (pOperator != NULL) { + if (pOperator != NULL) { // todo moved away pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; } @@ -2346,6 +1543,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { terrno = TSDB_CODE_INVALID_PARA; + taosMemoryFree(ops); return NULL; } @@ -2396,9 +1594,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) { - SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo; - +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam)); @@ -2415,18 +1611,23 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT if (NULL == pDeleterParam) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t tbNum = tableListGetSize(pTask->pTableInfoList); - pDeleterParam->suid = tableListGetSuid(pTask->pTableInfoList); + + SArray* pInfoList = getTableListInfo(pTask); + STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); + taosArrayDestroy(pInfoList); + + pDeleterParam->suid = tableListGetSuid(pTableListInfo); // TODO extract uid list - pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); + int32_t numOfTables = tableListGetSize(pTableListInfo); + pDeleterParam->pUidList = taosArrayInit(numOfTables, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); return TSDB_CODE_OUT_OF_MEMORY; } - for (int32_t i = 0; i < tbNum; ++i) { - STableKeyInfo* pTable = tableListGetInfo(pTask->pTableInfoList, i); + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i); taosArrayPush(pDeleterParam->pUidList, &pTable->uid); } @@ -2440,17 +1641,14 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT return TSDB_CODE_SUCCESS; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - char* sql, EOPTR_EXEC_MODEL model) { - uint64_t queryId = pPlan->id.queryId; - - *pTaskInfo = createExecTaskInfo(queryId, taskId, model, pPlan->dbFName); +int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { + *pTaskInfo = doCreateExecTaskInfo(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName); if (*pTaskInfo == NULL) { goto _complete; } if (pHandle) { - /*(*pTaskInfo)->streamInfo.fillHistoryVer1 = pHandle->fillHistoryVer1;*/ if (pHandle->pStateBackend) { (*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend; } @@ -2468,7 +1666,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - (*pTaskInfo)->cost.created = taosGetTimestampUs(); return TSDB_CODE_SUCCESS; _complete: @@ -2484,8 +1681,6 @@ static void freeBlock(void* pParam) { void doDestroyTask(SExecTaskInfo* pTaskInfo) { qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); - - pTaskInfo->pTableInfoList = tableListDestroy(pTaskInfo->pTableInfoList); destroyOperatorInfo(pTaskInfo->pRoot); cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); cleanupStreamInfo(&pTaskInfo->streamInfo); @@ -2776,22 +1971,61 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta } void qStreamCloseTsdbReader(void* task) { - if (task == NULL) return; + if (task == NULL) { + return; + } + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; SOperatorInfo* pOp = pTaskInfo->pRoot; - qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid, - pTaskInfo->streamInfo.lastStatus.ts); - pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; + + qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid, + pTaskInfo->streamInfo.currentOffset.ts); + + // todo refactor, other thread may already use this read to extract data. + pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0}; while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pInfo = pDownstreamOp->info; if (pInfo->pTableScanOp) { STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + + setOperatorCompleted(pInfo->pTableScanOp); + while (pTaskInfo->owner != 0) { + taosMsleep(100); + qDebug("wait for the reader stopping"); + } + tsdbReaderClose(pTSInfo->base.dataReader); pTSInfo->base.dataReader = NULL; + + // restore the status, todo refactor. + pInfo->pTableScanOp->status = OP_OPENED; + pTaskInfo->status = TASK_NOT_COMPLETED; return; } } } } + +static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pScanInfo = pOperator->info; + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pScanInfo = pOperator->info; + taosArrayPush(pList, &pScanInfo->base.pTableListInfo); + } else { + if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) { + extractTableList(pList, pOperator->pDownstream[0]); + } + } +} + +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { + SArray* pArray = taosArrayInit(0, POINTER_BYTES); + SOperatorInfo* pOperator = pTaskInfo->pRoot; + extractTableList(pArray, pOperator); + return pArray; +} \ No newline at end of file diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 2a33e3527ac02b82cb12832b8f484d321f1b220e..234f1a666ce58bdd006f9768f1b076b29e5c1bf7 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -447,9 +447,14 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { return NULL; } +void destroySPoint(void* ptr) { + SPoint* point = (SPoint*) ptr; + taosMemoryFreeClear(point->val); +} + void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) { - taosArrayDestroy(pFillLinear->pDeltaVal); - taosArrayDestroy(pFillLinear->pNextDeltaVal); + taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint); + taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint); taosMemoryFree(pFillLinear); return NULL; } @@ -611,19 +616,15 @@ static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pR } } -static void calcRowDeltaData(SResultRowData* pStartRow, SResultRowData* pEndRow, SArray* pDelta, SFillColInfo* pFillCol, - int32_t numOfCol, int32_t winCount) { +static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol, + int32_t numOfCol) { for (int32_t i = 0; i < numOfCol; i++) { if (!pFillCol[i].notFillCol) { int32_t slotId = GET_DEST_SLOT_ID(pFillCol + i); - SResultCellData* pSCell = getResultCell(pStartRow, slotId); - double start = 0.0; - GET_TYPED_DATA(start, double, pSCell->type, pSCell->pData); SResultCellData* pECell = getResultCell(pEndRow, slotId); - double end = 0.0; - GET_TYPED_DATA(end, double, pECell->type, pECell->pData); - double delta = (end - start) / winCount; - taosArraySet(pDelta, slotId, &delta); + SPoint* pPoint = taosArrayGet(pEndPoins, slotId); + pPoint->key = pEndRow->key; + memcpy(pPoint->val, pECell->pData, pECell->bytes); } } } @@ -674,10 +675,8 @@ void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillS setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo); pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(pFillSup->prev.key, pFillSup->next.key, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); pFillInfo->pResRow = &pFillSup->prev; pFillInfo->pLinearInfo->winIndex = 0; } break; @@ -780,25 +779,19 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); pFillInfo->pos = FILL_POS_MID; pFillInfo->pLinearInfo->nextEnd = nextWKey; - int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); pFillInfo->pResRow = &pFillSup->prev; - numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, - pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pNextDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); pFillInfo->pLinearInfo->hasNext = true; } else if (hasPrevWindow(pFillSup)) { setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo); pFillInfo->pos = FILL_POS_END; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(prevWKey, ts, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->prev, &pFillSup->cur, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); + calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); pFillInfo->pResRow = &pFillSup->prev; pFillInfo->pLinearInfo->hasNext = false; } else { @@ -806,10 +799,8 @@ void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillS setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo); pFillInfo->pos = FILL_POS_START; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - int32_t numOfWins = taosTimeCountInterval(ts, nextWKey, pFillSup->interval.sliding, - pFillSup->interval.slidingUnit, pFillSup->interval.precision); - calcRowDeltaData(&pFillSup->cur, &pFillSup->next, pFillInfo->pLinearInfo->pDeltaVal, pFillSup->pAllColInfo, - pFillSup->numOfAllCols, numOfWins); + calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo, + pFillSup->numOfAllCols); pFillInfo->pResRow = &pFillSup->cur; pFillInfo->pLinearInfo->hasNext = false; } @@ -906,13 +897,18 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* colDataSetNULL(pColData, index); continue; } - double* pDelta = taosArrayGet(pFillInfo->pLinearInfo->pDeltaVal, slotId); + SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId); double vCell = 0; - GET_TYPED_DATA(vCell, double, pCell->type, pCell->pData); - vCell += (*pDelta) * pFillInfo->pLinearInfo->winIndex; - int64_t result = 0; - SET_TYPED_DATA(&result, pCell->type, vCell); - colDataSetVal(pColData, index, (const char*)&result, false); + SPoint start = {0}; + start.key = pFillInfo->pResRow->key; + start.val = pCell->pData; + + SPoint cur = {0}; + cur.key = pFillInfo->current; + cur.val = taosMemoryCalloc(1, pCell->bytes); + taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type); + colDataSetVal(pColData, index, (const char*)cur.val, false); + destroySPoint(&cur); } } pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit, @@ -953,8 +949,7 @@ static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) { pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->winIndex = 0; - taosArrayClear(pFillInfo->pLinearInfo->pDeltaVal); - taosArrayAddAll(pFillInfo->pLinearInfo->pDeltaVal, pFillInfo->pLinearInfo->pNextDeltaVal); + taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints); pFillInfo->pResRow = &pFillSup->cur; setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo); doStreamFillLinear(pFillSup, pFillInfo, pRes); @@ -1359,15 +1354,19 @@ SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo)); pFillInfo->pLinearInfo->hasNext = false; pFillInfo->pLinearInfo->nextEnd = INT64_MIN; - pFillInfo->pLinearInfo->pDeltaVal = NULL; - pFillInfo->pLinearInfo->pNextDeltaVal = NULL; + pFillInfo->pLinearInfo->pEndPoints = NULL; + pFillInfo->pLinearInfo->pNextEndPoints = NULL; if (pFillSup->type == TSDB_FILL_LINEAR) { - pFillInfo->pLinearInfo->pDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); - pFillInfo->pLinearInfo->pNextDeltaVal = taosArrayInit(pFillSup->numOfAllCols, sizeof(double)); + pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); + pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint)); for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) { - double value = 0.0; - taosArrayPush(pFillInfo->pLinearInfo->pDeltaVal, &value); - taosArrayPush(pFillInfo->pLinearInfo->pNextDeltaVal, &value); + SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i); + SPoint value = {0}; + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value); + + value.val = taosMemoryCalloc(1, pColData->info.bytes); + taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value); } } pFillInfo->pLinearInfo->winIndex = 0; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 0fc2cdb46a6d6c4d4ac8dba7054778b747cf7512..3d9bacf39f3ea14e5e753d24dd618d6b53de0cbe 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData size_t numOfGroupCols = taosArrayGetSize(pGroupCols); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = taosArrayGet(pGroupCols, i); + SColumn* pCol = (SColumn*) taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); + // valid range check. todo: return error code. + if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) { + continue; + } + if (pBlock->pBlockAgg != NULL) { pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } @@ -277,6 +282,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { terrno = TSDB_CODE_SUCCESS; int32_t num = 0; + uint64_t groupId = 0; for (int32_t j = 0; j < pBlock->info.rows; ++j) { // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (!pInfo->isInit) { @@ -473,6 +479,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* initResultRowInfo(&pInfo->binfo.resultRowInfo); setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); + pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); @@ -917,7 +925,7 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; SResultRow* pResultRow = - doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); + doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); assert(pResultRow != NULL); setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 49bc5af634b82c410e903d023d8c1e5e62b9d3fd..86c49e0fc82b76a595480e61b53167aab4b81f18 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -161,10 +161,9 @@ static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo if (pLimitInfo->remainGroupOffset > 0) { return PROJECT_RETRIEVE_CONTINUE; } - } - // set current group id of the project operator - pLimitInfo->currentGroupId = pBlock->info.id.groupId; + pLimitInfo->currentGroupId = 0; + } } return PROJECT_RETRIEVE_DONE; @@ -175,19 +174,29 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S // here check for a new group data, we need to handle the data of the previous group. ASSERT(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1); - if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.id.groupId) { + bool newGroup = false; + if (0 == pBlock->info.id.groupId) { + pLimitInfo->numOfOutputGroups = 1; + } else if (pLimitInfo->currentGroupId != pBlock->info.id.groupId) { + pLimitInfo->currentGroupId = pBlock->info.id.groupId; pLimitInfo->numOfOutputGroups += 1; - if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - setOperatorCompleted(pOperator); - return PROJECT_RETRIEVE_DONE; - } + newGroup = true; + } else { + return PROJECT_RETRIEVE_CONTINUE; + } - // reset the value for a new group data - // existing rows that belongs to previous group. - resetLimitInfoForNextGroup(pLimitInfo); + if ((pLimitInfo->slimit.limit >= 0) && (pLimitInfo->slimit.limit < pLimitInfo->numOfOutputGroups)) { + setOperatorCompleted(pOperator); + return PROJECT_RETRIEVE_DONE; } - return PROJECT_RETRIEVE_DONE; + // reset the value for a new group data + // existing rows that belongs to previous group. + if (newGroup) { + resetLimitInfoForNextGroup(pLimitInfo); + } + + return PROJECT_RETRIEVE_CONTINUE; } // todo refactor @@ -199,7 +208,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS if (pBlock->info.rows == 0) { return PROJECT_RETRIEVE_CONTINUE; } else { - if (limitReached && (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { setOperatorCompleted(pOperator); } } @@ -218,17 +227,8 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.submit.msgStr) { - pOperator->status = OP_OPENED; - } if (pOperator->status == OP_EXEC_DONE) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - pOperator->status = OP_OPENED; - qDebug("projection in queue model, set status open and return null"); - return NULL; - } - return NULL; } @@ -254,23 +254,14 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { - pOperator->status = OP_OPENED; - if (pOperator->status == OP_EXEC_RECV) { - continue; - } else { - return NULL; - } - } - qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, - pFinalRes->info.rows); + qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; } - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - qDebug("set status recv"); - pOperator->status = OP_EXEC_RECV; - } +// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { +// qDebug("set status recv"); +// pOperator->status = OP_EXEC_RECV; +// } // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || @@ -328,7 +319,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { - qDebug("project return %d rows, status %d", pFinalRes->info.rows, pOperator->status); + qDebug("project return %" PRId64 " rows, status %d", pFinalRes->info.rows, pOperator->status); break; } } else { @@ -580,7 +571,7 @@ void setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SA int64_t tid = 0; int64_t groupId = 0; SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, - pTaskInfo, false, pSup); + pTaskInfo, false, pSup, true); for (int32_t i = 0; i < numOfExprs; ++i) { struct SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, rowEntryInfoOffset); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2f3b75724115fe7a79a62bc6d073d6dd47068c1e..2389c7252ef56265f0e695a97fd025b4a4a09675 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -31,6 +31,9 @@ #include "thash.h" #include "ttypes.h" +int32_t scanDebug = 0; + +#define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -44,8 +47,20 @@ typedef struct STableMergeScanSortSourceParam { int32_t readerIdx; uint64_t uid; SSDataBlock* inputBlock; + bool multiReader; + STsdbReader* dataReader; } STableMergeScanSortSourceParam; +typedef struct STableCountScanOperatorInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + + STableCountScanSupp supp; + + int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and/or stable_name +} STableCountScanOperatorInfo; + static bool processBlockWithProbability(const SSampleExecInfo* pInfo); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -85,7 +100,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord struct tm tm; time_t t = (time_t)key; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; @@ -305,16 +320,17 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca taosMemoryFreeClear(pBlock->pBlockAgg); if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%" PRIu64, GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, + GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, + pBlockInfo->id.uid); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; @@ -323,9 +339,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead - qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { @@ -343,8 +359,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca size_t size = taosArrayGetSize(pBlock->pDataBlock); bool keep = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows); if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, + GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; @@ -360,8 +376,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, + GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); *status = FUNC_DATA_REQUIRED_FILTEROUT; @@ -391,7 +407,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms", + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", elapsed time:%.2f ms", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el); } else { qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el); @@ -581,10 +597,16 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int if (isNullVal) { colDataSetNNULL(pColInfoData, 0, pBlock->info.rows); } else if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) { - colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows); + code = colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows, false); if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } + if (code) { + if (freeReader) { + metaReaderClear(&mr); + } + return code; + } } else { // todo opt for json tag for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataSetVal(pColInfoData, i, data, false); @@ -631,10 +653,22 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { STableScanInfo* pTableScanInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSDataBlock* pBlock = pTableScanInfo->pResBlock; + bool hasNext = false; + int32_t code = TSDB_CODE_SUCCESS; int64_t st = taosGetTimestampUs(); - while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) { + while (true) { + code = tsdbNextDataBlock(pTableScanInfo->base.dataReader, &hasNext); + if (code) { + tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); + T_LONG_JMP(pTaskInfo->env, code); + } + + if (!hasNext) { + break; + } + if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); @@ -651,11 +685,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - ASSERT(pBlock->info.id.uid != 0); - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + if (pBlock->info.id.uid) { + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); + } uint32_t status = 0; - int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -673,11 +708,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { // todo refactor /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; + // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; + // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - ASSERT(pBlock->info.id.uid != 0); return pBlock; } return NULL; @@ -747,42 +781,51 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // scan table one by one sequentially if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - int32_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t numOfTables = 0; // tableListGetSize(pTaskInfo->pTableListInfo); + STableKeyInfo tInfo = {0}; while (1) { SSDataBlock* result = doGroupedTableScan(pOperator); - if (result) { + if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) { return result; } // if no data, switch to next table and continue scan pInfo->currentTable++; + + taosRLockLatch(&pTaskInfo->lock); + numOfTables = tableListGetSize(pInfo->base.pTableListInfo); + if (pInfo->currentTable >= numOfTables) { + qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); + taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - STableKeyInfo* pTableInfo = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->currentTable); - tsdbSetTableList(pInfo->base.dataReader, pTableInfo, 1); - qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, - pInfo->currentTable, pTaskInfo->id.str); + tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + taosRUnLockLatch(&pTaskInfo->lock); + + tsdbSetTableList(pInfo->base.dataReader, &tInfo, 1); + qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, + pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; } } else { // scan table group by group sequentially if (pInfo->currentGroupId == -1) { - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); return NULL; } int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -794,11 +837,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { - ASSERT(result->info.id.uid != 0); return result; } - if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pTaskInfo->pTableInfoList)) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); return NULL; } @@ -809,7 +851,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, &pList, &num); tsdbSetTableList(pInfo->base.dataReader, pList, num); tsdbReaderReset(pInfo->base.dataReader, &pInfo->base.cond); @@ -834,28 +876,35 @@ static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptr return 0; } -static void destroyTableScanOperatorInfo(void* param) { - STableScanInfo* pTableScanInfo = (STableScanInfo*)param; - blockDataDestroy(pTableScanInfo->pResBlock); - cleanupQueryTableDataCond(&pTableScanInfo->base.cond); +static void destroyTableScanBase(STableScanBase* pBase) { + cleanupQueryTableDataCond(&pBase->cond); - tsdbReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; + tsdbReaderClose(pBase->dataReader); + pBase->dataReader = NULL; - if (pTableScanInfo->base.matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); + if (pBase->matchInfo.pList != NULL) { + taosArrayDestroy(pBase->matchInfo.pList); } - taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); - cleanupExprSupp(&pTableScanInfo->base.pseudoSup); + tableListDestroy(pBase->pTableListInfo); + taosLRUCacheCleanup(pBase->metaCache.pTableMetaEntryCache); + cleanupExprSupp(&pBase->pseudoSup); +} + +static void destroyTableScanOperatorInfo(void* param) { + STableScanInfo* pTableScanInfo = (STableScanInfo*)param; + blockDataDestroy(pTableScanInfo->pResBlock); + destroyTableScanBase(&pTableScanInfo->base); taosMemoryFreeClear(param); } SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { + int32_t code = 0; STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -863,7 +912,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - int32_t code = + code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -908,12 +957,17 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { code = terrno; goto _error; } + if (scanDebug) { + pInfo->countOnly = true; + } + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, optrDefaultBufFn, getTableScannerExecInfo); @@ -985,6 +1039,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; tsdbReaderClose(pTableScanInfo->base.dataReader); + qDebug("1"); pTableScanInfo->base.dataReader = NULL; } @@ -1004,21 +1059,29 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); return NULL; } - if (tsdbNextDataBlock(pReader)) { + bool hasNext = false; + code = tsdbNextDataBlock(pReader, &hasNext); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + T_LONG_JMP(pTaskInfo->env, code); + return NULL; + } + + if (hasNext) { /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); } tsdbReaderClose(pReader); - qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 + qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1035,7 +1098,8 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, } static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { - return getTableGroupId(pInfo->pTableScanOp->pTaskInfo->pTableInfoList, uid); + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + return getTableGroupId(pTableScanInfo->base.pTableListInfo, uid); } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1143,6 +1207,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTableScanInfo->base.dataReader); + qDebug("2"); pTableScanInfo->base.dataReader = NULL; return NULL; } @@ -1192,6 +1257,8 @@ static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, if (code != TSDB_CODE_SUCCESS) { SET_SESSION_WIN_KEY_INVALID(pKey); } + + taosMemoryFree(pCur); return code; } @@ -1506,7 +1573,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.version = pBlock->info.version; - pInfo->pRes->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + pInfo->pRes->info.id.groupId = getTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -1562,29 +1630,23 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; + const char* id = GET_TASKID(pTaskInfo); - qDebug("start to exec queue scan"); + qDebug("start to exec queue scan, %s", id); if (pTaskInfo->streamInfo.submit.msgStr != NULL) { if (pInfo->tqReader->msg2.msgStr == NULL) { - /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ - - /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ - /*void* msgStr = pTaskInfo->streamInfo.*/ SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { qError("submit msg messed up when initing stream submit block %p", submit.msgStr); - pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; - ASSERT(0); + return NULL; } } blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -1600,73 +1662,54 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64, pResult->info.rows, - pResult->info.window.skey, pResult->info.window.ekey); - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, - pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); - pTaskInfo->streamInfo.returned = 1; + qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, + pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, + pInfo->tqReader->pWalReader->curVersion); + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; - } else { - if (!pTaskInfo->streamInfo.returned) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; - tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); - if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { - tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer); - return NULL; - } - } else { - return NULL; - } } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); + if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + return NULL; + } + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { SFetchRet ret = {0}; - if (tqNextBlock(pInfo->tqReader, &ret) < 0) { - // if the end is reached, terrno is 0 - if (terrno != 0) { - qError("failed to get next log block since %s", terrstr()); - } - } + tqNextBlock(pInfo->tqReader, &ret); + tqOffsetResetToLog( + &pTaskInfo->streamInfo.currentOffset, + pInfo->tqReader->pWalReader->curVersion - 1); // curVersion move to next, so currentOffset = curVersion - 1 if (ret.fetchType == FETCH_TYPE__DATA) { + qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, ret.data.info.rows, + pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { - pOperator->status = OP_EXEC_RECV; - qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); + qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, + pTaskInfo->streamInfo.currentOffset.version); return pInfo->pRes; } - } else if (ret.fetchType == FETCH_TYPE__META) { - qError("unexpected ret.fetchType:%d", ret.fetchType); - continue; - // pTaskInfo->streamInfo.lastStatus = ret.offset; - // pTaskInfo->streamInfo.metaBlk = ret.meta; - // return NULL; - } else if (ret.fetchType == FETCH_TYPE__NONE || - (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) { - pTaskInfo->streamInfo.lastStatus = ret.offset; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, &ret.offset); - qDebug("queue scan log return null, offset %s", formatBuf); - pOperator->status = OP_OPENED; + } else if (ret.fetchType == FETCH_TYPE__NONE) { + qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version); return NULL; } } } else { - qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.prepareStatus.type); + qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type); return NULL; } } @@ -1724,9 +1767,9 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } } -static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey) { +static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) { if (pInfo->pUpdateInfo) { - checkUpdateData(pInfo, true, pInfo->pRes, true); + checkUpdateData(pInfo, true, pBlock, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); if (pInfo->pUpdateDataRes->info.rows > 0) { pInfo->updateResIndex = 0; @@ -1758,25 +1801,28 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1; qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion); + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN1; } else { pTSInfo->base.cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->base.cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion); + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN2; } /*resetTableScanInfo(pTSInfo, pWin);*/ tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; pTaskInfo->streamInfo.recoverScanFinished = false; } - if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN) { + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1 || + pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN2) { if (pInfo->blockRecoverContiCnt > 100) { pInfo->blockRecoverTotCnt += pInfo->blockRecoverContiCnt; pInfo->blockRecoverContiCnt = 0; @@ -1789,6 +1835,37 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } break; + case STREAM_SCAN_FROM_UPDATERES: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + printDataBlock(pInfo->pUpdateRes, "recover update"); + return pInfo->pUpdateRes; + } break; + case STREAM_SCAN_FROM_DELETE_DATA: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + printDataBlock(pInfo->pDeleteDataRes, "recover delete"); + return pInfo->pDeleteDataRes; + } break; + case STREAM_SCAN_FROM_DATAREADER_RANGE: { + SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + if (pSDB) { + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + uint64_t version = getReaderMaxVersion(pTableScanInfo->base.dataReader); + updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version); + pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; + checkUpdateData(pInfo, true, pSDB, false); + printDataBlock(pSDB, "scan recover update"); + calBlockTbName(pInfo, pSDB); + return pSDB; + } + blockDataCleanup(pInfo->pUpdateDataRes); + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } break; default: break; } @@ -1798,20 +1875,26 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->blockRecoverContiCnt++; calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__SCAN1) { + TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + } else { + doCheckUpdate(pInfo, pInfo->pRecoverRes->info.window.ekey, pInfo->pRecoverRes); + } } if (pInfo->pCreateTbRes->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_RES; + printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } - qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; pTSInfo->base.cond.startVersion = -1; @@ -1837,6 +1920,7 @@ FETCH_NEXT_BLOCK: if (pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } + // TODO move into scan pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.ekey = INT64_MAX; @@ -1910,7 +1994,7 @@ FETCH_NEXT_BLOCK: switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey); + doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey, pInfo->pRes); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); @@ -1959,6 +2043,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock); blockDataCleanup(pSup->pScanBlock); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; return pInfo->pUpdateRes; } @@ -1978,8 +2063,7 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ - if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); continue; @@ -1988,7 +2072,7 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -2011,7 +2095,7 @@ FETCH_NEXT_BLOCK: return pInfo->pCreateTbRes; } - doCheckUpdate(pInfo, pBlockInfo->window.ekey); + doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); @@ -2032,7 +2116,7 @@ FETCH_NEXT_BLOCK: pOperator->resultInfo.totalRows += pBlockInfo->rows; // printDataBlock(pInfo->pRes, "stream scan"); - qDebug("scan rows: %d", pBlockInfo->rows); + qDebug("scan rows: %" PRId64, pBlockInfo->rows); if (pBlockInfo->rows > 0) { return pInfo->pRes; } @@ -2065,45 +2149,50 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamRawScanInfo* pInfo = pOperator->info; + int32_t code = TSDB_CODE_SUCCESS; pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; qDebug("tmqsnap doRawScan called"); - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { + bool hasNext = false; + if (pInfo->dataReader) { + code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); + if (code) { + tsdbReleaseDataBlock(pInfo->dataReader); + T_LONG_JMP(pTaskInfo->env, code); + } + } + + if (pInfo->dataReader && hasNext) { if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(pInfo->dataReader); - longjmp(pTaskInfo->env, pTaskInfo->code); + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); if (pBlock == NULL) { - longjmp(pTaskInfo->env, terrno); + T_LONG_JMP(pTaskInfo->env, terrno); } qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pBlock->info.id.uid, pBlock->info.window.ekey); return pBlock; } SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; - pTaskInfo->streamInfo.lastStatus.version = pInfo->sContext->snapVersion; + tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); } else { - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.prepareStatus.ts = INT64_MIN; + tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); - qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType); } + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); tDeleteSSchemaWrapper(mtInfo.schema); - qDebug("tmqsnap stream scan tsdb return null"); return NULL; - } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) { + } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; void* data = NULL; int32_t dataLen = 0; @@ -2115,16 +2204,12 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return NULL; } - if (!sContext->queryMetaOrData) { // change to get data next poll request - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.metaRsp.rspOffset.uid = 0; - pTaskInfo->streamInfo.metaRsp.rspOffset.ts = INT64_MIN; + if (!sContext->queryMeta) { // change to get data next poll request + STqOffsetVal offset = {0}; + tqOffsetResetToData(&offset, 0, INT64_MIN); + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); } else { - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.lastStatus; + tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); pTaskInfo->streamInfo.metaRsp.resMsgType = type; pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; pTaskInfo->streamInfo.metaRsp.metaRsp = data; @@ -2177,6 +2262,7 @@ static void destroyRawScanOperatorInfo(void* param) { SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param; tsdbReaderClose(pRawScan->dataReader); destroySnapContext(pRawScan->sContext); + tableListDestroy(pRawScan->pTableListInfo); taosMemoryFree(pRawScan); } @@ -2198,6 +2284,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT goto _end; } + pInfo->pTableListInfo = tableListCreate(); pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; @@ -2216,9 +2303,11 @@ _end: static void destroyStreamScanOperatorInfo(void* param) { SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; + if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { destroyOperatorInfo(pStreamScan->pTableScanOp); } + if (pStreamScan->tqReader) { tqCloseReader(pStreamScan->tqReader); } @@ -2245,13 +2334,14 @@ static void destroyStreamScanOperatorInfo(void* param) { } SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SArray* pColIds = NULL; SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } @@ -2265,6 +2355,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys int32_t code = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { + tableListDestroy(pTableListInfo); goto _error; } @@ -2284,11 +2375,14 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } + pInfo->tbnameCalSup.pExprInfo = pSubTableExpr; createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0); if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1) != 0) { + tableListDestroy(pTableListInfo); goto _error; } } @@ -2298,10 +2392,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } } @@ -2309,11 +2405,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + tableListDestroy(pTableListInfo); goto _error; } if (pHandle->vnode) { - SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); + SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { pTSInfo->base.cond.endVersion = pHandle->version; @@ -2321,12 +2418,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys STableKeyInfo* pList = NULL; int32_t num = 0; - tableListGetGroupList(pTaskInfo->pTableInfoList, 0, &pList, &num); + tableListGetGroupList(pTableListInfo, 0, &pList, &num); if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - pTaskInfo->streamInfo.lastStatus.uid = -1; } if (pHandle->initTqReader) { @@ -2345,23 +2441,24 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } pInfo->readHandle = *pHandle; - pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); - SArray* tableIdList = extractTableIdList(pTaskInfo->pTableInfoList); + SArray* tableIdList = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo); code = tqReaderSetTbUidList(pInfo->tqReader, tableIdList); if (code != 0) { taosArrayDestroy(tableIdList); goto _error; } + taosArrayDestroy(tableIdList); memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond)); } else { taosArrayDestroy(pColIds); + tableListDestroy(pTableListInfo); pColIds = NULL; } @@ -2396,7 +2493,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; + __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScan : doQueueScan; pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); @@ -2427,7 +2524,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); - int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); + int32_t size = tableListGetSize(pInfo->pTableListInfo); if (size == 0) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; @@ -2439,7 +2536,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -2500,11 +2597,12 @@ static void destroyTagScanOperatorInfo(void* param) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); taosArrayDestroy(pInfo->matchInfo.pList); + pInfo->pTableListInfo = tableListDestroy(pInfo->pTableListInfo); taosMemoryFreeClear(param); } SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2526,6 +2624,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } + pInfo->pTableListInfo = pTableListInfo; pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; @@ -2554,24 +2653,41 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; + int32_t code = 0; SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pInfo->base.pTableListInfo, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; - int32_t code = - tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); - if (code != 0) { - T_LONG_JMP(pTaskInfo->env, code); + if (NULL == source->dataReader || !source->multiReader) { + code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false); + if (code != 0) { + T_LONG_JMP(pTaskInfo->env, code); + } } + pInfo->base.dataReader = source->dataReader; STsdbReader* reader = pInfo->base.dataReader; + bool hasNext = false; qTrace("tsdb/read-table-data: %p, enter next reader", reader); - while (tsdbNextDataBlock(reader)) { + + while (true) { + code = tsdbNextDataBlock(reader, &hasNext); + if (code != 0) { + tsdbReleaseDataBlock(reader); + pInfo->base.dataReader = NULL; + T_LONG_JMP(pTaskInfo->env, code); + } + + if (!hasNext) { + break; + } + if (isTaskKilled(pTaskInfo)) { tsdbReleaseDataBlock(reader); + pInfo->base.dataReader = NULL; T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2599,18 +2715,24 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { continue; } - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + pBlock->info.id.groupId = getTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; qTrace("tsdb/read-table-data: %p, close reader", reader); - tsdbReaderClose(pInfo->base.dataReader); + if (!source->multiReader) { + tsdbReaderClose(pInfo->base.dataReader); + source->dataReader = NULL; + } pInfo->base.dataReader = NULL; return pBlock; } - tsdbReaderClose(pInfo->base.dataReader); + if (!source->multiReader) { + tsdbReaderClose(pInfo->base.dataReader); + source->dataReader = NULL; + } pInfo->base.dataReader = NULL; return NULL; } @@ -2649,10 +2771,10 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; { - size_t numOfTables = tableListGetSize(pTaskInfo->pTableInfoList); + size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { - STableKeyInfo* tableKeyInfo = tableListGetInfo(pTaskInfo->pTableInfoList, i); + STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); if (tableKeyInfo->groupId != pInfo->groupId) { break; } @@ -2682,6 +2804,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanSortSourceParam param = {0}; param.readerIdx = i; param.pOperator = pOperator; + param.multiReader = (numOfTable <= MULTI_READER_MAX_TABLE_NUM) ? true : false; param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); @@ -2725,6 +2848,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { for (int32_t i = 0; i < numOfTable; ++i) { STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); blockDataDestroy(param->inputBlock); + tsdbReaderClose(param->dataReader); + param->dataReader = NULL; } taosArrayClear(pInfo->sortSourceParams); @@ -2764,12 +2889,9 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* } bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, pInfo->limitInfo.numOfOutputRows); - if (limitReached) { - resetLimitInfoForNextGroup(&pInfo->limitInfo); - } return (pResBlock->info.rows > 0) ? pResBlock : NULL; } @@ -2786,7 +2908,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } - size_t tableListSize = tableListGetSize(pTaskInfo->pTableInfoList); + size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo); if (!pInfo->hasGroupId) { pInfo->hasGroupId = true; @@ -2795,7 +2917,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { return NULL; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex))->groupId; + pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } @@ -2820,8 +2942,9 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId; + pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; startGroupTableMergeScan(pOperator); + resetLimitInfoForNextGroup(&pInfo->limitInfo); } } @@ -2837,35 +2960,29 @@ void destroyTableMergeScanOperatorInfo(void* param) { for (int32_t i = 0; i < numOfTable; i++) { STableMergeScanSortSourceParam* p = taosArrayGet(pTableScanInfo->sortSourceParams, i); blockDataDestroy(p->inputBlock); + tsdbReaderClose(p->dataReader); + p->dataReader = NULL; } + tsdbReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; + taosArrayDestroy(pTableScanInfo->sortSourceParams); tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; - tsdbReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; - for (int i = 0; i < taosArrayGetSize(pTableScanInfo->queryConds); i++) { SQueryTableDataCond* pCond = taosArrayGet(pTableScanInfo->queryConds, i); taosMemoryFree(pCond->colList); } - taosArrayDestroy(pTableScanInfo->queryConds); - if (pTableScanInfo->base.matchInfo.pList != NULL) { - taosArrayDestroy(pTableScanInfo->base.matchInfo.pList); - } + taosArrayDestroy(pTableScanInfo->queryConds); + destroyTableScanBase(&pTableScanInfo->base); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); - cleanupExprSupp(&pTableScanInfo->base.pseudoSup); - - tsdbReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; - taosLRUCacheCleanup(pTableScanInfo->base.metaCache.pTableMetaEntryCache); - taosMemoryFreeClear(param); } @@ -2884,7 +3001,7 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla } SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2926,6 +3043,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->base.limitInfo.limit.limit = -1; pInfo->base.limitInfo.slimit.limit = -1; + pInfo->base.pTableListInfo = pTableListInfo; pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); @@ -3155,7 +3273,7 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc size_t perfdbTableNum; getPerfDbMeta(NULL, &perfdbTableNum); - if (pSupp->groupByDbName) { + if (pSupp->groupByDbName || pSupp->groupByStbName) { buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum); return (pRes->info.rows > 0) ? pRes : NULL; } else { @@ -3180,11 +3298,23 @@ static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountSca STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, size_t perfdbTableNum) { if (pInfo->currGrpIdx == 0) { - uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + uint64_t groupId = 0; + if (pSupp->groupByDbName) { + groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + } else { + groupId = calcGroupId("", 0); + } + pRes->info.id.groupId = groupId; fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); } else if (pInfo->currGrpIdx == 1) { - uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + uint64_t groupId = 0; + if (pSupp->groupByDbName) { + groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + } else { + groupId = calcGroupId("", 0); + } + pRes->info.id.groupId = groupId; fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); } else { @@ -3222,7 +3352,7 @@ static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCount tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&sn, dbName); - if (pSupp->groupByDbName) { + if (pSupp->groupByDbName || pSupp->groupByStbName) { buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName); } else { buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName); @@ -3283,7 +3413,10 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + if (pSupp->groupByDbName) { + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + } + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); @@ -3298,7 +3431,12 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + if (pSupp->groupByDbName) { + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + } else { + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", stbName); + } + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); pRes->info.id.groupId = groupId; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 90c7fa10ca766c0d3c6ba67115b30070d569b6cd..cb0f1aa06845575e40d69ff4dca2ccaf96809b0a 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -698,7 +698,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData pDataBlock->info.dataLoad = 1; } - qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%" PRId64 , GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 24f42ff178c45660a13f58df2f3c025eb3e78048..1abe678ac6453ed0edacd9012937c8463000ceb3 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -57,9 +57,11 @@ typedef struct SSysTableScanInfo { const char* pUser; bool sysInfo; bool showRewrite; + bool restore; SNode* pCondition; // db_name filter condition, to discard data that are not in current database SMTbCursor* pCur; // cursor for iterate the local table meta store. SSysTableIndex* pIdx; // idx for local table meta + SHashObj* pSchema; SColMatchInfo matchInfo; SName name; SSDataBlock* pRes; @@ -81,10 +83,11 @@ typedef struct MergeIndex { } MergeIndex; typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + STableListInfo* pTableListInfo; + uint64_t uid; // table uid } SBlockDistInfo; static int32_t sysChkFilter__Comm(SNode* pNode); @@ -514,9 +517,23 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); } - SHashObj* stableSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - taosHashSetFreeFp(stableSchema, tDeleteSSchemaWrapperForHash); - while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0) { + if (pInfo->pSchema == NULL) { + pInfo->pSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + taosHashSetFreeFp(pInfo->pSchema, tDeleteSSchemaWrapperForHash); + } + + if (!pInfo->pCur || !pInfo->pSchema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("sysTableScanUserCols failed since %s", terrstr(terrno)); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + int32_t restore = pInfo->restore; + pInfo->restore = false; + while (restore || ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0)) { + if (restore) restore = false; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -524,33 +541,36 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) { qDebug("sysTableScanUserCols cursor get super table"); - void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); + void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); if (schema == NULL) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); - taosHashPut(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); + taosHashPut(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); } continue; } else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { qDebug("sysTableScanUserCols cursor get child table"); STR_TO_VARSTR(typeName, "CHILD_TABLE"); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t)); + void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t)); if (schema != NULL) { schemaRow = *(SSchemaWrapper**)schema; } else { - tDecoderClear(&pInfo->pCur->mr.coder); - int code = metaGetTableEntryByUid(&pInfo->pCur->mr, suid); + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + int code = metaGetTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { // terrno has been set by metaGetTableEntryByName, therefore, return directly qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); + metaReaderClear(&smrSuperTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; - taosHashCleanup(stableSchema); return NULL; } - schemaRow = &pInfo->pCur->mr.me.stbEntry.schemaRow; + SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); + taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); + schemaRow = schemaWrapper; + metaReaderClear(&smrSuperTable); } } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { qDebug("sysTableScanUserCols cursor get normal table"); @@ -562,20 +582,19 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { continue; } - sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); - - if (numOfRows >= pOperator->resultInfo.capacity) { + if ((numOfRows + schemaRow->nCols) > pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; + pInfo->restore = true; if (pInfo->pRes->info.rows > 0) { break; } + } else { + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); } } - taosHashCleanup(stableSchema); - if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); numOfRows = 0; @@ -695,7 +714,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) { - metaTbCursorPrev(pInfo->pCur); + metaTbCursorPrev(pInfo->pCur, TSDB_TABLE_MAX); blockFull = true; } else { sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, @@ -1609,7 +1628,7 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(varTbName, name); - colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows); + colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows, true); } doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); @@ -1789,6 +1808,11 @@ void destroySysScanOperator(void* param) { pInfo->pIdx = NULL; } + if(pInfo->pSchema) { + taosHashCleanup(pInfo->pSchema); + pInfo->pSchema = NULL; + } + taosArrayDestroy(pInfo->matchInfo.pList); taosMemoryFreeClear(pInfo->pUser); @@ -2191,6 +2215,7 @@ static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); tsdbReaderClose(pDistInfo->pHandle); + tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2222,7 +2247,7 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, - SExecTaskInfo* pTaskInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2240,11 +2265,11 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); + pInfo->pTableListInfo = pTableListInfo; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f4c8521b941b4e82e22195e2c42098705dacc251..e59ea253ccee8e3badda35445d4f681e76c1542e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -44,7 +44,7 @@ static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, } else { p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; } - + SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx); doSetVal(pDstColInfo, rowIndex, pKey); } @@ -578,7 +578,12 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* GET_TYPED_DATA(v1, double, inputType, point1->val); GET_TYPED_DATA(v2, double, inputType, point2->val); - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + double r = 0; + if (!IS_BOOLEAN_TYPE(inputType)) { + r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + } else { + r = (v1 < 1 || v2 < 1) ? 0 : 1; + } SET_TYPED_DATA(point->val, outputType, r); return TSDB_CODE_SUCCESS; @@ -630,6 +635,7 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn nodesValueNodeToVariant(pv, &pFillCol[i].fillVal); } } + pFillCol->numOfFillExpr = numOfFillExpr; for (int32_t i = 0; i < numOfNoFillExpr; ++i) { SExprInfo* pExprInfo = &pNotFillExpr[i]; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 911700be8534d7b6b7762854b82bd7ceaad12489..f0e25d8cc5a3786a66720de1e314c406361af11a 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -156,6 +156,16 @@ static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* return TSDB_CODE_SUCCESS; } +static bool isIrowtsPseudoColumn(SExprInfo* pExprInfo) { + char *name = pExprInfo->pExpr->_function.functionName; + return (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_irowts") == 0); +} + +static bool isIsfilledPseudoColumn(SExprInfo* pExprInfo) { + char *name = pExprInfo->pExpr->_function.functionName; + return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0); +} + static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { int32_t rows = pResBlock->info.rows; @@ -170,10 +180,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + if (isIrowtsPseudoColumn(pExprInfo)) { colDataSetVal(pDst, rows, (char*)&pSliceInfo->current, false); continue; - } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = true; colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; @@ -203,6 +213,14 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); colDataSetVal(pDst, rows, (char*)&v, false); + } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { + bool v = false; + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, bool, pVar->nType, &pVar->i); + } else { + v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); + } + colDataSetVal(pDst, rows, (char*)&v, false); } break; } @@ -288,9 +306,9 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + if (isIrowtsPseudoColumn(pExprInfo)) { colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = false; colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); } else { @@ -643,6 +661,9 @@ void destroyTimeSliceOperatorInfo(void* param) { taosArrayDestroy(pInfo->pLinearInfo); cleanupExprSupp(&pInfo->scalarSup); + for (int32_t i = 0; i < pInfo->pFillColInfo->numOfFillExpr; ++i) { + taosVariantDestroy(&pInfo->pFillColInfo[i].fillVal); + } taosMemoryFree(pInfo->pFillColInfo); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 6411d862ae20ba89b3de24a7c400d934a7a69180..007a6f63d16d59da4b648f43803a058963a6c487 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -78,7 +78,7 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, - masterscan, tableGroupId, pTaskInfo, true, pAggSup); + masterscan, tableGroupId, pTaskInfo, true, pAggSup, true); if (pResultRow == NULL) { *pResult = NULL; @@ -128,8 +128,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } } else { @@ -137,8 +138,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); @@ -281,7 +283,7 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o struct tm tm; time_t t = (time_t)key; - taosLocalTime(&t, &tm); + taosLocalTime(&t, &tm, NULL); int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; @@ -936,10 +938,10 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } + TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); - ASSERT(forwardRows > 0); // prev time window not interpolation yet. if (pInfo->timeWindowInterpo) { @@ -2111,10 +2113,12 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); if (code != TSDB_CODE_SUCCESS) { - qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); - pTaskInfo->code = code; - T_LONG_JMP(pTaskInfo->env, code); + qError("%s apply combine functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); } + } else if (pDestCtx[k].fpSet.combine == NULL) { + char* funName = fmGetFuncName(pDestCtx[k].functionId); + qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName); + taosMemoryFreeClear(funName); } } } @@ -2172,13 +2176,6 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S } } -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { - SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); - return p1 == NULL; -} - bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; @@ -2336,9 +2333,15 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } +static void setStreamDataVersion(SExecTaskInfo* pTaskInfo, int64_t version, int64_t ckId) { + pTaskInfo->streamInfo.dataVersion = version; + pTaskInfo->streamInfo.checkPointId = ckId; +} + static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -2435,10 +2438,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); - SWinKey key = { - .ts = nextWin.skey, - .groupId = groupId, - }; + + SWinKey key = { .ts = nextWin.skey, .groupId = groupId }; saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize); releaseOutputBuf(pInfo->pState, &key, pResult); if (pInfo->delKey.ts > key.ts) { @@ -2506,6 +2507,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamIntervalOperator(pInfo); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); qDebug("===stream===clear semi operator"); } else { deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, @@ -2770,6 +2772,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; + pInfo->ignoreExpiredDataSaved = false; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->delIndex = 0; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); @@ -2778,6 +2781,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->numOfDatapack = 0; pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; + pInfo->dataVersion = 0; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -3128,6 +3132,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; @@ -3588,8 +3594,10 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pStUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3900,6 +3908,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; + + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; @@ -3912,7 +3923,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); for (int32_t i = 0; i < rows; i += winRows) { - if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { + if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) { i++; continue; } @@ -4113,8 +4124,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; + pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pSeUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4750,6 +4763,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { &pInfo->delKey); setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); return NULL; } @@ -4758,6 +4772,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (!pInfo->pUpdated) { pInfo->pUpdated = taosArrayInit(4, sizeof(SWinKey)); } + if (!pInfo->pUpdatedMap) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pUpdatedMap = tSimpleHashInit(1024, hashFn); @@ -4770,6 +4785,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->numOfDatapack = 0; break; } + pInfo->numOfDatapack++; printDataBlock(pBlock, "single interval recv"); @@ -4805,10 +4821,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); } + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); - - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); } pOperator->status = OP_RES_TO_RETURN; removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); @@ -4887,6 +4902,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->interval = interval; pInfo->twAggSup = twAggSupp; pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; + pInfo->ignoreExpiredDataSaved = false; pInfo->isFinal = false; SExprSupp* pSup = &pOperator->exprSupp; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6d734901ab3e43a165bdba05690abd222ff96385..6c8e581b3fe837aa1bf101bf8cf6748a2cac5f4d 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -108,12 +108,18 @@ static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { return TSDB_CODE_SUCCESS; } -void tsortClearOrderdSource(SArray* pOrderedSource) { +void tsortClearOrderdSource(SArray* pOrderedSource, int64_t *fetchUs, int64_t *fetchNum) { for (size_t i = 0; i < taosArrayGetSize(pOrderedSource); i++) { SSortSource** pSource = taosArrayGet(pOrderedSource, i); if (NULL == *pSource) { continue; } + + if (fetchUs) { + *fetchUs += (*pSource)->fetchUs; + *fetchNum += (*pSource)->fetchNum; + } + // release pageIdList if ((*pSource)->pageIdList) { taosArrayDestroy((*pSource)->pageIdList); @@ -147,7 +153,10 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { taosMemoryFreeClear(pSortHandle->idStr); blockDataDestroy(pSortHandle->pDataBlock); - tsortClearOrderdSource(pSortHandle->pOrderedSource); + int64_t fetchUs = 0, fetchNum = 0; + tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum); + qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); + taosArrayDestroy(pSortHandle->pOrderedSource); taosMemoryFreeClear(pSortHandle); } @@ -214,7 +223,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { if (pPage == NULL) { taosArrayDestroy(pPageIdList); blockDataDestroy(p); - taosArrayDestroy(pPageIdList); return terrno; } @@ -308,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 } int64_t et = taosGetTimestampUs(); - qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); + qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; @@ -366,7 +374,10 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT releaseBufPage(pHandle->pBuf, pPage); } } else { + int64_t st = taosGetTimestampUs(); pSource->src.pBlock = pHandle->fetchfp(((SSortSource*)pSource)->param); + pSource->fetchUs += taosGetTimestampUs() - st; + pSource->fetchNum++; if (pSource->src.pBlock == NULL) { (*numOfCompleted) += 1; pSource->src.rowIndex = -1; @@ -603,7 +614,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { } } - tsortClearOrderdSource(pHandle->pOrderedSource); + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); taosArrayAddAll(pHandle->pOrderedSource, pResList); taosArrayDestroy(pResList); @@ -645,7 +656,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { SSortSource* source = *pSource; *pSource = NULL; - tsortClearOrderdSource(pHandle->pOrderedSource); + tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL); while (1) { SSDataBlock* pBlock = pHandle->fetchfp(source->param); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index dc884a058193e0abbefa04cff086788733f59a87..c3afc30a7b203469b0da23244dc281e64f32bd69 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -235,6 +235,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getGroupKeyFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t groupKeyFunction(SqlFunctionCtx* pCtx); int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index c69d19b8a6187ad1d6817a6b5f8793b0905817e4..27d3b7930f80905f76704250765dc63040d33353 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -41,7 +41,7 @@ typedef struct SUdfSetupRequest { typedef struct SUdfSetupResponse { int64_t udfHandle; int8_t outputType; - int32_t outputLen; + int32_t bytes; int32_t bufSize; } SUdfSetupResponse; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0257b3d5e6c2e0062b00b79dab221f4ddb606d4a..fe98a1dd537af5fc48b66c431dcfe524fdefe972 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -213,8 +213,9 @@ static int32_t addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); struct tm tmInfo; - taosLocalTime(&t, &tmInfo); - strftime(buf, sizeof(buf), "%z", &tmInfo); + if (taosLocalTime(&t, &tmInfo, buf) != NULL) { + strftime(buf, sizeof(buf), "%z", &tmInfo); + } int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -480,14 +481,16 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = + (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } @@ -509,13 +512,11 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_NUMERIC_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - for (int32_t i = 1; i < numOfParams; ++i) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i); pValue->notReserved = true; @@ -1574,7 +1575,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 0)); uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType) || QUERY_NODE_VALUE == nodeType) { + if ((!IS_NUMERIC_TYPE(paraType) && !IS_BOOLEAN_TYPE(paraType))|| QUERY_NODE_VALUE == nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1932,14 +1933,35 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l } static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + int16_t resType = TSDB_DATA_TYPE_BIGINT; + + if (1 != numOfParams && 2 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (2 == numOfParams) { + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + if (pValue->datum.i == 1) { + resType = TSDB_DATA_TYPE_TIMESTAMP; + } else if (pValue->datum.i == 0) { + resType = TSDB_DATA_TYPE_BIGINT; + } else { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TO_UNIXTIMESTAMP function second parameter should be 0/1"); + } + } + // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); @@ -1947,7 +1969,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -2375,7 +2397,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "spread", .type = FUNCTION_TYPE_SPREAD, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .translateFunc = translateSpread, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getSpreadFuncEnv, @@ -2417,7 +2439,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, @@ -2457,7 +2479,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "interp", .type = FUNCTION_TYPE_INTERP, .classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateInterp, .getEnvFunc = getSelectivityFuncEnv, .initFunc = functionSetup, @@ -2480,7 +2502,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "irate", .type = FUNCTION_TYPE_IRATE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateIrate, .getEnvFunc = getIrateFuncEnv, .initFunc = irateFuncSetup, @@ -2491,7 +2513,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateFirstLast, .dynDataRequiredFunc = lastDynDataReq, .getEnvFunc = getFirstLastFuncEnv, @@ -2500,7 +2522,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = firstLastScalarFunction, .pPartialFunc = "_last_row_partial", .pMergeFunc = "_last_row_merge", - .finalizeFunc = firstLastFinalize + .finalizeFunc = firstLastFinalize, + .combineFunc = lastCombine }, { .name = "_cache_last_row", @@ -2809,7 +2832,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mode", .type = FUNCTION_TYPE_MODE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMode, .getEnvFunc = getModeFuncEnv, .initFunc = modeFunctionSetup, @@ -3212,7 +3235,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_block_dist", .type = FUNCTION_TYPE_BLOCK_DIST, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateBlockDistFunc, .getEnvFunc = getBlockDistFuncEnv, .initFunc = blockDistSetup, @@ -3234,6 +3257,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = groupKeyFunction, .finalizeFunc = groupKeyFinalize, + .combineFunc = groupKeyCombine, .pPartialFunc = "_group_key", .pMergeFunc = "_group_key" }, @@ -3276,7 +3300,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_irowts", .type = FUNCTION_TYPE_IROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fbc60041b242ed039e873bd5877e3c5c8a4f3828..a8ecd9b0a2f0da0900673448233c8cfc51af01c4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -32,9 +32,6 @@ #define TAIL_MAX_POINTS_NUM 100 #define TAIL_MAX_OFFSET 100 -#define UNIQUE_MAX_RESULT_SIZE (1024 * 1024 * 10) -#define MODE_MAX_RESULT_SIZE UNIQUE_MAX_RESULT_SIZE - #define HLL_BUCKET_BITS 14 // The bits of the bucket #define HLL_DATA_BITS (64 - HLL_BUCKET_BITS) #define HLL_BUCKETS (1 << HLL_BUCKET_BITS) @@ -244,12 +241,11 @@ typedef struct SUniqueInfo { typedef struct SModeItem { int64_t count; + STuplePos dataPos; STuplePos tuplePos; - char data[]; } SModeItem; typedef struct SModeInfo { - int32_t numOfPoints; uint8_t colType; int16_t colBytes; SHashObj* pHash; @@ -257,7 +253,7 @@ typedef struct SModeInfo { STuplePos nullTuplePos; bool nullTupleSaved; - char pItems[]; + char* buf; // serialize data buffer } SModeInfo; typedef struct SDerivInfo { @@ -494,8 +490,8 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; +static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { + int64_t numOfElem = 0; /* * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataSMAIsSet == true; @@ -528,7 +524,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; + int64_t numOfElem = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -555,7 +551,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { } int32_t countInvertFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumOfElems(pCtx); + int64_t numOfElem = getNumOfElems(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); @@ -871,6 +867,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); @@ -1929,7 +1931,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - qDebug("%s total %d rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); + qDebug("%s total %" PRId64 " rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { @@ -3091,6 +3093,12 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid for (int32_t i = 0; i < pSubsidiaryies->num; ++i) { SqlFunctionCtx* pc = pSubsidiaryies->pCtx[i]; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0]; int32_t srcSlotId = pFuncParam->pCol->slotId; @@ -3113,7 +3121,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid return buf; } -static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key, +static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey* key, STuplePos* pPos) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { @@ -3149,8 +3157,8 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy - if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) >= 0) { - p.streamTupleKey = key; + if (streamStateFuncPut(pHandle->pState, key, pBuf, length) >= 0) { + p.streamTupleKey = *key; } } @@ -3174,7 +3182,7 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key, pPos); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, &key, pPos); } static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { @@ -3345,7 +3353,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) { goto _spread_over; } double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) { tmin = (double)GET_INT64_VAL(&pAgg->min); tmax = (double)GET_INT64_VAL(&pAgg->max); } else if (IS_FLOAT_TYPE(type)) { @@ -4949,7 +4957,7 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { } bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - pEnv->calcMemSize = sizeof(SModeInfo) + MODE_MAX_RESULT_SIZE; + pEnv->calcMemSize = sizeof(SModeInfo); return true; } @@ -4959,7 +4967,6 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { } SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->numOfPoints = 0; pInfo->colType = pCtx->resDataInfo.type; pInfo->colBytes = pCtx->resDataInfo.bytes; if (pInfo->pHash != NULL) { @@ -4970,38 +4977,60 @@ bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->nullTupleSaved = false; pInfo->nullTuplePos.pageId = -1; + pInfo->buf = taosMemoryMalloc(pInfo->colBytes); + return true; } +static void modeFunctionCleanup(SModeInfo * pInfo) { + taosHashCleanup(pInfo->pHash); + taosMemoryFreeClear(pInfo->buf); +} + +static int32_t saveModeTupleData(SqlFunctionCtx* pCtx, char* data, SModeInfo *pInfo, STuplePos* pPos) { + if (IS_VAR_DATA_TYPE(pInfo->colType)) { + memcpy(pInfo->buf, data, varDataTLen(data)); + } else { + memcpy(pInfo->buf, data, pInfo->colBytes); + } + + return doSaveTupleData(&pCtx->saveHandle, pInfo->buf, pInfo->colBytes, NULL, pPos); +} + static int32_t doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx, char* data) { - int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; - SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); + int32_t code = TSDB_CODE_SUCCESS; + int32_t hashKeyBytes = IS_STR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + + SModeItem* pHashItem = (SModeItem *)taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { - int32_t size = sizeof(SModeItem) + pInfo->colBytes; - SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); - memcpy(pItem->data, data, hashKeyBytes); - pItem->count += 1; + int32_t size = sizeof(SModeItem); + SModeItem item = {0}; + + item.count += 1; + code = saveModeTupleData(pCtx, data, pInfo, &item.dataPos); + if (code != TSDB_CODE_SUCCESS) { + return code; + } if (pCtx->subsidiaries.num > 0) { - int32_t code = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pItem->tuplePos); + code = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &item.tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } } - taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); - pInfo->numOfPoints++; + taosHashPut(pInfo->pHash, data, hashKeyBytes, &item, sizeof(SModeItem)); } else { - (*pHashItem)->count += 1; + pHashItem->count += 1; if (pCtx->subsidiaries.num > 0) { - int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &((*pHashItem)->tuplePos)); + int32_t code = updateTupleData(pCtx, rowIndex, pCtx->pSrcBlock, &pHashItem->tuplePos); if (code != TSDB_CODE_SUCCESS) { return code; } } } - return TSDB_CODE_SUCCESS; + return code; } int32_t modeFunction(SqlFunctionCtx* pCtx) { @@ -5024,18 +5053,15 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); int32_t code = doModeAdd(pInfo, i, pCtx, data); if (code != TSDB_CODE_SUCCESS) { + modeFunctionCleanup(pInfo); return code; } - - if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { - taosHashCleanup(pInfo->pHash); - return TSDB_CODE_OUT_OF_MEMORY; - } } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { int32_t code = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); if (code != TSDB_CODE_SUCCESS) { + modeFunctionCleanup(pInfo); return code; } pInfo->nullTupleSaved = true; @@ -5054,26 +5080,37 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; - int32_t resIndex = -1; + STuplePos resDataPos, resTuplePos; int32_t maxCount = 0; - for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { - SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes)); + + void *pIter = taosHashIterate(pInfo->pHash, NULL); + while (pIter != NULL) { + SModeItem *pItem = (SModeItem *)pIter; if (pItem->count >= maxCount) { maxCount = pItem->count; - resIndex = i; + resDataPos = pItem->dataPos; + resTuplePos = pItem->tuplePos; } + + pIter = taosHashIterate(pInfo->pHash, pIter); } if (maxCount != 0) { - SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataSetVal(pCol, currentRow, pResItem->data, false); - code = setSelectivityValue(pCtx, pBlock, &pResItem->tuplePos, currentRow); + const char* pData = loadTupleData(pCtx, &resDataPos); + if (pData == NULL) { + code = TSDB_CODE_NO_AVAIL_DISK; + modeFunctionCleanup(pInfo); + return code; + } + + colDataSetVal(pCol, currentRow, pData, false); + code = setSelectivityValue(pCtx, pBlock, &resTuplePos, currentRow); } else { colDataSetNULL(pCol, currentRow); code = setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); } - taosHashCleanup(pInfo->pHash); + modeFunctionCleanup(pInfo); return code; } @@ -5900,6 +5937,39 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t groupKeyCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { + SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); + SGroupKeyInfo* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); + + SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); + SGroupKeyInfo* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); + + // escape rest of data blocks to avoid first entry to be overwritten. + if (pDBuf->hasResult) { + goto _group_key_over; + } + + if (pSBuf->isNull) { + pDBuf->isNull = true; + pDBuf->hasResult = true; + goto _group_key_over; + } + + if (IS_VAR_DATA_TYPE(pSourceCtx->resDataInfo.type)) { + memcpy(pDBuf->data, pSBuf->data, + (pSourceCtx->resDataInfo.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(pSBuf->data) : varDataTLen(pSBuf->data)); + } else { + memcpy(pDBuf->data, pSBuf->data, pSourceCtx->resDataInfo.bytes); + } + + pDBuf->hasResult = true; + +_group_key_over: + + SET_VAL(pDResInfo, 1, 1); + return TSDB_CODE_SUCCESS; +} + int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index e3127fcd7ba233d478c58cc16b7543587b5eaf90..18f6e8050b56e1c73a28cf64bb24c1da3844651f 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -174,6 +174,8 @@ bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } +bool fmIsDateTimeFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DATETIME_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } @@ -184,6 +186,7 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWi bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); } + bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); } @@ -447,3 +450,10 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc return code; } + +char* fmGetFuncName(int32_t funcId) { + if (fmIsUserDefinedFunc(funcId) || funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return taosStrdup("invalid function"); + } + return taosStrdup(funcMgtBuiltins[funcId].name); +} diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 8f5cd070dca186651d552ce2cd274856a703e119..6b70422ac8fe20bf8089eb8efba11ecc3c5e6ae5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -343,7 +343,7 @@ typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN + 1]; UdfcFuncHandle handle; int32_t refCount; - int64_t lastRefTime; + int64_t createTime; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -363,6 +363,7 @@ typedef struct SUdfcProxy { uv_mutex_t udfStubsMutex; SArray *udfStubs; // SUdfcFuncStub + SArray *expiredUdfStubs; //SUdfcFuncStub uv_mutex_t udfcUvMutex; int8_t initialized; @@ -376,7 +377,7 @@ typedef struct SUdfcUvSession { uv_pipe_t *udfUvPipe; int8_t outputType; - int32_t outputLen; + int32_t bytes; int32_t bufSize; char udfName[TSDB_FUNC_NAME_LEN + 1]; @@ -614,7 +615,7 @@ int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) { int32_t len = 0; len += taosEncodeFixedI64(buf, setupRsp->udfHandle); len += taosEncodeFixedI8(buf, setupRsp->outputType); - len += taosEncodeFixedI32(buf, setupRsp->outputLen); + len += taosEncodeFixedI32(buf, setupRsp->bytes); len += taosEncodeFixedI32(buf, setupRsp->bufSize); return len; } @@ -622,7 +623,7 @@ int32_t encodeUdfSetupResponse(void **buf, const SUdfSetupResponse *setupRsp) { void *decodeUdfSetupResponse(const void *buf, SUdfSetupResponse *setupRsp) { buf = taosDecodeFixedI64(buf, &setupRsp->udfHandle); buf = taosDecodeFixedI8(buf, &setupRsp->outputType); - buf = taosDecodeFixedI32(buf, &setupRsp->outputLen); + buf = taosDecodeFixedI32(buf, &setupRsp->bytes); buf = taosDecodeFixedI32(buf, &setupRsp->bufSize); return (void *)buf; } @@ -808,6 +809,26 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo } int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { + SUdfColumnMeta* meta = &udfCol->colMeta; + + SColumnInfoData colInfoData = createColumnInfoData(meta->type, meta->bytes, 1); + blockDataAppendColInfo(block, &colInfoData); + blockDataEnsureCapacity(block, udfCol->colData.numOfRows); + + SColumnInfoData *col = bdGetColumnInfoData(block, 0); + for (int i = 0; i < udfCol->colData.numOfRows; ++i) { + if (udfColDataIsNull(udfCol, i)) { + colDataSetNULL(col, i); + } else { + char* data = udfColDataGetData(udfCol, i); + colDataSetVal(col, i, data, false); + } + } + block->info.rows = udfCol->colData.numOfRows; + return 0; +} + +int32_t convertUdfColumnToDataBlock2(SUdfColumn *udfCol, SSDataBlock *block) { block->info.rows = udfCol->colData.numOfRows; block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); @@ -899,9 +920,11 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { typedef struct SUdfAggRes { int8_t finalResNum; int8_t interResNum; + int32_t interResBufLen; char *finalResBuf; char *interResBuf; } SUdfAggRes; + void onUdfcPipeClose(uv_handle_t *handle); int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask); void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf); @@ -937,7 +960,7 @@ int32_t udfcOpen(); int32_t udfcClose(); int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle); -void releaseUdfFuncHandle(char *udfName); +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle); int32_t cleanUpUdfs(); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); @@ -945,6 +968,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); +void cleanupNotExpiredUdfs(); +void cleanupExpiredUdfs(); int compareUdfcFuncSub(const void *elem1, const void *elem2) { SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; @@ -960,16 +985,24 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return 0; + int64_t currUs = taosGetTimestampUs(); + bool expired = (currUs - foundStub->createTime) >= 10 * 1000 * 1000; + if (!expired) { + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return 0; + } else { + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + foundStub->refCount, foundStub->createTime); + taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + } } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", udfName, - foundStub->refCount, foundStub->lastRefTime); + fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub); + taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub); } } *pHandle = NULL; @@ -979,7 +1012,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); stub.handle = *pHandle; ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); + stub.createTime = taosGetTimestampUs(); taosArrayPush(gUdfcProxy.udfStubs, &stub); taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub); } else { @@ -990,32 +1023,51 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { return code; } -void releaseUdfFuncHandle(char *udfName) { +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { + SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub && !expiredStub) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return; } - if (foundStub->refCount > 0) { + if (foundStub != NULL && foundStub->handle == handle && foundStub->refCount > 0) { --foundStub->refCount; } + if (expiredStub != NULL && expiredStub->handle == handle && expiredStub->refCount > 0) { + --expiredStub->refCount; + } uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); } -int32_t cleanUpUdfs() { - int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); - if (!initialized) { - return TSDB_CODE_SUCCESS; +void cleanupExpiredUdfs() { + int32_t i = 0; + SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + taosArrayPush(expiredUdfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); + } + } + ++i; } + taosArrayDestroy(gUdfcProxy.expiredUdfStubs); + gUdfcProxy.expiredUdfStubs = expiredUdfStubs; +} - uv_mutex_lock(&gUdfcProxy.udfStubsMutex); - if (gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) { - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return TSDB_CODE_SUCCESS; - } +void cleanupNotExpiredUdfs() { SArray *udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); int32_t i = 0; while (i < taosArrayGetSize(gUdfcProxy.udfStubs)) { @@ -1024,20 +1076,38 @@ int32_t cleanUpUdfs() { fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %" PRId64 ", handle: %p", stub->udfName, - stub->refCount, stub->lastRefTime, stub->handle); + fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); UdfcFuncHandle handle = stub->handle; if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { taosArrayPush(udfStubs, stub); } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); } } ++i; } taosArrayDestroy(gUdfcProxy.udfStubs); gUdfcProxy.udfStubs = udfStubs; +} + +int32_t cleanUpUdfs() { + int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); + if (!initialized) { + return TSDB_CODE_SUCCESS; + } + + uv_mutex_lock(&gUdfcProxy.udfStubsMutex); + if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && + (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return TSDB_CODE_SUCCESS; + } + + cleanupNotExpiredUdfs(); + cleanupExpiredUdfs(); + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } @@ -1048,19 +1118,26 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, if (code != 0) { return code; } + SUdfcUvSession *session = handle; code = doCallUdfScalarFunc(handle, input, numOfCols, output); + if (code != TSDB_CODE_SUCCESS) { + fnError("udfc scalar function execution failure"); + releaseUdfFuncHandle(udfName, handle); + return code; + } + if (output->columnData == NULL) { fnError("udfc scalar function calculate error. no column data"); code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } else { - if (session->outputType != output->columnData->info.type || session->outputLen != output->columnData->info.bytes) { + if (session->outputType != output->columnData->info.type || session->bytes != output->columnData->info.bytes) { fnError("udfc scalar function calculate error. type mismatch. session type: %d(%d), output type: %d(%d)", - session->outputType, session->outputLen, output->columnData->info.type, output->columnData->info.bytes); + session->outputType, session->bytes, output->columnData->info.type, output->columnData->info.bytes); code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } } - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1084,27 +1161,28 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult } SUdfcUvSession *session = (SUdfcUvSession *)handle; SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(pResultCellInfo); - int32_t envSize = sizeof(SUdfAggRes) + session->outputLen + session->bufSize; + int32_t envSize = sizeof(SUdfAggRes) + session->bytes + session->bufSize; memset(udfRes, 0, envSize); udfRes->finalResBuf = (char *)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->bytes; SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } - udfRes->interResNum = buf.numOfResult; if (buf.bufLen <= session->bufSize) { memcpy(udfRes->interResBuf, buf.buf, buf.bufLen); + udfRes->interResBufLen = buf.bufLen; + udfRes->interResNum = buf.numOfResult; } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&buf); return true; } @@ -1120,7 +1198,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { SUdfcUvSession *session = handle; SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char *)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->bytes; SInputColumnInfoData *pInput = &pCtx->input; int32_t numOfCols = pInput->numOfInputCols; @@ -1136,7 +1214,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows); - SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; + SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = udfRes->interResBufLen, .numOfResult = udfRes->interResNum}; SUdfInterBuf newState = {0}; udfCode = doCallUdfAggProcess(session, inputBlock, &state, &newState); @@ -1144,24 +1222,24 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { fnError("udfAggProcess error. code: %d", udfCode); newState.numOfResult = 0; } else { - udfRes->interResNum = newState.numOfResult; if (newState.bufLen <= session->bufSize) { memcpy(udfRes->interResBuf, newState.buf, newState.bufLen); + udfRes->interResBufLen = newState.bufLen; + udfRes->interResNum = newState.numOfResult; } else { fnError("udfc inter buf size %d is greater than function bufSize %d", newState.bufLen, session->bufSize); udfCode = TSDB_CODE_UDF_INVALID_BUFSIZE; } } - if (newState.numOfResult == 1 || state.numOfResult == 1) { - GET_RES_INFO(pCtx)->numOfRes = 1; - } + + GET_RES_INFO(pCtx)->numOfRes = udfRes->interResNum; blockDataDestroy(inputBlock); taosArrayDestroy(pTempBlock->pDataBlock); taosMemoryFree(pTempBlock); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&newState); return udfCode; } @@ -1177,31 +1255,36 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) { SUdfcUvSession *session = handle; SUdfAggRes *udfRes = (SUdfAggRes *)GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); udfRes->finalResBuf = (char *)udfRes + sizeof(SUdfAggRes); - udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->outputLen; + udfRes->interResBuf = (char *)udfRes + sizeof(SUdfAggRes) + session->bytes; SUdfInterBuf resultBuf = {0}; - SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, .numOfResult = udfRes->interResNum}; + SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = udfRes->interResBufLen, .numOfResult = udfRes->interResNum}; int32_t udfCallCode = 0; udfCallCode = doCallUdfAggFinalize(session, &state, &resultBuf); if (udfCallCode != 0) { fnError("udfAggFinalize error. doCallUdfAggFinalize step. udf code:%d", udfCallCode); GET_RES_INFO(pCtx)->numOfRes = 0; } else { - if (resultBuf.bufLen <= session->outputLen) { - memcpy(udfRes->finalResBuf, resultBuf.buf, session->outputLen); - udfRes->finalResNum = resultBuf.numOfResult; - GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; - } else { - fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->outputLen); + if (resultBuf.numOfResult == 0) { + udfRes->finalResNum = 0; GET_RES_INFO(pCtx)->numOfRes = 0; - udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } else { + if (resultBuf.bufLen <= session->bytes) { + memcpy(udfRes->finalResBuf, resultBuf.buf, resultBuf.bufLen); + udfRes->finalResNum = resultBuf.numOfResult; + GET_RES_INFO(pCtx)->numOfRes = udfRes->finalResNum; + } else { + fnError("udfc inter buf size %d is greater than function output size %d", resultBuf.bufLen, session->bytes); + GET_RES_INFO(pCtx)->numOfRes = 0; + udfCallCode = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; + } } } freeUdfInterBuf(&resultBuf); int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return udfCallCode == 0 ? numOfResults : udfCallCode; } @@ -1365,8 +1448,9 @@ void udfcUvHandleError(SClientUvConn *conn) { QUEUE_REMOVE(&task->procTaskQueue); uv_sem_post(&task->taskSem); } - - uv_close((uv_handle_t *)conn->pipe, onUdfcPipeClose); + if (!uv_is_closing((uv_handle_t *)conn->pipe)) { + uv_close((uv_handle_t *)conn->pipe, onUdfcPipeClose); + } } void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -1496,7 +1580,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = pipe->data; @@ -1514,11 +1598,13 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_DISCONNECT: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); - uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose); + if (!uv_is_closing((uv_handle_t *)uvTask->pipe)) { + uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose); + } code = 0; } break; @@ -1625,6 +1711,7 @@ int32_t udfcOpen() { uv_barrier_wait(&proxy->initBarrier); uv_mutex_init(&proxy->udfStubsMutex); proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); + proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); uv_mutex_init(&proxy->udfcUvMutex); fnInfo("udfc initialized") return 0; } @@ -1641,6 +1728,7 @@ int32_t udfcClose() { uv_thread_join(&udfc->loopThread); uv_mutex_destroy(&udfc->taskQueueMutex); uv_barrier_destroy(&udfc->initBarrier); + taosArrayDestroy(udfc->expiredUdfStubs); taosArrayDestroy(udfc->udfStubs); uv_mutex_destroy(&udfc->udfStubsMutex); uv_mutex_destroy(&udfc->udfcUvMutex); @@ -1671,9 +1759,6 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { } int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { - if (gUdfcProxy.udfcState != UDFC_STATE_READY) { - return TSDB_CODE_UDF_INVALID_STATE; - } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); @@ -1696,13 +1781,13 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; task->session->outputType = rsp->outputType; - task->session->outputLen = rsp->outputLen; + task->session->bytes = rsp->bytes; task->session->bufSize = rsp->bufSize; strncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN); if (task->errCode != 0) { fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode) } else { - fnInfo("sucessfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); + fnInfo("successfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session); *funcHandle = task->session; } int32_t err = task->errCode; @@ -1716,7 +1801,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SUdfcUvSession *session = (SUdfcUvSession *)handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; @@ -1840,7 +1925,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if (session->udfUvPipe == NULL) { fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); taosMemoryFree(session); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index dc6fc3ad7420ae7ebe08feecf336aa57121324c3..5034af2f8264dab3312950369c15d4fb890b38b7 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -21,6 +21,7 @@ #include "tudf.h" #include "tudfInt.h" +#include "version.h" #include "tdatablock.h" #include "tdataformat.h" @@ -30,6 +31,186 @@ #include "tmisce.h" // clang-format on +#define UDFD_MAX_SCRIPT_PLUGINS 64 +#define UDFD_MAX_SCRIPT_TYPE 1 +#define UDFD_MAX_PLUGIN_FUNCS 9 + +typedef struct SUdfCPluginCtx { + uv_lib_t lib; + + TUdfScalarProcFunc scalarProcFunc; + + TUdfAggStartFunc aggStartFunc; + TUdfAggProcessFunc aggProcFunc; + TUdfAggFinishFunc aggFinishFunc; + TUdfAggMergeFunc aggMergeFunc; + + TUdfInitFunc initFunc; + TUdfDestroyFunc destroyFunc; +} SUdfCPluginCtx; + +int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; } + +int32_t udfdCPluginClose() { return 0; } + +const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { + char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); + return udfName; +} + +void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { + char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(processFuncName, udfName); + uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); + + char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *startSuffix = "_start"; + strncpy(startFuncName, processFuncName, sizeof(startFuncName)); + strncat(startFuncName, startSuffix, strlen(startSuffix)); + uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)); + + char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; + char *finishSuffix = "_finish"; + strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); + strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)); + + char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *mergeSuffix = "_merge"; + strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); + strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); +} + +int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { + int32_t err = 0; + SUdfCPluginCtx *udfCtx = taosMemoryCalloc(1, sizeof(SUdfCPluginCtx)); + err = uv_dlopen(udf->path, &udfCtx->lib); + if (err != 0) { + fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); + return TSDB_CODE_UDF_LOAD_UDF_FAILURE; + } + const char *udfName = udf->name; + + udfdCPluginUdfInitLoadInitDestoryFuncs(udfCtx, udfName); + + if (udf->funcType == UDF_FUNC_TYPE_SCALAR) { + char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(processFuncName, udfName); + uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)); + } else if (udf->funcType == UDF_FUNC_TYPE_AGG) { + udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName); + } + + int32_t code = 0; + if (udfCtx->initFunc) { + code = (udfCtx->initFunc)(); + if (code != 0) { + uv_dlclose(&udfCtx->lib); + taosMemoryFree(udfCtx); + return code; + } + } + *pUdfCtx = udfCtx; + return 0; +} + +int32_t udfdCPluginUdfDestroy(void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + int32_t code = 0; + if (ctx->destroyFunc) { + code = (ctx->destroyFunc)(); + } + uv_dlclose(&ctx->lib); + taosMemoryFree(ctx); + return code; +} + +int32_t udfdCPluginUdfScalarProc(SUdfDataBlock *block, SUdfColumn *resultCol, void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + if (ctx->scalarProcFunc) { + return ctx->scalarProcFunc(block, resultCol); + } else { + fnError("udfd c plugin scalar proc not implemented"); + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } +} + +int32_t udfdCPluginUdfAggStart(SUdfInterBuf *buf, void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + if (ctx->aggStartFunc) { + return ctx->aggStartFunc(buf); + } else { + fnError("udfd c plugin aggregation start not implemented"); + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } + return 0; +} + +int32_t udfdCPluginUdfAggProc(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf, void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + if (ctx->aggProcFunc) { + return ctx->aggProcFunc(block, interBuf, newInterBuf); + } else { + fnError("udfd c plugin aggregation process not implemented"); + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } +} + +int32_t udfdCPluginUdfAggMerge(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf, + void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + if (ctx->aggMergeFunc) { + return ctx->aggMergeFunc(inputBuf1, inputBuf2, outputBuf); + } else { + fnError("udfd c plugin aggregation merge not implemented"); + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } +} + +int32_t udfdCPluginUdfAggFinish(SUdfInterBuf *buf, SUdfInterBuf *resultData, void *udfCtx) { + SUdfCPluginCtx *ctx = udfCtx; + if (ctx->aggFinishFunc) { + return ctx->aggFinishFunc(buf, resultData); + } else { + fnError("udfd c plugin aggregation finish not implemented"); + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } + return 0; +} + +// for c, the function pointer are filled directly and libloaded = true; +// for others, dlopen/dlsym to find function pointers +typedef struct SUdfScriptPlugin { + int8_t scriptType; + + char libPath[PATH_MAX]; + bool libLoaded; + uv_lib_t lib; + + TScriptUdfScalarProcFunc udfScalarProcFunc; + TScriptUdfAggStartFunc udfAggStartFunc; + TScriptUdfAggProcessFunc udfAggProcFunc; + TScriptUdfAggMergeFunc udfAggMergeFunc; + TScriptUdfAggFinishFunc udfAggFinishFunc; + + TScriptUdfInitFunc udfInitFunc; + TScriptUdfDestoryFunc udfDestroyFunc; + + TScriptOpenFunc openFunc; + TScriptCloseFunc closeFunc; +} SUdfScriptPlugin; + typedef struct SUdfdContext { uv_loop_t *loop; uv_pipe_t ctrlPipe; @@ -37,13 +218,18 @@ typedef struct SUdfdContext { char listenPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; uv_pipe_t listeningPipe; - void *clientRpc; - SCorEpSet mgmtEp; + void *clientRpc; + SCorEpSet mgmtEp; + uv_mutex_t udfsMutex; SHashObj *udfsHash; + uv_mutex_t scriptPluginsMutex; + SUdfScriptPlugin *scriptPlugins[UDFD_MAX_SCRIPT_PLUGINS]; + SArray *residentFuncs; + char udfDataDir[PATH_MAX]; bool printVersion; } SUdfdContext; @@ -70,16 +256,13 @@ typedef struct SUvUdfWork { struct SUvUdfWork *pWorkNext; } SUvUdfWork; -typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState; +typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY } EUdfState; typedef struct SUdf { - int32_t refCount; - EUdfState state; - uv_mutex_t lock; - uv_cond_t condReady; - bool resident; - char name[TSDB_FUNC_NAME_LEN + 1]; + int32_t version; + int64_t createdTime; + int8_t funcType; int8_t scriptType; int8_t outputType; @@ -88,20 +271,19 @@ typedef struct SUdf { char path[PATH_MAX]; - uv_lib_t lib; - - TUdfScalarProcFunc scalarProcFunc; + int32_t refCount; + EUdfState state; + uv_mutex_t lock; + uv_cond_t condReady; + bool resident; - TUdfAggStartFunc aggStartFunc; - TUdfAggProcessFunc aggProcFunc; - TUdfAggFinishFunc aggFinishFunc; - TUdfAggMergeFunc aggMergeFunc; + SUdfScriptPlugin *scriptPlugin; + void *scriptUdfCtx; - TUdfInitFunc initFunc; - TUdfDestroyFunc destroyFunc; + int64_t lastFetchTime; // last fetch time in milliseconds + bool expired; } SUdf; -// TODO: add private udf structure. typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; @@ -121,7 +303,6 @@ typedef struct SUdfdRpcSendRecvInfo { static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); -static int32_t udfdLoadUdf(char *udfName, SUdf *udf); static bool udfdRpcRfp(int32_t code, tmsg_t msgType); static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); @@ -155,6 +336,159 @@ static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); static void udfdConnectMnodeThreadFunc(void *args); +SUdf *udfdNewUdf(const char *udfName); +void udfdGetFuncBodyPath(const SUdf *udf, char *path); + +void udfdInitializeCPlugin(SUdfScriptPlugin *plugin) { + plugin->scriptType = TSDB_FUNC_SCRIPT_BIN_LIB; + plugin->openFunc = udfdCPluginOpen; + plugin->closeFunc = udfdCPluginClose; + plugin->udfInitFunc = udfdCPluginUdfInit; + plugin->udfDestroyFunc = udfdCPluginUdfDestroy; + plugin->udfScalarProcFunc = udfdCPluginUdfScalarProc; + plugin->udfAggStartFunc = udfdCPluginUdfAggStart; + plugin->udfAggProcFunc = udfdCPluginUdfAggProc; + plugin->udfAggMergeFunc = udfdCPluginUdfAggMerge; + plugin->udfAggFinishFunc = udfdCPluginUdfAggFinish; + + SScriptUdfEnvItem items[1] = {{"LD_LIBRARY_PATH", tsUdfdLdLibPath}}; + plugin->openFunc(items, 1); + return; +} + +int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[], void **func[], int numOfFuncs) { + int err = uv_dlopen(libPath, pLib); + if (err != 0) { + fnError("can not load library %s. error: %s", libPath, uv_strerror(err)); + return TSDB_CODE_UDF_LOAD_UDF_FAILURE; + } + + for (int i = 0; i < numOfFuncs; ++i) { + err = uv_dlsym(pLib, funcName[i], func[i]); + if (err != 0) { + fnError("load library function failed. lib %s function %s", libPath, funcName[i]); + } + } + return 0; +} + +int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) { + plugin->scriptType = TSDB_FUNC_SCRIPT_PYTHON; + // todo: windows support + sprintf(plugin->libPath, "%s", "libtaospyudf.so"); + plugin->libLoaded = false; + const char *funcName[UDFD_MAX_PLUGIN_FUNCS] = {"pyOpen", "pyClose", "pyUdfInit", + "pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart", + "pyUdfAggFinish", "pyUdfAggProc", "pyUdfAggMerge"}; + void **funcs[UDFD_MAX_PLUGIN_FUNCS] = { + (void **)&plugin->openFunc, (void **)&plugin->closeFunc, (void **)&plugin->udfInitFunc, + (void **)&plugin->udfDestroyFunc, (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc, + (void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc, (void **)&plugin->udfAggMergeFunc}; + int32_t err = udfdLoadSharedLib(plugin->libPath, &plugin->lib, funcName, funcs, UDFD_MAX_PLUGIN_FUNCS); + if (err != 0) { + fnError("can not load python plugin. lib path %s", plugin->libPath); + return err; + } + + if (plugin->openFunc) { + int16_t lenPythonPath = + strlen(tsUdfdLdLibPath) + strlen(global.udfDataDir) + 1 + 1; // global.udfDataDir:tsUdfdLdLibPath + char *pythonPath = taosMemoryMalloc(lenPythonPath); +#ifdef WINDOWS + snprintf(pythonPath, lenPythonPath, "%s;%s", global.udfDataDir, tsUdfdLdLibPath); +#else + snprintf(pythonPath, lenPythonPath, "%s:%s", global.udfDataDir, tsUdfdLdLibPath); +#endif + SScriptUdfEnvItem items[] = {{"PYTHONPATH", pythonPath}, {"LOGDIR", tsLogDir}}; + err = plugin->openFunc(items, 2); + taosMemoryFree(pythonPath); + } + if (err != 0) { + fnError("udf script python plugin open func failed. error: %d", err); + uv_dlclose(&plugin->lib); + return err; + } + plugin->libLoaded = true; + + return 0; +} + +void udfdDeinitCPlugin(SUdfScriptPlugin *plugin) { + if (plugin->closeFunc) { + plugin->closeFunc(); + } + plugin->openFunc = NULL; + plugin->closeFunc = NULL; + plugin->udfInitFunc = NULL; + plugin->udfDestroyFunc = NULL; + plugin->udfScalarProcFunc = NULL; + plugin->udfAggStartFunc = NULL; + plugin->udfAggProcFunc = NULL; + plugin->udfAggMergeFunc = NULL; + plugin->udfAggFinishFunc = NULL; + return; +} + +void udfdDeinitPythonPlugin(SUdfScriptPlugin *plugin) { + if (plugin->closeFunc) { + plugin->closeFunc(); + } + uv_dlclose(&plugin->lib); + if (plugin->libLoaded) { + plugin->libLoaded = false; + } + plugin->openFunc = NULL; + plugin->closeFunc = NULL; + plugin->udfInitFunc = NULL; + plugin->udfDestroyFunc = NULL; + plugin->udfScalarProcFunc = NULL; + plugin->udfAggStartFunc = NULL; + plugin->udfAggProcFunc = NULL; + plugin->udfAggMergeFunc = NULL; + plugin->udfAggFinishFunc = NULL; +} + +int32_t udfdInitScriptPlugin(int8_t scriptType) { + SUdfScriptPlugin *plugin = taosMemoryCalloc(1, sizeof(SUdfScriptPlugin)); + + switch (scriptType) { + case TSDB_FUNC_SCRIPT_BIN_LIB: + udfdInitializeCPlugin(plugin); + break; + case TSDB_FUNC_SCRIPT_PYTHON: { + int32_t err = udfdInitializePythonPlugin(plugin); + if (err != 0) { + taosMemoryFree(plugin); + return err; + } + break; + } + default: + fnError("udf script type %d not supported", scriptType); + taosMemoryFree(plugin); + return TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED; + } + + global.scriptPlugins[scriptType] = plugin; + return TSDB_CODE_SUCCESS; +} + +void udfdDeinitScriptPlugins() { + SUdfScriptPlugin *plugin = NULL; + plugin = global.scriptPlugins[TSDB_FUNC_SCRIPT_PYTHON]; + if (plugin != NULL) { + udfdDeinitPythonPlugin(plugin); + taosMemoryFree(plugin); + } + + plugin = global.scriptPlugins[TSDB_FUNC_SCRIPT_BIN_LIB]; + if (plugin != NULL) { + udfdDeinitCPlugin(plugin); + taosMemoryFree(plugin); + } + return; +} + void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); SUdfRequest request = {0}; @@ -180,51 +514,134 @@ void udfdProcessRequest(uv_work_t *req) { } } +void convertUdf2UdfInfo(SUdf *udf, SScriptUdfInfo *udfInfo) { + udfInfo->bufSize = udf->bufSize; + if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) { + udfInfo->funcType = UDF_FUNC_TYPE_AGG; + } else if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { + udfInfo->funcType = UDF_FUNC_TYPE_SCALAR; + } + udfInfo->name = udf->name; + udfInfo->version = udf->version; + udfInfo->createdTime = udf->createdTime; + udfInfo->outputLen = udf->outputLen; + udfInfo->outputType = udf->outputType; + udfInfo->path = udf->path; + udfInfo->scriptType = udf->scriptType; +} + +int32_t udfdInitUdf(char *udfName, SUdf *udf) { + int32_t err = 0; + err = udfdFillUdfInfoFromMNode(global.clientRpc, udfName, udf); + if (err != 0) { + fnError("can not retrieve udf from mnode. udf name %s", udfName); + return TSDB_CODE_UDF_LOAD_UDF_FAILURE; + } + if (udf->scriptType > UDFD_MAX_SCRIPT_TYPE) { + fnError("udf name %s script type %d not supported", udfName, udf->scriptType); + return TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED; + } + + uv_mutex_lock(&global.scriptPluginsMutex); + SUdfScriptPlugin *scriptPlugin = global.scriptPlugins[udf->scriptType]; + if (scriptPlugin == NULL) { + err = udfdInitScriptPlugin(udf->scriptType); + if (err != 0) { + uv_mutex_unlock(&global.scriptPluginsMutex); + return err; + } + } + uv_mutex_unlock(&global.scriptPluginsMutex); + udf->scriptPlugin = global.scriptPlugins[udf->scriptType]; + + SScriptUdfInfo info = {0}; + convertUdf2UdfInfo(udf, &info); + err = udf->scriptPlugin->udfInitFunc(&info, &udf->scriptUdfCtx); + if (err != 0) { + fnError("udf name %s init failed. error %d", udfName, err); + return err; + } + + fnInfo("udf init succeeded. name %s type %d context %p", udf->name, udf->scriptType, (void *)udf->scriptUdfCtx); + return 0; +} + +SUdf *udfdNewUdf(const char *udfName) { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->lastFetchTime = taosGetTimestampMs(); + strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); + + udfNew->state = UDF_STATE_INIT; + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + + udfNew->resident = false; + udfNew->expired = false; + for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { + char *funcName = taosArrayGet(global.residentFuncs, i); + if (strcmp(udfName, funcName) == 0) { + udfNew->resident = true; + break; + } + } + return udfNew; +} + +SUdf *udfdGetOrCreateUdf(const char *udfName) { + uv_mutex_lock(&global.udfsMutex); + SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName)); + int64_t currTime = taosGetTimestampMs(); + bool expired = false; + if (pUdfHash) { + expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000; // 10s + if (!expired) { + ++(*pUdfHash)->refCount; + SUdf *udf = *pUdfHash; + uv_mutex_unlock(&global.udfsMutex); + fnInfo("udfd reuse existing udf. udf %s udf version %d, udf created time %" PRIx64, udf->name, udf->version, + udf->createdTime); + return udf; + } else { + (*pUdfHash)->expired = true; + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); + fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, + (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); + } + } + + SUdf *udf = udfdNewUdf(udfName); + + SUdf **pUdf = &udf; + taosHashPut(global.udfsHash, udfName, strlen(udfName), pUdf, POINTER_BYTES); + uv_mutex_unlock(&global.udfsMutex); + + return udf; +} + void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { // TODO: tracable id from client. connect, setup, call, teardown fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); + SUdfSetupRequest *setup = &request->setup; int32_t code = TSDB_CODE_SUCCESS; SUdf *udf = NULL; - uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); - if (udfInHash) { - ++(*udfInHash)->refCount; - udf = *udfInHash; - uv_mutex_unlock(&global.udfsMutex); - } else { - SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); - udfNew->refCount = 1; - udfNew->state = UDF_STATE_INIT; - uv_mutex_init(&udfNew->lock); - uv_cond_init(&udfNew->condReady); - udf = udfNew; - SUdf **pUdf = &udf; - taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), pUdf, POINTER_BYTES); - uv_mutex_unlock(&global.udfsMutex); - } + udf = udfdGetOrCreateUdf(setup->udfName); uv_mutex_lock(&udf->lock); if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; - code = udfdLoadUdf(setup->udfName, udf); - if (udf->initFunc) { - udf->initFunc(); - } - udf->resident = false; - for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { - char *funcName = taosArrayGet(global.residentFuncs, i); - if (strcmp(setup->udfName, funcName) == 0) { - udf->resident = true; - break; - } + code = udfdInitUdf(setup->udfName, udf); + if (code == 0) { + udf->state = UDF_STATE_READY; + } else { + udf->state = UDF_STATE_INIT; } - udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); } else { - while (udf->state != UDF_STATE_READY) { + while (udf->state == UDF_STATE_LOADING) { uv_cond_wait(&udf->condReady, &udf->lock); } uv_mutex_unlock(&udf->lock); @@ -235,10 +652,10 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfResponse rsp; rsp.seqNum = request->seqNum; rsp.type = request->type; - rsp.code = code; + rsp.code = (code != 0) ? TSDB_CODE_UDF_FUNC_EXEC_FAILURE : 0; rsp.setupRsp.udfHandle = (int64_t)(handle); rsp.setupRsp.outputType = udf->outputType; - rsp.setupRsp.outputLen = udf->outputLen; + rsp.setupRsp.bytes = udf->outputLen; rsp.setupRsp.bufSize = udf->bufSize; int32_t len = encodeUdfResponse(NULL, &rsp); @@ -267,10 +684,15 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { switch (call->callType) { case TSDB_UDF_CALL_SCALA_PROC: { SUdfColumn output = {0}; + output.colMeta.bytes = udf->outputLen; + output.colMeta.type = udf->outputType; + output.colMeta.precision = 0; + output.colMeta.scale = 0; + udfColEnsureCapacity(&output, call->block.info.rows); SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); - code = udf->scalarProcFunc(&input, &output); + code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx); freeUdfDataDataBlock(&input); convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); freeUdfColumn(&output); @@ -278,7 +700,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } case TSDB_UDF_CALL_AGG_INIT: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - udf->aggStartFunc(&outBuf); + code = udf->scriptPlugin->udfAggStartFunc(&outBuf, udf->scriptUdfCtx); subRsp->resultBuf = outBuf; break; } @@ -286,7 +708,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggProcFunc(&input, &call->interBuf, &outBuf); + code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx); freeUdfInterBuf(&call->interBuf); freeUdfDataDataBlock(&input); subRsp->resultBuf = outBuf; @@ -295,7 +717,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } case TSDB_UDF_CALL_AGG_MERGE: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf); + code = udf->scriptPlugin->udfAggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf, udf->scriptUdfCtx); freeUdfInterBuf(&call->interBuf); freeUdfInterBuf(&call->interBuf2); subRsp->resultBuf = outBuf; @@ -304,7 +726,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } case TSDB_UDF_CALL_AGG_FIN: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->aggFinishFunc(&call->interBuf, &outBuf); + code = udf->scriptPlugin->udfAggFinishFunc(&call->interBuf, &outBuf, udf->scriptUdfCtx); freeUdfInterBuf(&call->interBuf); subRsp->resultBuf = outBuf; break; @@ -315,7 +737,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { rsp->seqNum = request->seqNum; rsp->type = request->type; - rsp->code = code; + rsp->code = (code != 0) ? TSDB_CODE_UDF_FUNC_EXEC_FAILURE : 0; subRsp->callType = call->callType; int32_t len = encodeUdfResponse(NULL, rsp); @@ -366,18 +788,17 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { uv_mutex_lock(&global.udfsMutex); udf->refCount--; - if (udf->refCount == 0 && !udf->resident) { + if (udf->refCount == 0 && (!udf->resident || udf->expired)) { unloadUdf = true; taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } uv_mutex_unlock(&global.udfsMutex); if (unloadUdf) { + fnInfo("udf teardown. udf name: %s type %d: context %p", udf->name, udf->scriptType, (void *)(udf->scriptUdfCtx)); uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); - if (udf->destroyFunc) { - (udf->destroyFunc)(); - } - uv_dlclose(&udf->lib); + code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx); + fnDebug("udfd destroy function returns %d", code); taosMemoryFree(udf); } taosMemoryFree(handle); @@ -398,6 +819,61 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { return; } +void udfdGetFuncBodyPath(const SUdf *udf, char *path) { + if (udf->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64 ".dll", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/lib%s_%d_%" PRIx64 ".so", global.udfDataDir, udf->name, udf->version, + udf->createdTime); +#endif + } else if (udf->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64 ".py", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/%s_%d_%" PRIx64 ".py", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#endif + } else { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64, global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/lib%s_%d_%" PRIx64, global.udfDataDir, udf->name, udf->version, udf->createdTime); +#endif + } +} + +int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { + if (!osDataSpaceAvailable()) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + fnError("udfd create shared library failed since %s", terrstr(terrno)); + return terrno; + } + + char path[PATH_MAX] = {0}; + udfdGetFuncBodyPath(udf, path); + bool fileExist = !(taosStatFile(path, NULL, NULL) < 0); + if (fileExist) { + strncpy(udf->path, path, PATH_MAX); + fnInfo("udfd func body file. reuse existing file %s", path); + return TSDB_CODE_SUCCESS; + } + + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (file == NULL) { + fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); + return TSDB_CODE_FILE_CORRUPTED; + } + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + if (count != pFuncInfo->codeSize) { + fnError("udfd write udf shared library failed"); + return TSDB_CODE_FILE_CORRUPTED; + } + taosCloseFile(&file); + + strncpy(udf->path, path, PATH_MAX); + return TSDB_CODE_SUCCESS; +} + void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; @@ -436,9 +912,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) { SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp); - if (retrieveRsp.pFuncInfos == NULL) { - goto _return; - } + SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); SUdf *udf = msgInfo->param; udf->funcType = pFuncInfo->funcType; @@ -447,38 +921,16 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->outputLen = pFuncInfo->outputLen; udf->bufSize = pFuncInfo->bufSize; - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - msgInfo->code = terrno; - fnError("udfd create shared library failed since %s", terrstr(terrno)); - goto _return; + SFuncExtraInfo *pFuncExtraInfo = (SFuncExtraInfo *)taosArrayGet(retrieveRsp.pFuncExtraInfos, 0); + udf->version = pFuncExtraInfo->funcVersion; + udf->createdTime = pFuncExtraInfo->funcCreatedTime; + msgInfo->code = udfdSaveFuncBodyToFile(pFuncInfo, udf); + if (msgInfo->code != 0) { + udf->lastFetchTime = 0; } - - char path[PATH_MAX] = {0}; -#ifdef WINDOWS - snprintf(path, sizeof(path), "%s%s.dll", tsTempDir, pFuncInfo->name); -#else - snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name); -#endif - - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); - if (file == NULL) { - fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); - msgInfo->code = TSDB_CODE_FILE_CORRUPTED; - goto _return; - } - - int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); - if (count != pFuncInfo->codeSize) { - fnError("udfd write udf shared library failed"); - msgInfo->code = TSDB_CODE_FILE_CORRUPTED; - goto _return; - } - taosCloseFile(&file); - strncpy(udf->path, path, PATH_MAX); tFreeSFuncInfo(pFuncInfo); taosArrayDestroy(retrieveRsp.pFuncInfos); - msgInfo->code = 0; + taosArrayDestroy(retrieveRsp.pFuncExtraInfos); } _return: @@ -527,6 +979,7 @@ int32_t udfdConnectToMnode() { tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd)); connReq.pid = taosGetPId(); connReq.startTime = taosGetTimestampMs(); + strcpy(connReq.sVer, version); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); void *pReq = rpcMallocCont(contLen); @@ -550,60 +1003,6 @@ int32_t udfdConnectToMnode() { return code; } -int32_t udfdLoadUdf(char *udfName, SUdf *udf) { - strncpy(udf->name, udfName, TSDB_FUNC_NAME_LEN); - int32_t err = 0; - - err = udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); - if (err != 0) { - fnError("can not retrieve udf from mnode. udf name %s", udfName); - return TSDB_CODE_UDF_LOAD_UDF_FAILURE; - } - - err = uv_dlopen(udf->path, &udf->lib); - if (err != 0) { - fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - return TSDB_CODE_UDF_LOAD_UDF_FAILURE; - } - - char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *initSuffix = "_init"; - strcpy(initFuncName, udfName); - strncat(initFuncName, initSuffix, strlen(initSuffix)); - uv_dlsym(&udf->lib, initFuncName, (void **)(&udf->initFunc)); - - char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *destroySuffix = "_destroy"; - strcpy(destroyFuncName, udfName); - strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); - uv_dlsym(&udf->lib, destroyFuncName, (void **)(&udf->destroyFunc)); - - if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { - char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(processFuncName, udfName); - uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->scalarProcFunc)); - } else if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) { - char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(processFuncName, udfName); - uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->aggProcFunc)); - char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *startSuffix = "_start"; - strncpy(startFuncName, processFuncName, sizeof(startFuncName)); - strncat(startFuncName, startSuffix, strlen(startSuffix)); - uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc)); - char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; - char *finishSuffix = "_finish"; - strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); - char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *mergeSuffix = "_merge"; - strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); - strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); - uv_dlsym(&udf->lib, mergeFuncName, (void **)(&udf->aggMergeFunc)); - } - return 0; -} static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED || code == TSDB_CODE_SYN_RESTORING || @@ -965,6 +1364,8 @@ static void udfdCloseWalkCb(uv_handle_t *handle, void *arg) { } static int32_t udfdRun() { + uv_mutex_init(&global.scriptPluginsMutex); + global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); uv_mutex_init(&global.udfsMutex); @@ -1021,13 +1422,11 @@ int32_t udfdDeinitResidentFuncs() { char *funcName = taosArrayGet(global.residentFuncs, i); SUdf **udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); if (udfInHash) { - SUdf *udf = *udfInHash; - if (udf->destroyFunc) { - (udf->destroyFunc)(); - } - uv_dlclose(&udf->lib); - taosMemoryFree(udf); + SUdf *udf = *udfInHash; + int32_t code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx); + fnDebug("udfd destroy function returns %d", code); taosHashRemove(global.udfsHash, funcName, strlen(funcName)); + taosMemoryFree(udf); } } taosArrayDestroy(global.residentFuncs); @@ -1040,6 +1439,24 @@ int32_t udfdCleanup() { return 0; } +int32_t udfdCreateUdfSourceDir() { + snprintf(global.udfDataDir, PATH_MAX, "%s/.udf", tsDataDir); + int32_t code = taosMkDir(global.udfDataDir); + if (code != TSDB_CODE_SUCCESS) { + snprintf(global.udfDataDir, PATH_MAX, "%s/.udf", tsTempDir); + code = taosMkDir(global.udfDataDir); + } + fnInfo("udfd create udf source directory %s. result: %s", global.udfDataDir, tstrerror(code)); + + return code; +} + +int32_t udfdDestroyUdfSourceDir() { + fnInfo("destory udf source directory %s", global.udfDataDir); + taosRemoveDir(global.udfDataDir); + return 0; +} + int main(int argc, char *argv[]) { if (!taosCheckSystemIsLittleEnd()) { printf("failed to start since on non-little-end machines\n"); @@ -1068,10 +1485,15 @@ int main(int argc, char *argv[]) { initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); if (udfdOpenClientRpc() != 0) { - fnError("open rpc connection to mnode failure"); + fnError("open rpc connection to mnode failed"); return -3; } + if (udfdCreateUdfSourceDir() != 0) { + fnError("create udf source directory failed"); + return -4; + } + if (udfdUvInit() != 0) { fnError("uv init failure"); return -5; @@ -1085,9 +1507,13 @@ int main(int argc, char *argv[]) { udfdRun(); removeListeningPipe(); + udfdDestroyUdfSourceDir(); udfdCloseClientRpc(); udfdDeinitResidentFuncs(); + + udfdDeinitScriptPlugins(); + udfdCleanup(); return 0; } diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 71d30b67557847dcdf451cefb20c7b144d214cf8..5b950879962b56dc38a1c26f8665c13cd6c29c9c 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -14,15 +14,8 @@ DLL_EXPORT int32_t udf1_init() { return 0; } DLL_EXPORT int32_t udf1_destroy() { return 0; } DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { - SUdfColumnMeta *meta = &resultCol->colMeta; - meta->bytes = 4; - meta->type = TSDB_DATA_TYPE_INT; - meta->scale = 0; - meta->precision = 0; - SUdfColumnData *resultData = &resultCol->colData; - resultData->numOfRows = block->numOfRows; - for (int32_t i = 0; i < resultData->numOfRows; ++i) { + for (int32_t i = 0; i < block->numOfRows; ++i) { int j = 0; for (; j < block->numOfCols; ++j) { if (udfColDataIsNull(block->udfCols[j], i)) { @@ -31,7 +24,7 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { } } if (j == block->numOfCols) { - int32_t luckyNum = 88; + int32_t luckyNum = 1; udfColDataSet(resultCol, i, (char *)&luckyNum, false); } } @@ -42,5 +35,6 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { #ifdef WINDOWS Sleep(1); #endif + resultData->numOfRows = block->numOfRows; return 0; -} \ No newline at end of file +} diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index e24789d0fb4095f2fdc75d8d1ce3e8dc1355e4c6..faf4daa4e53cacd70cbfd4ff319178c2cdbb8a56 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -12,12 +12,15 @@ DLL_EXPORT int32_t udf2_destroy() { return 0; } DLL_EXPORT int32_t udf2_start(SUdfInterBuf* buf) { *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); - buf->numOfResult = 0; + buf->numOfResult = 1; return 0; } DLL_EXPORT int32_t udf2(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) { - double sumSquares = *(double*)interBuf->buf; + double sumSquares = 0; + if (interBuf->numOfResult == 1) { + sumSquares = *(double*)interBuf->buf; + } int8_t numNotNull = 0; for (int32_t i = 0; i < block->numOfCols; ++i) { SUdfColumn* col = block->udfCols[i]; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 0d13db2f358587c349ef79b7411e8ac45810b883..ec5fc5ad2a50c07d1c4784911d7c39f0fedce9fb 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -634,7 +634,7 @@ static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxF } static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { - int32_t code = 0; + int32_t code = -1; if (sifValidOp(node->opType) < 0) { code = TSDB_CODE_QRY_INVALID_INPUT; ctx->code = code; @@ -654,7 +654,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); - if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { + if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { output->status = SFLT_NOT_INDEX; goto _return; } @@ -664,6 +664,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { sif_func_t operFn = sifNullFunc; if (!ctx->noExec) { + code = 0; SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); SIF_ERR_JRET(operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output)); } else { @@ -672,11 +673,17 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { output->status = SFLT_NOT_INDEX; goto _return; } + code = 0; SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); } _return: for (int i = 0; i < nParam; i++) sifFreeParam(¶ms[i]); taosMemoryFree(params); + if (code != 0) { + output->status = SFLT_NOT_INDEX; + } else { + output->status = SFLT_COARSE_INDEX; + } return code; } @@ -717,7 +724,7 @@ _return: static EDealRes sifWalkFunction(SNode *pNode, void *context) { SFunctionNode *node = (SFunctionNode *)pNode; - SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t))}; + SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t)), .status = SFLT_COARSE_INDEX}; SIFCtx *ctx = context; ctx->code = sifExecFunction(node, ctx, &output); @@ -735,7 +742,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) { static EDealRes sifWalkLogic(SNode *pNode, void *context) { SLogicConditionNode *node = (SLogicConditionNode *)pNode; - SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t))}; + SIFParam output = {.result = taosArrayInit(8, sizeof(uint64_t)), .status = SFLT_COARSE_INDEX}; SIFCtx *ctx = context; ctx->code = sifExecLogic(node, ctx, &output); @@ -831,6 +838,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (res->result != NULL) { taosArrayAddAll(pDst->result, res->result); } + pDst->status = res->status; sifFreeParam(res); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); @@ -887,16 +895,20 @@ int32_t doFilterTag(SNode *pFilterNode, SIndexMetaArg *metaArg, SArray *result, SFilterInfo *filter = NULL; SArray *output = taosArrayInit(8, sizeof(uint64_t)); - SIFParam param = {.arg = *metaArg, .result = output}; + SIFParam param = {.arg = *metaArg, .result = output, .status = SFLT_NOT_INDEX}; int32_t code = sifCalculate((SNode *)pFilterNode, ¶m); if (code != 0) { sifFreeParam(¶m); return code; } + if (param.status == SFLT_NOT_INDEX) { + *status = param.status; + } else { + *status = st; + } taosArrayAddAll(result, param.result); sifFreeParam(¶m); - *status = st; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 40c50ed9cb97fd0ea64eb8d50bcf04801a24a89b..9e7ed5210425520f7b7c02a7f9b2edc6d98756ed 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -127,8 +127,6 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); ASSERTS(blk->nread <= kBlockSize, "index read incomplete data"); - if (blk->nread > kBlockSize) break; - if (blk->nread < kBlockSize && blk->nread < len) { taosMemoryFree(blk); break; diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index b23a36d4df3acaef40116b2fd66559a661e3f8c1..949e91198adcb69301fbc177b721289d6d736651 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -210,6 +210,8 @@ static void monGenClusterJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "vnodes_total", pInfo->vnodes_total); tjsonAddDoubleToObject(pJson, "vnodes_alive", pInfo->vnodes_alive); tjsonAddDoubleToObject(pJson, "connections_total", pInfo->connections_total); + tjsonAddDoubleToObject(pJson, "topics_total", pInfo->topics_toal); + tjsonAddDoubleToObject(pJson, "streams_total", pInfo->streams_total); SJson *pDnodesJson = tjsonAddArrayToObject(pJson, "dnodes"); if (pDnodesJson == NULL) return; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e18de1c1d2fd6215f81bd08178a81400dadb4932..0aeb83ce0812cf52bd3add4b2b0276fe051d00f3 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -173,6 +173,8 @@ const char* nodesNodeName(ENodeType type) { return "DropStreamStmt"; case QUERY_NODE_BALANCE_VGROUP_STMT: return "BalanceVgroupStmt"; + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: + return "BalanceVgroupLeaderStmt"; case QUERY_NODE_MERGE_VGROUP_STMT: return "MergeVgroupStmt"; case QUERY_NODE_SHOW_DB_ALIVE_STMT: @@ -6433,6 +6435,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return dropStreamStmtToJson(pObj, pJson); case QUERY_NODE_BALANCE_VGROUP_STMT: return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize. + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to serialize. case QUERY_NODE_MERGE_VGROUP_STMT: return mergeVgroupStmtToJson(pObj, pJson); case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: @@ -6741,6 +6745,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDropStreamStmt(pJson, pObj); case QUERY_NODE_BALANCE_VGROUP_STMT: return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize. + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupLeaderStmt has no fields to deserialize. case QUERY_NODE_MERGE_VGROUP_STMT: return jsonToMergeVgroupStmt(pJson, pObj); case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 4e23999ec2c32e2c82a6b0c569b3ff5c60770ef7..156744ef1dceb9e47860de780904c54997023f5c 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -136,6 +136,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); + COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); return true; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 22afc7ef5536b3ef5e6eb4d852029b833af1e165..3f571e22ae720d3dd9b18d33da7a8b42c60d2b48 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -386,6 +386,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SDropStreamStmt)); case QUERY_NODE_BALANCE_VGROUP_STMT: return makeNode(type, sizeof(SBalanceVgroupStmt)); + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: + return makeNode(type, sizeof(SBalanceVgroupLeaderStmt)); case QUERY_NODE_MERGE_VGROUP_STMT: return makeNode(type, sizeof(SMergeVgroupStmt)); case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: @@ -923,9 +925,15 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFree(((SDescribeStmt*)pNode)->pMeta); break; case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field - case QUERY_NODE_COMPACT_DATABASE_STMT: // no pointer field - case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field - case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field + break; + case QUERY_NODE_COMPACT_DATABASE_STMT: { + SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)pNode; + nodesDestroyNode(pStmt->pStart); + nodesDestroyNode(pStmt->pEnd); + break; + } + case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field + case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field break; case QUERY_NODE_CREATE_STREAM_STMT: { SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pNode; @@ -935,9 +943,10 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pSubtable); break; } - case QUERY_NODE_DROP_STREAM_STMT: // no pointer field - case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field - case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field + case QUERY_NODE_DROP_STREAM_STMT: // no pointer field + case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field + case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field break; case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: nodesDestroyList(((SRedistributeVgroupStmt*)pNode)->pDnodes); @@ -1462,7 +1471,7 @@ SListCell* nodesListErase(SNodeList* pList, SListCell* pCell) { } void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc) { - if (NULL == pTarget || NULL == pPos || NULL == pSrc) { + if (NULL == pTarget || NULL == pPos || NULL == pSrc || NULL == pSrc->pHead) { return; } diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 10d4adf17db22e4a8068a9c702cd1fda933ec6b3..e08b77e681918ee1b00c46120f9c239c62a2fbcd 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -82,6 +82,11 @@ typedef struct SAlterOption { SNodeList* pList; } SAlterOption; +typedef struct STokenPair { + SToken first; + SToken second; +} STokenPair; + extern SToken nil_token; void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); @@ -212,7 +217,8 @@ SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt); SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, - const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage); + const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage, + bool orReplace); SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* setStreamOptions(SAstCreateContext* pCxt, SNode* pOptions, EStreamOptionsSetFlag setflag, SToken* pToken, @@ -223,12 +229,15 @@ SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToke SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt); +SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt); SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2); SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes); SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId); SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName); -SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); -SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); +SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond); +SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond); SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere); SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index ce5a63f5d07d35fb7a432e878d60372b9bab6cf9..850571eea1533541318c4dc57c60b14311fcc5a8 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -97,9 +97,8 @@ int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* p int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); -int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, +int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, SParseMetaCache* pMetaCache); -int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); @@ -110,8 +109,7 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); -int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, - bool* pPass); +int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes); int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo); int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 994f99a7979603fde3372fb11d933bbf6164146a..d7a6baaffeacd0f9c009467b3f41e4a49d335263 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -94,8 +94,8 @@ sysinfo_opt(A) ::= . sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B). { A = taosStr2Int8(B.z, NULL, 10); } /************************************************ grant/revoke ********************************************************/ -cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); } -cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); } +cmd ::= GRANT privileges(A) ON priv_level(B) with_opt(D) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C, D); } +cmd ::= REVOKE privileges(A) ON priv_level(B) with_opt(D) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C, D); } %type privileges { int64_t } %destructor privileges { } @@ -113,11 +113,15 @@ priv_type_list(A) ::= priv_type_list(B) NK_COMMA priv_type(C). priv_type(A) ::= READ. { A = PRIVILEGE_TYPE_READ; } priv_type(A) ::= WRITE. { A = PRIVILEGE_TYPE_WRITE; } -%type priv_level { SToken } +%type priv_level { STokenPair } %destructor priv_level { } -priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR. { A = B; } -priv_level(A) ::= db_name(B) NK_DOT NK_STAR. { A = B; } -priv_level(A) ::= topic_name(B). { A = B; } +priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; } +priv_level(A) ::= db_name(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; } +priv_level(A) ::= db_name(B) NK_DOT table_name(C). { A.first = B; A.second = C; } +priv_level(A) ::= topic_name(B). { A.first = B; A.second = nil_token; } + +with_opt(A) ::= . { A = NULL; } +with_opt(A) ::= WITH search_condition(B). { A = B; } /************************************************ create/drop/alter dnode *********************************************/ cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } @@ -236,6 +240,19 @@ alter_db_option(A) ::= REPLICA NK_INTEGER(B). //alter_db_option(A) ::= STRICT NK_STRING(B). { A.type = DB_OPTION_STRICT; A.val = B; } alter_db_option(A) ::= WAL_LEVEL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } alter_db_option(A) ::= STT_TRIGGER NK_INTEGER(B). { A.type = DB_OPTION_STT_TRIGGER; A.val = B; } +alter_db_option(A) ::= MINROWS NK_INTEGER(B). { A.type = DB_OPTION_MINROWS; A.val = B; } +alter_db_option(A) ::= WAL_RETENTION_PERIOD NK_INTEGER(B). { A.type = DB_OPTION_WAL_RETENTION_PERIOD; A.val = B; } +alter_db_option(A) ::= WAL_RETENTION_PERIOD NK_MINUS(B) NK_INTEGER(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A.type = DB_OPTION_WAL_RETENTION_PERIOD; A.val = t; + } +alter_db_option(A) ::= WAL_RETENTION_SIZE NK_INTEGER(B). { A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = B; } +alter_db_option(A) ::= WAL_RETENTION_SIZE NK_MINUS(B) NK_INTEGER(C). { + SToken t = B; + t.n = (C.z + C.n) - B.z; + A.type = DB_OPTION_WAL_RETENTION_SIZE; A.val = t; + } %type integer_list { SNodeList* } %destructor integer_list { nodesDestroyList($$); } @@ -329,7 +346,7 @@ column_def_list(A) ::= column_def(B). column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } -column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } +//column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } %type type_name { SDataType } %destructor type_name { } @@ -540,8 +557,9 @@ explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C). explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C). { A = setExplainRatio(pCxt, B, &C); } /************************************************ create/drop function ************************************************/ -cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B) - AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E) language_opt(G). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E, &G); } +cmd ::= CREATE or_replace_opt(H) agg_func_opt(A) FUNCTION not_exists_opt(F) + function_name(B) AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E) + language_opt(G). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E, &G, H); } cmd ::= DROP FUNCTION exists_opt(B) function_name(A). { pCxt->pRootNode = createDropFunctionStmt(pCxt, B, &A); } %type agg_func_opt { bool } @@ -559,6 +577,11 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). language_opt(A) ::= . { A = nil_token; } language_opt(A) ::= LANGUAGE NK_STRING(B). { A = B; } +%type or_replace_opt { bool } +%destructor or_replace_opt { } +or_replace_opt(A) ::= . { A = false; } +or_replace_opt(A) ::= OR REPLACE. { A = true; } + /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO full_table_name(C) col_list_opt(H) tag_def_or_ref_opt(F) subtable_opt(G) @@ -596,6 +619,7 @@ cmd ::= KILL TRANSACTION NK_INTEGER(A). /************************************************ merge/redistribute/ vgroup ******************************************/ cmd ::= BALANCE VGROUP. { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } +cmd ::= BALANCE VGROUP LEADER. { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); } cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B). { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); } cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B). { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); } cmd ::= SPLIT VGROUP NK_INTEGER(A). { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); } @@ -1023,8 +1047,8 @@ sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } -fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } %type fill_mode { EFillMode } %destructor fill_mode { } @@ -1122,5 +1146,5 @@ null_ordering_opt(A) ::= NULLS FIRST. null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } %fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL - FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT + FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE RESTRICT ROW SEMI STAR STATEMENT STRICT STRING TIMES VALUES VARIABLE VIEW WAL. diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a761b7a7b08decfefd3c2bfd6c8c974afce0945c..2afe34c1f7fcc2a1ca91df04659e496d20731b9e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -925,8 +925,8 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = -1; pOptions->singleStable = -1; pOptions->schemaless = -1; - pOptions->walRetentionPeriod = -1; - pOptions->walRetentionSize = -1; + pOptions->walRetentionPeriod = -2; // -1 is a valid value + pOptions->walRetentionSize = -2; // -1 is a valid value pOptions->walRollPeriod = -1; pOptions->walSegmentSize = -1; pOptions->sstTrigger = -1; @@ -935,7 +935,8 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { return (SNode*)pOptions; } -SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { +static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal, + bool alter) { CHECK_PARSER_STATUS(pCxt); SDatabaseOptions* pDbOptions = (SDatabaseOptions*)pOptions; switch (type) { @@ -986,7 +987,9 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti break; case DB_OPTION_REPLICA: pDbOptions->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); - updateWalOptionsDefault(pDbOptions); + if (!alter) { + updateWalOptionsDefault(pDbOptions); + } break; case DB_OPTION_STRICT: COPY_STRING_FORM_STR_TOKEN(pDbOptions->strictStr, (SToken*)pVal); @@ -1033,16 +1036,20 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti return pOptions; } +SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { + return setDatabaseOptionImpl(pCxt, pOptions, type, pVal, false); +} + SNode* setAlterDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, SAlterOption* pAlterOption) { CHECK_PARSER_STATUS(pCxt); switch (pAlterOption->type) { case DB_OPTION_KEEP: case DB_OPTION_RETENTIONS: - return setDatabaseOption(pCxt, pOptions, pAlterOption->type, pAlterOption->pList); + return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, pAlterOption->pList, true); default: break; } - return setDatabaseOption(pCxt, pOptions, pAlterOption->type, &pAlterOption->val); + return setDatabaseOptionImpl(pCxt, pOptions, pAlterOption->type, &pAlterOption->val, true); } SNode* createCreateDatabaseStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pDbName, SNode* pOptions) { @@ -1794,7 +1801,8 @@ static int32_t convertUdfLanguageType(SAstCreateContext* pCxt, const SToken* pLa } SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, - const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage) { + const SToken* pLibPath, SDataType dataType, int32_t bufSize, const SToken* pLanguage, + bool orReplace) { CHECK_PARSER_STATUS(pCxt); if (pLibPath->n <= 2) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; @@ -1806,6 +1814,7 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool } SCreateFunctionStmt* pStmt = (SCreateFunctionStmt*)nodesMakeNode(QUERY_NODE_CREATE_FUNCTION_STMT); CHECK_OUT_OF_MEM(pStmt); + pStmt->orReplace = orReplace; pStmt->ignoreExists = ignoreExists; COPY_STRING_FORM_ID_TOKEN(pStmt->funcName, pFuncName); pStmt->isAgg = aggFunc; @@ -1945,6 +1954,13 @@ SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) { return (SNode*)pStmt; } +SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt) { + CHECK_PARSER_STATUS(pCxt); + SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_STMT); + CHECK_OUT_OF_MEM(pStmt); + return (SNode*)pStmt; +} + SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) { CHECK_PARSER_STATUS(pCxt); SMergeVgroupStmt* pStmt = (SMergeVgroupStmt*)nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT); @@ -1978,29 +1994,39 @@ SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) { return pStmt; } -SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) { +SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) { + if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) { return NULL; } SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->privileges = privileges; - COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName); + COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first); + if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) { + COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second); + } COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); + pStmt->pTagCond = pTagCond; return (SNode*)pStmt; } -SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) { +SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) { + if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) { return NULL; } SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->privileges = privileges; - COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName); + COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first); + if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) { + COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second); + } COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); + pStmt->pTagCond = pTagCond; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index cd4b455e025c3ff67bd22fddbc6e44936a35fbc3..5db1f5dbdbca70bed529373c3bb2caa76d74efd2 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -154,7 +154,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, authType, pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, pTable, authType, + pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache); @@ -247,7 +248,7 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, AUTH_TYPE_WRITE, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE, pCxt->pMetaCache); } return code; @@ -267,8 +268,8 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, AUTH_TYPE_WRITE, - pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, NULL, + AUTH_TYPE_WRITE, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS != code) { break; @@ -286,6 +287,10 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, + pClause->tableName, AUTH_TYPE_WRITE, pCxt->pMetaCache); + } if (TSDB_CODE_SUCCESS != code) { break; } @@ -293,6 +298,11 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS return code; } +static int32_t collectMetaKeyFromDropStable(SCollectMetaKeyCxt* pCxt, SDropSuperTableStmt* pStmt) { + return reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTableStmt* pStmt) { int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { @@ -301,6 +311,10 @@ static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTabl if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); + } return code; } @@ -309,6 +323,10 @@ static int32_t collectMetaKeyFromAlterStable(SCollectMetaKeyCxt* pCxt, SAlterTab if (TSDB_CODE_SUCCESS == code) { code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); + } return code; } @@ -439,8 +457,9 @@ static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, - ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); } return code; } @@ -457,8 +476,9 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, - ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); } return code; } @@ -561,7 +581,8 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCacheExt(pCxt->pParseCxt->pUser, &name, AUTH_TYPE_READ, pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + pCxt->pMetaCache); } return code; } @@ -610,6 +631,13 @@ static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SComp return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } +static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pStmt) { + if ('\0' == pStmt->tabName[0]) { + return TSDB_CODE_SUCCESS; + } + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { pCxt->pStmt = pStmt; switch (nodeType(pStmt)) { @@ -627,6 +655,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_DROP_TABLE_STMT: return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return collectMetaKeyFromDropStable(pCxt, (SDropSuperTableStmt*)pStmt); case QUERY_NODE_ALTER_TABLE_STMT: return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt); case QUERY_NODE_ALTER_SUPER_TABLE_STMT: @@ -645,6 +675,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt); case QUERY_NODE_CREATE_STREAM_STMT: return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); + case QUERY_NODE_GRANT_STMT: + return collectMetaKeyFromGrant(pCxt, (SGrantStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_MNODES_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index e4de60fd051d6f7f4e781160305ffb95a3ec7d3e..b06d48a690c47889bf6a7cc3e74ff56f5b338487 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -23,49 +23,112 @@ typedef struct SAuthCxt { int32_t errCode; } SAuthCxt; +typedef struct SSelectAuthCxt { + SAuthCxt* pAuthCxt; + SSelectStmt* pSelect; +} SSelectAuthCxt; + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt); -static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, AUTH_TYPE type) { +static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, + SUserAuthInfo* pAuth) { + snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pUser); + if (NULL == pTabName) { + tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); + } else { + toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); + } + pAuth->type = type; +} + +static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) { SParseContext* pParseCxt = pCxt->pParseCxt; if (pParseCxt->isSuperUser) { return TSDB_CODE_SUCCESS; } - SName name; - tNameSetDbName(&name, pParseCxt->acctId, pDbName, strlen(pDbName)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); - int32_t code = TSDB_CODE_SUCCESS; - bool pass = false; + + SUserAuthInfo authInfo = {0}; + setUserAuthInfo(pCxt->pParseCxt, pDbName, pTabName, type, &authInfo); + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthRes authRes = {0}; if (NULL != pCxt->pMetaCache) { - code = getUserAuthFromCache(pCxt->pMetaCache, pParseCxt->pUser, dbFname, type, &pass); + code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes); } else { SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter, .requestId = pParseCxt->requestId, .requestObjRefId = pParseCxt->requestRid, .mgmtEps = pParseCxt->mgmtEpSet}; - - code = catalogChkAuth(pParseCxt->pCatalog, &conn, pParseCxt->pUser, dbFname, type, &pass); + code = catalogChkAuth(pParseCxt->pCatalog, &conn, &authInfo, &authRes); + } + if (TSDB_CODE_SUCCESS == code && NULL != pCond) { + *pCond = authRes.pCond; } - return TSDB_CODE_SUCCESS == code ? (pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; + return TSDB_CODE_SUCCESS == code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) { return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } +static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogicCond->condType = LOGIC_COND_TYPE_AND; + int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + *pWhere = (SNode*)pLogicCond; + } else { + nodesDestroyNode((SNode*)pLogicCond); + } + return code; +} + +static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { + SNode* pTagCondCopy = nodesCloneNode(pTagCond); + if (NULL == pTagCondCopy) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL == *pWhere) { + *pWhere = pTagCondCopy; + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); + } + + return mergeStableTagCond(pWhere, &pTagCondCopy); +} + static EDealRes authSelectImpl(SNode* pNode, void* pContext) { - SAuthCxt* pCxt = pContext; + SSelectAuthCxt* pCxt = pContext; + SAuthCxt* pAuthCxt = pCxt->pAuthCxt; if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) { - pCxt->errCode = checkAuth(pCxt, ((SRealTableNode*)pNode)->table.dbName, AUTH_TYPE_READ); - return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pNode; + pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond); + if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) { + pAuthCxt->errCode = appendStableTagCond(&pCxt->pSelect->pWhere, pTagCond); + } + return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) { - return authSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + return authSubquery(pAuthCxt, ((STempTableNode*)pNode)->pSubquery); } return DEAL_RES_CONTINUE; } static int32_t authSelect(SAuthCxt* pCxt, SSelectStmt* pSelect) { - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, pCxt); + SSelectAuthCxt cxt = {.pAuthCxt = pCxt, .pSelect = pSelect}; + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, &cxt); return pCxt->errCode; } @@ -85,11 +148,20 @@ static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { } static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) { - return checkAuth(pCxt, ((SRealTableNode*)pDelete->pFromTable)->table.dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pDelete->pFromTable; + int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond); + if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { + code = appendStableTagCond(&pDelete->pWhere, pTagCond); + } + return code; } static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) { - int32_t code = checkAuth(pCxt, ((SRealTableNode*)pInsert->pTable)->table.dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pInsert->pTable; + // todo check tag condition for subtable + int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond); if (TSDB_CODE_SUCCESS == code) { code = authQuery(pCxt, pInsert->pQuery); } @@ -97,22 +169,27 @@ static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) { } static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { - return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE); + return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, NULL, AUTH_TYPE_READ_OR_WRITE, NULL); } static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { - return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ); + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { - return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_WRITE, &pTagCond); } static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { - code = checkAuth(pCxt, ((SCreateSubTableClause*)pNode)->dbName, AUTH_TYPE_WRITE); + SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; + code = checkAuth(pCxt, pClause->dbName, NULL, AUTH_TYPE_WRITE, NULL); if (TSDB_CODE_SUCCESS != code) { break; } @@ -120,6 +197,29 @@ static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStm return code; } +static int32_t authDropTable(SAuthCxt* pCxt, SDropTableStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTables) { + SDropTableClause* pClause = (SDropTableClause*)pNode; + code = checkAuth(pCxt, pClause->dbName, pClause->tableName, AUTH_TYPE_WRITE, NULL); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t authDropStable(SAuthCxt* pCxt, SDropSuperTableStmt* pStmt) { + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL); +} + +static int32_t authAlterTable(SAuthCxt* pCxt, SAlterTableStmt* pStmt) { + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL); +} + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: @@ -136,6 +236,13 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { return authCreateTable(pCxt, (SCreateTableStmt*)pStmt); case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); + case QUERY_NODE_DROP_TABLE_STMT: + return authDropTable(pCxt, (SDropTableStmt*)pStmt); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return authDropStable(pCxt, (SDropSuperTableStmt*)pStmt); + case QUERY_NODE_ALTER_TABLE_STMT: + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return authAlterTable(pCxt, (SAlterTableStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MODULES_STMT: @@ -146,7 +253,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_DB_ALIVE_STMT: - case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index b2fc88add1506336869dce1c33c19ff9e077a4d2..c25d0e7036aef4d979069976e6f0e221c2b84615 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -183,16 +183,18 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { } else { code = scalarCalculateConstants(pProject, pNew); } - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { + if (TSDB_CODE_SUCCESS == code) { strcpy(((SExprNode*)*pNew)->aliasName, aliasName); - int32_t size = taosArrayGetSize(pAssociation); - for (int32_t i = 0; i < size; ++i) { - SNode** pCol = taosArrayGetP(pAssociation, i); - nodesDestroyNode(*pCol); - *pCol = nodesCloneNode(*pNew); - if (NULL == *pCol) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; + if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { + int32_t size = taosArrayGetSize(pAssociation); + for (int32_t i = 0; i < size; ++i) { + SNode** pCol = taosArrayGetP(pAssociation, i); + nodesDestroyNode(*pCol); + *pCol = nodesCloneNode(*pNew); + if (NULL == *pCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } } } } @@ -335,8 +337,14 @@ static SNodeList* getChildProjection(SNode* pStmt) { static void eraseSetOpChildProjection(SSetOperator* pSetOp, int32_t index) { SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft); nodesListErase(pLeftProjs, nodesListGetCell(pLeftProjs, index)); + if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pLeft)) { + eraseSetOpChildProjection((SSetOperator*)pSetOp->pLeft, index); + } SNodeList* pRightProjs = getChildProjection(pSetOp->pRight); nodesListErase(pRightProjs, nodesListGetCell(pRightProjs, index)); + if (QUERY_NODE_SET_OPERATOR == nodeType(pSetOp->pRight)) { + eraseSetOpChildProjection((SSetOperator*)pSetOp->pRight, index); + } } typedef struct SNotRefByOrderByCxt { @@ -361,11 +369,30 @@ static bool notRefByOrderBy(SColumnNode* pCol, SNodeList* pOrderByList) { return !cxt.hasThisCol; } +static bool isSetUselessCol(SSetOperator* pSetOp, int32_t index, SExprNode* pProj) { + if (!isUselessCol(pProj)) { + return false; + } + + SNodeList* pLeftProjs = getChildProjection(pSetOp->pLeft); + if (!isUselessCol((SExprNode*)nodesListGetNode(pLeftProjs, index))) { + return false; + } + + SNodeList* pRightProjs = getChildProjection(pSetOp->pRight); + if (!isUselessCol((SExprNode*)nodesListGetNode(pRightProjs, index))) { + return false; + } + + return true; +} + static int32_t calcConstSetOpProjections(SCalcConstContext* pCxt, SSetOperator* pSetOp, bool subquery) { int32_t index = 0; SNode* pProj = NULL; WHERE_EACH(pProj, pSetOp->pProjectionList) { - if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && isUselessCol((SExprNode*)pProj)) { + if (subquery && notRefByOrderBy((SColumnNode*)pProj, pSetOp->pOrderByList) && + isSetUselessCol(pSetOp, index, (SExprNode*)pProj)) { ERASE_NODE(pSetOp->pProjectionList); eraseSetOpChildProjection(pSetOp, index); continue; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 4742921d083257b56d7095d0e8aca8fe705900c2..1c921b2d7c2dcfee70200498b3ee10511331251d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -25,6 +25,8 @@ static void clearColValArray(SArray* pCols) { if (TSDB_DATA_TYPE_NCHAR == pCol->type) { taosMemoryFreeClear(pCol->value.pData); } + pCol->flag = CV_FLAG_NONE; + pCol->value.val = 0; } } @@ -70,7 +72,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); + uTrace("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); if (index < 0 && t > 0) { index = insFindCol(&sToken, 0, t, pSchema); } @@ -123,6 +125,12 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); + if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildCol error col not same %s", pTagSchema->name); + goto end; + } + taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); @@ -201,6 +209,7 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSmlKv* kv = (SSmlKv*)data; if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ ret = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildCol error col not same %s", pColSchema->name); goto end; } if (kv->type == TSDB_DATA_TYPE_NCHAR) { @@ -344,6 +353,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { if (errno == E2BIG) { + uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value); buildInvalidOperationMsg(&pBuf, "value too long"); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 3fbe23592a8ed2344b4df8898cec9f80c5dc906d..eb2efd573d1767ccbe1dac7e20f5e712796a84d7 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -15,6 +15,7 @@ #include "parInsertUtil.h" #include "parToken.h" +#include "scalar.h" #include "tglobal.h" #include "ttime.h" @@ -565,6 +566,120 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs return TSDB_CODE_SUCCESS; } +typedef struct SRewriteTagCondCxt { + SArray* pTagVals; + SArray* pTagName; + int32_t code; +} SRewriteTagCondCxt; + +static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { + SValueNode* pValue = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pValue) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pValue->node.resType.type = pVal->type; + switch (pVal->type) { + case TSDB_DATA_TYPE_BOOL: + pValue->datum.b = *(int8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_TINYINT: + pValue->datum.i = *(int8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_SMALLINT: + pValue->datum.i = *(int16_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_INT: + pValue->datum.i = *(int32_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_BIGINT: + pValue->datum.i = pVal->i64; + break; + case TSDB_DATA_TYPE_FLOAT: + pValue->datum.d = *(float*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_DOUBLE: + pValue->datum.d = *(double*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_NCHAR: + pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE); + if (NULL == pValue->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + varDataSetLen(pValue->datum.p, pVal->nData); + memcpy(varDataVal(pValue->datum.p), pVal->pData, pVal->nData); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + pValue->datum.i = pVal->i64; + break; + case TSDB_DATA_TYPE_UTINYINT: + pValue->datum.i = *(uint8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_USMALLINT: + pValue->datum.i = *(uint16_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_UINT: + pValue->datum.i = *(uint32_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_UBIGINT: + pValue->datum.i = *(uint64_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + default: + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t rewriteTagCondColumn(SArray* pTagVals, SArray* pTagName, SNode** pNode) { + SColumnNode* pCol = (SColumnNode*)*pNode; + int32_t ntags = taosArrayGetSize(pTagName); + for (int32_t i = 0; i < ntags; ++i) { + char* pTagColName = taosArrayGet(pTagName, i); + if (0 == strcmp(pTagColName, pCol->colName)) { + return rewriteTagCondColumnImpl(taosArrayGet(pTagVals, i), pNode); + } + } + return TSDB_CODE_PAR_PERMISSION_DENIED; +} + +static EDealRes rewriteTagCond(SNode** pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SRewriteTagCondCxt* pCxt = pContext; + pCxt->code = rewriteTagCondColumn(pCxt->pTagVals, pCxt->pTagName, pNode); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + return DEAL_RES_CONTINUE; +} + +static int32_t setTagVal(SArray* pTagVals, SArray* pTagName, SNode* pCond) { + SRewriteTagCondCxt cxt = {.code = TSDB_CODE_SUCCESS, .pTagVals = pTagVals, .pTagName = pTagName}; + nodesRewriteExpr(&pCond, rewriteTagCond, &cxt); + return cxt.code; +} + +static int32_t checkTagCondResult(SNode* pResult) { + return (QUERY_NODE_VALUE == nodeType(pResult) && ((SValueNode*)pResult)->datum.b) ? TSDB_CODE_SUCCESS + : TSDB_CODE_PAR_PERMISSION_DENIED; +} + +int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) { + int32_t code = setTagVal(pTagVals, pTagName, pCond); + SNode* pNew = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = scalarCalculateConstants(pCond, &pNew); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkTagCondResult(pNew); + } + nodesDestroyNode(pNew); + return code; +} + // pSql -> tag1_value, ...) static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; @@ -722,25 +837,31 @@ static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpS return code; } +static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo* pInfo) { + snprintf(pInfo->user, sizeof(pInfo->user), "%s", pCxt->pUser); + memcpy(&pInfo->tbName, pTbName, sizeof(SName)); + pInfo->type = AUTH_TYPE_WRITE; +} + static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTbName, dbFName); - int32_t code = TSDB_CODE_SUCCESS; - bool pass = true; - bool exists = true; + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthInfo authInfo = {0}; + setUserAuthInfo(pCxt, pTbName, &authInfo); + SUserAuthRes authRes = {0}; + bool exists = true; if (pCxt->async) { - code = catalogChkAuthFromCache(pCxt->pCatalog, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass, &exists); + code = catalogChkAuthFromCache(pCxt->pCatalog, &authInfo, &authRes, &exists); } else { SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, .requestId = pCxt->requestId, .requestObjRefId = pCxt->requestRid, .mgmtEps = pCxt->mgmtEpSet}; - code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + code = catalogChkAuth(pCxt->pCatalog, &conn, &authInfo, &authRes); } if (TSDB_CODE_SUCCESS == code) { if (!exists) { *pMissCache = true; - } else if (!pass) { + } else if (!authRes.pass) { code = TSDB_CODE_PAR_PERMISSION_DENIED; } } @@ -1492,6 +1613,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm pStmt->pBoundCols = NULL; pStmt->usingTableProcessing = false; pStmt->fileProcessing = false; + pStmt->usingTableName.type = 0; } // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... @@ -1539,6 +1661,10 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif } } + if (TK_NK_ID != pTbName->type && TK_NK_STRING != pTbName->type && TK_NK_QUESTION != pTbName->type) { + return buildSyntaxErrMsg(&pCxt->msg, "table_name is expected", pTbName->z); + } + *pHasData = true; return TSDB_CODE_SUCCESS; } @@ -1901,7 +2027,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE}; snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser); - tNameGetFullDbName(pName, userAuth.dbFName); + memcpy(&userAuth.tbName, pName, sizeof(SName)); taosArrayPush(*pUserAuth, &userAuth); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 01a635e4b2d34bbb7982d45400429ac323e9547e..922a0f45ff38fba30fc4b766a3b6427026b49b36 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -251,7 +251,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in goto _return; } - if (bind[c].buffer_type != pColSchema->type) { + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 52d3569dcd15be1370ee2930796e1bd126efc09a..ac504b9809ad495175d54025d881853d0f443ca9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -404,6 +404,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); if (TSDB_CODE_SUCCESS == code) { taosArrayPush(pVgroupList, &pVgCxt); +// uDebug("td23101 2vgId:%d, uid:%" PRIu64, pVgCxt->vgId, pTableCxt->pMeta->uid); *pOutput = pVgCxt; } else { insDestroyVgroupDataCxt(pVgCxt); @@ -469,11 +470,11 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; - void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); - if (NULL == p) { + void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == pp) { code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); } else { - pVgCxt = *(SVgroupDataCxt**)p; + pVgCxt = *(SVgroupDataCxt**)pp; } if (TSDB_CODE_SUCCESS == code) { code = fillVgroupDataCxt(pTableCxt, pVgCxt); @@ -546,6 +547,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); +// uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); @@ -564,53 +566,19 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, return code; } -static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) { - bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); - if (NULL == pUseCols) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pBoundInfo->numOfBound = 0; - - int16_t lastColIdx = -1; // last column found - int32_t code = TSDB_CODE_SUCCESS; +static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { for (int i = 0; i < numFields; i++) { - SToken token; - token.z = fields[i].name; - token.n = strlen(fields[i].name); - - int16_t t = lastColIdx + 1; - int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); - if (index < 0 && t > 0) { - index = insFindCol(&token, 0, t, pSchema); - } - if (index < 0) { - uError("can not find column name:%s", token.z); - code = TSDB_CODE_PAR_INVALID_COLUMN; - break; - } else if (pUseCols[index]) { - code = TSDB_CODE_PAR_INVALID_COLUMN; - uError("duplicated column name:%s", token.z); - break; - } else { - lastColIdx = index; - pUseCols[index] = true; - pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; - ++pBoundInfo->numOfBound; + if(strcmp(pSchema->name, fields[i].name) == 0){ + return true; } } - if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { - uError("primary timestamp column can not be null:"); - code = TSDB_CODE_PAR_INVALID_COLUMN; - } - - taosMemoryFree(pUseCols); - return code; + return false; } int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, int numFields, bool needChangeLength) { + void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); @@ -618,19 +586,14 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate uError("insGetTableDataCxt error"); goto end; } - if (tFields != NULL) { - ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); + + if(tmp == NULL){ + ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { - uError("bindFileds error"); + uError("initTableColSubmitData error"); goto end; } } - // no need to bind, because select * get all fields - ret = initTableColSubmitData(pTableCxt); - if (ret != TSDB_CODE_SUCCESS) { - uError("initTableColSubmitData error"); - goto end; - } char* p = (char*)data; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column @@ -658,35 +621,83 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; - if (boundInfo->numOfBound != numOfCols) { - uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); + if (tFields != NULL && numFields != numOfCols) { + uError("numFields:%d != numOfCols:%d", numFields, numOfCols); ret = TSDB_CODE_INVALID_PARA; goto end; } - for (int c = 0; c < boundInfo->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; - SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); + if (tFields != NULL && numFields > boundInfo->numOfBound) { + uError("numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + if(tFields == NULL){ + for (int j = 0; j < boundInfo->numOfBound; j++){ + SSchema* pColSchema = &pSchema[j]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } - if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - uError("type or bytes not equal"); - ret = TSDB_CODE_INVALID_PARA; - goto end; + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); + } + char* pData = pStart; + + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + if (needChangeLength) { + pStart += htonl(colLength[j]); + } else { + pStart += colLength[j]; + } } + }else{ + for (int i = 0; i < numFields; i++) { + for (int j = 0; j < boundInfo->numOfBound; j++){ + SSchema* pColSchema = &pSchema[j]; + if(strcmp(pColSchema->name, tFields[i].name) == 0){ + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); + } + char* pData = pStart; + + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + if (needChangeLength) { + pStart += htonl(colLength[i]); + } else { + pStart += colLength[i]; + } + boundInfo->pColIndex[j] = -1; + break; + } + } - int8_t* offset = pStart; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - pStart += numOfRows * sizeof(int32_t); - } else { - pStart += BitmapLen(numOfRows); } - char* pData = pStart; - tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); - fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength) { - pStart += htonl(colLength[c]); - } else { - pStart += colLength[c]; + for (int c = 0; c < boundInfo->numOfBound; ++c) { + if( boundInfo->pColIndex[c] != -1){ + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); + tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); + }else{ + boundInfo->pColIndex[c] = c; // restore for next block + } } } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 366831ee1af056e09ed17f9fec0e5975322c059e..312584994fe593085a83dca11e763b9ef4d4410b 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -127,6 +127,7 @@ static SKeyword keywordTable[] = { {"LANGUAGE", TK_LANGUAGE}, {"LAST", TK_LAST}, {"LAST_ROW", TK_LAST_ROW}, + {"LEADER", TK_LEADER}, {"LICENCES", TK_LICENCES}, {"LIKE", TK_LIKE}, {"LIMIT", TK_LIMIT}, @@ -177,6 +178,7 @@ static SKeyword keywordTable[] = { {"READ", TK_READ}, {"REDISTRIBUTE", TK_REDISTRIBUTE}, {"RENAME", TK_RENAME}, + {"REPLACE", TK_REPLACE}, {"REPLICA", TK_REPLICA}, {"RESET", TK_RESET}, {"RETENTIONS", TK_RETENTIONS}, @@ -237,7 +239,7 @@ static SKeyword keywordTable[] = { {"TTL", TK_TTL}, {"UNION", TK_UNION}, {"UNSIGNED", TK_UNSIGNED}, - {"UPDATE", TK_UPDATE}, + {"UPDATE", TK_UPDATE}, {"USE", TK_USE}, {"USER", TK_USER}, {"USERS", TK_USERS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fdc0a08371ee31f1e6a7d53dd3ed550b1a042d20..25e92a55ec2adce02505fc4b3c78615764ece5b9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -645,6 +645,10 @@ static bool isSelectStmt(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt); } +static bool isDeleteStmt(SNode* pCurrStmt) { + return NULL != pCurrStmt && QUERY_NODE_DELETE_STMT == nodeType(pCurrStmt); +} + static bool isSetOperator(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt); } @@ -669,6 +673,9 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) { if (NULL != pCurrStmt && QUERY_NODE_CREATE_STREAM_STMT == nodeType(pCurrStmt)) { return getPrecisionFromCurrStmt(((SCreateStreamStmt*)pCurrStmt)->pQuery, defaultVal); } + if (isDeleteStmt(pCurrStmt)) { + return ((SDeleteStmt*)pCurrStmt)->precision; + } return defaultVal; } @@ -688,6 +695,10 @@ static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isInterpPseudoColumnFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } @@ -755,9 +766,11 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { } else if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { SFunctionNode* pFunc = (SFunctionNode*)pExpr; if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || - FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { + FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType || + FUNCTION_TYPE_LAST_ROW == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); - } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) { + } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || + FUNCTION_TYPE_IROWTS == pFunc->funcType) { return true; } } @@ -976,7 +989,8 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { - if (NULL == pCxt->pCurrStmt || isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (NULL == pCxt->pCurrStmt || + (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName); } @@ -1232,6 +1246,10 @@ static int32_t calcTypeBytes(SDataType dt) { } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + if (pVal->translate) { + return TSDB_CODE_SUCCESS; + } + SDataType dt = pVal->node.resType; dt.bytes = calcTypeBytes(dt); return translateValueImpl(pCxt, pVal, dt, false); @@ -1292,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -1331,6 +1350,32 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) return code; } +static bool isCountNotNullValue(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_VALUE == nodeType(pPara) && !((SValueNode*)pPara)->isNull); +} + +// count(1) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountNotNullValue(STranslateContext* pCxt, SFunctionNode* pCount) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + NODES_DESTORY_LIST(pCount->pParameterList); + code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); + } + } + return code; +} + static bool isCountTbname(SFunctionNode* pFunc) { if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { return false; @@ -1396,6 +1441,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (isCountStar(pFunc)) { return rewriteCountStar(pCxt, pFunc); } + if (isCountNotNullValue(pFunc)) { + return rewriteCountNotNullValue(pCxt, pFunc); + } if (isCountTbname(pFunc)) { return rewriteCountTbname(pCxt, pFunc); } @@ -1490,6 +1538,21 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s must be used in select statements", pFunc->functionName); + } + if (pCxt->currClause == SQL_CLAUSE_WHERE) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, + "%s is not allowed in where clause", pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsTimelineFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1507,6 +1570,21 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu return TSDB_CODE_SUCCESS; } +static int32_t translateDateTimeFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsDateTimeFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + + if (!isSelectStmt(pCxt->pCurrStmt)) { + return TSDB_CODE_SUCCESS; + } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + pFunc->node.resType.precision = pSelect->precision; + + return TSDB_CODE_SUCCESS; +} + static bool hasFillClause(SNode* pCurrStmt) { if (!isSelectStmt(pCurrStmt)) { return false; @@ -1605,13 +1683,15 @@ static bool isTableStar(SNode* pNode) { (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); } +static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); } + static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (isStar(pPara) || isTableStar(pPara)) { + if (isStarParam(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s(*) is only supported in SELECTed list", pFunc->functionName); } @@ -1625,7 +1705,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu static int32_t getMultiResFuncNum(SNodeList* pParameterList) { if (1 == LIST_LENGTH(pParameterList)) { - return isStar(nodesListGetNode(pParameterList, 0)) ? 2 : 1; + return isStarParam(nodesListGetNode(pParameterList, 0)) ? 2 : 1; } return LIST_LENGTH(pParameterList); } @@ -1644,6 +1724,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); + if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { pSelect->hasIndefiniteRowsFunc = true; pSelect->returnRows = fmGetFuncReturnRows(pFunc); @@ -1658,6 +1739,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); + pSelect->hasInterpPseudoColFunc = + pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType); pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); @@ -1784,9 +1867,15 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateInterpFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateInterpPseudoColumnFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateDateTimeFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateBlockDistFunc(pCxt, pFunc); } @@ -2041,7 +2130,7 @@ static int32_t getGroupByErrorCode(STranslateContext* pCxt) { if (isSelectStmt(pCxt->pCurrStmt) && NULL != ((SSelectStmt*)pCxt->pCurrStmt)->pGroupByList) { return TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION; } - return TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN; + return TSDB_CODE_PAR_INVALID_OPTR_USAGE; } static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode) { @@ -2114,13 +2203,13 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } else { return rewriteColToSelectValFunc(pCxt, pNode); } } if (isVectorFunc(*pNode) && isDistinctOrderBy(pCxt)) { - return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt)); + return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); } return DEAL_RES_CONTINUE; } @@ -2500,9 +2589,8 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { STempTableNode* pTempTable = (STempTableNode*)pTable; code = translateSubquery(pCxt, pTempTable->pSubquery); if (TSDB_CODE_SUCCESS == code) { - if (QUERY_NODE_SELECT_STMT == nodeType(pTempTable->pSubquery) && - ((SSelectStmt*)pTempTable->pSubquery)->isEmptyResult && - isSelectStmt(pCxt->pCurrStmt)) { + if (QUERY_NODE_SELECT_STMT == nodeType(pTempTable->pSubquery) && + ((SSelectStmt*)pTempTable->pSubquery)->isEmptyResult && isSelectStmt(pCxt->pCurrStmt)) { ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } @@ -2843,6 +2931,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList if (TSDB_CODE_SUCCESS == code) { code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + } else if (TSDB_CODE_SUCCESS != code) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + } return code; } @@ -2859,9 +2952,9 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } - if (TSDB_CODE_SUCCESS != - convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + int32_t code = convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo); + if (TSDB_CODE_SUCCESS != code) { + return code; } ++fillNo; } @@ -2935,16 +3028,20 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = translateFillValues(pCxt, pSelect); } + if (NULL == pSelect->pProjectionList || 0 >= pSelect->pProjectionList->length) { + code = TSDB_CODE_PAR_INVALID_SELECTED_EXPR; + } return code; } static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + if (NULL == pSelect->pGroupByList && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && + NULL != pSelect->pHaving) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); } pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, &pSelect->pHaving); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && (NULL != pSelect->pGroupByList || NULL != pSelect->pWindow)) { code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; @@ -3086,6 +3183,19 @@ static const char* getPrecisionStr(uint8_t precision) { return "unknown"; } +static void convertVarDuration(SValueNode* pOffset, uint8_t precision) { + const int64_t factors[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; + const int8_t units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND}; + + if (pOffset->unit == 'n') { + pOffset->datum.i = pOffset->datum.i * 31 * (NANOSECOND_PER_DAY / factors[precision]); + } else { + pOffset->datum.i = pOffset->datum.i * 365 * (NANOSECOND_PER_DAY / factors[precision]); + } + + pOffset->unit = units[precision]; +} + static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; @@ -3110,6 +3220,10 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); } + + if (pOffset->unit == 'n' || pOffset->unit == 'y') { + convertVarDuration(pOffset, precision); + } } if (NULL != pInterval->pSliding) { @@ -3317,6 +3431,10 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } + if (pSelect->hasInterpPseudoColFunc) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "Has Interp pseudo column(s) but missing interp function"); + } return TSDB_CODE_SUCCESS; } @@ -3370,8 +3488,8 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_OFFSET_LESS_ZERO); } - if (NULL != pSelect->pSlimit && NULL == pSelect->pPartitionByList) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY); + if (NULL != pSelect->pSlimit && (NULL == pSelect->pPartitionByList && NULL == pSelect->pGroupByList)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_GROUP_BY); } return TSDB_CODE_SUCCESS; @@ -3684,6 +3802,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { + pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision; code = translateDeleteWhere(pCxt, pDelete); } pCxt->currClause = SQL_CLAUSE_SELECT; @@ -4251,6 +4370,9 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->replications = pStmt->pOptions->replica; pReq->sstTrigger = pStmt->pOptions->sstTrigger; + pReq->minRows = pStmt->pOptions->minRowsPerBlock; + pReq->walRetentionPeriod = pStmt->pOptions->walRetentionPeriod; + pReq->walRetentionSize = pStmt->pOptions->walRetentionSize; return; } @@ -5083,26 +5205,32 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - int32_t tagsLen = 0; for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) { tagsLen += pTagsSchema[i].bytes; } if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || - TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) { + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType || + TSDB_ALTER_TABLE_DROP_TAG == pStmt->alterType) { if (TSDB_SUPER_TABLE != pTableMeta->tableType) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); - } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || - pSchema->bytes >= calcTypeBytes(pStmt->dataType)) { + return generateSyntaxErrMsg( + &pCxt->msgBuf, + (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType) + ? TSDB_CODE_PAR_INVALID_COLUMN + : TSDB_CODE_PAR_INVALID_TAG_NAME, + pStmt->colName); + } + + if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) && + (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || + pSchema->bytes >= calcTypeBytes(pStmt->dataType))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } @@ -5147,6 +5275,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } } + if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); + } + return TSDB_CODE_SUCCESS; } @@ -5570,7 +5702,8 @@ static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && + if (!pSelect->isDistinct && + (NULL != pSelect->pFromTable && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) && NULL == pSelect->pGroupByList && NULL == pSelect->pLimit && NULL == pSelect->pSlimit && NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; @@ -5618,6 +5751,14 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { // The statement is executed directly on the client without constructing a message. + if ('\0' != pStmt->value[0]) { + return TSDB_CODE_SUCCESS; + } + char* p = strchr(pStmt->config, ' '); + if (NULL != p) { + *p = 0; + strcpy(pStmt->value, p + 1); + } return TSDB_CODE_SUCCESS; } @@ -5890,11 +6031,15 @@ static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STabl return code; } +static bool isEventWindowQuery(SSelectStmt* pSelect) { + return NULL != pSelect->pWindow && QUERY_NODE_EVENT_WINDOW == nodeType(pSelect->pWindow); +} + static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || - crossTableWithUdaf(pSelect)) { + crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) { @@ -6389,12 +6534,25 @@ static int32_t translateCreateFunction(STranslateContext* pCxt, SCreateFunctionS if (fmIsBuiltinFunc(pStmt->funcName)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNCTION_NAME); } + + if (TSDB_DATA_TYPE_JSON == pStmt->outputDt.type || TSDB_DATA_TYPE_VARBINARY == pStmt->outputDt.type || + TSDB_DATA_TYPE_DECIMAL == pStmt->outputDt.type || TSDB_DATA_TYPE_BLOB == pStmt->outputDt.type || + TSDB_DATA_TYPE_MEDIUMBLOB == pStmt->outputDt.type) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Unsupported output type for UDF"); + } + + if (!pStmt->isAgg && pStmt->bufSize > 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "BUFSIZE can only be used with UDAF"); + } + SCreateFuncReq req = {0}; strcpy(req.name, pStmt->funcName); + req.orReplace = pStmt->orReplace; req.igExists = pStmt->ignoreExists; req.funcType = pStmt->isAgg ? TSDB_FUNC_TYPE_AGGREGATE : TSDB_FUNC_TYPE_SCALAR; req.scriptType = pStmt->language; req.outputType = pStmt->outputDt.type; + pStmt->outputDt.bytes = calcTypeBytes(pStmt->outputDt); req.outputLen = pStmt->outputDt.bytes; req.bufSize = pStmt->bufSize; int32_t code = readFromFile(pStmt->libraryPath, &req.codeLen, &req.pCode); @@ -6412,22 +6570,69 @@ static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt* return buildCmdMsg(pCxt, TDMT_MND_DROP_FUNC, (FSerializeFunc)tSerializeSDropFuncReq, &req); } +static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** pTable) { + SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pRealTable) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pRealTable->table.dbName, pStmt->objName); + strcpy(pRealTable->table.tableName, pStmt->tabName); + strcpy(pRealTable->table.tableAlias, pStmt->tabName); + *pTable = pRealTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) { + if (NULL == pStmt->pTagCond) { + return TSDB_CODE_SUCCESS; + } + if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "The With clause can only be used for table level privilege"); + } + + pCxt->pCurrStmt = (SNode*)pStmt; + SRealTableNode* pTable = NULL; + int32_t code = createRealTableForGrantTable(pStmt, &pTable); + if (TSDB_CODE_SUCCESS == code) { + SName name; + code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), + &(pTable->pMeta)); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNamespace(pCxt, pTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pStmt->pTagCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pStmt->pTagCond, false, &pReq->tagCond, &pReq->tagCondLen); + } + nodesDestroyNode((SNode*)pTable); + return code; +} + static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { SAlterUserReq req = {0}; if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) { - req.alterType = TSDB_ALTER_USER_ADD_ALL_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_ALL_DB : TSDB_ALTER_USER_ADD_ALL_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { - req.alterType = TSDB_ALTER_USER_ADD_READ_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_READ_DB : TSDB_ALTER_USER_ADD_READ_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { - req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_WRITE_DB : TSDB_ALTER_USER_ADD_WRITE_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC; } strcpy(req.user, pStmt->userName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); + sprintf(req.tabName, "%s", pStmt->tabName); + int32_t code = translateGrantTagCond(pCxt, pStmt, &req); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); + } + return code; } static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { @@ -6435,16 +6640,17 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) { - req.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_ALL_DB : TSDB_ALTER_USER_REMOVE_ALL_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { - req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_READ_DB : TSDB_ALTER_USER_REMOVE_READ_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { - req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_WRITE_DB : TSDB_ALTER_USER_REMOVE_WRITE_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC; } strcpy(req.user, pStmt->userName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + sprintf(req.tabName, "%s", pStmt->tabName); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } @@ -6453,6 +6659,11 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm return buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP, (FSerializeFunc)tSerializeSBalanceVgroupReq, &req); } +static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { + SBalanceVgroupLeaderReq req = {0}; + return buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); +} + static int32_t translateMergeVgroup(STranslateContext* pCxt, SMergeVgroupStmt* pStmt) { SMergeVgroupReq req = {.vgId1 = pStmt->vgId1, .vgId2 = pStmt->vgId2}; return buildCmdMsg(pCxt, TDMT_MND_MERGE_VGROUP, (FSerializeFunc)tSerializeSMergeVgroupReq, &req); @@ -6664,6 +6875,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_BALANCE_VGROUP_STMT: code = translateBalanceVgroup(pCxt, (SBalanceVgroupStmt*)pNode); break; + case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: + code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode); + break; case QUERY_NODE_MERGE_VGROUP_STMT: code = translateMergeVgroup(pCxt, (SMergeVgroupStmt*)pNode); break; @@ -8123,10 +8337,6 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) { static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SQuery* pQuery) { - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - if (TSDB_SUPER_TABLE == pTableMeta->tableType) { return TSDB_CODE_SUCCESS; } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 89a42cb03c29fcc97570610d52b5bc15a663fc9b..14da6f8aabcfba0cfdc3a71a731b1c34acdb32d5 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -17,7 +17,7 @@ #include "cJSON.h" #include "querynodes.h" -#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_DB_FNAME_LEN + 2 +#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_TABLE_FNAME_LEN + 2 const void* nullPointer = NULL; @@ -103,8 +103,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Incorrect TIMESTAMP value: %s"; case TSDB_CODE_PAR_OFFSET_LESS_ZERO: return "soffset/offset can not be less than 0"; - case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY: - return "slimit/soffset only available for PARTITION BY query"; + case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_GROUP_BY: + return "slimit/soffset only available for PARTITION/GROUP BY query"; case TSDB_CODE_PAR_INVALID_TOPIC_QUERY: return "Invalid topic query"; case TSDB_CODE_PAR_INVALID_DROP_STABLE: @@ -174,6 +174,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid usage of RANGE clause, EVERY clause or FILL clause"; case TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN: return "No valid function in window query"; + case TSDB_CODE_PAR_INVALID_OPTR_USAGE: + return "Invalid usage of expr: %s"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -442,7 +444,7 @@ static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pO } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { pVal = (SValueNode*)pOper->pRight; } - if (NULL == pCol || NULL == pVal) { + if (NULL == pCol || NULL == pVal || NULL == pVal->literal || 0 == strcmp(pVal->literal, "")) { return TSDB_CODE_SUCCESS; } @@ -494,24 +496,44 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) return TSDB_CODE_SUCCESS; } -static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); +static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, + char* pStr) { + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + type); } -static int32_t userAuthToStringExt(const char* pUser, const char* pDbFName, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%s*%d", pUser, pDbFName, type); +static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { + char* p = strchr(pStart, '*'); + char buf[10] = {0}; + if (NULL == p) { + strcpy(buf, pStart); + *pNext = NULL; + } else { + strncpy(buf, pStart, p - pStart); + *pNext = ++p; + } + return taosStr2Int32(buf, NULL, 10); +} + +static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { + char* p = strchr(pStart, '*'); + if (NULL == p) { + strcpy(pStr, pStart); + *pNext = NULL; + } else { + strncpy(pStr, pStart, p - pStart); + *pNext = ++p; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { - char* p1 = strchr(pStr, '*'); - strncpy(pUserAuth->user, pStr, p1 - pStr); - ++p1; - char* p2 = strchr(p1, '*'); - strncpy(pUserAuth->dbFName, p1, p2 - p1); - ++p2; - char buf[10] = {0}; - strncpy(buf, p2, len - (p2 - pStr)); - pUserAuth->type = taosStr2Int32(buf, NULL, 10); + char* p = NULL; + getStringFromAuthStr(pStr, pUserAuth->user, &p); + pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); + getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); + getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + pUserAuth->type = getIntegerFromAuthStr(p, &p); } static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { @@ -582,10 +604,12 @@ static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) { } void* p = taosHashIterate(pUserAuthHash, NULL); while (NULL != p) { - size_t len = 0; - char* pKey = taosHashGetKey(p, &len); + size_t len = 0; + char* pKey = taosHashGetKey(p, &len); + char key[USER_AUTH_KEY_MAX_LEN] = {0}; + strncpy(key, pKey, len); SUserAuthInfo userAuth = {0}; - stringToUserAuth(pKey, len, &userAuth); + stringToUserAuth(key, len, &userAuth); taosArrayPush(*pUserAuth, &userAuth); p = taosHashIterate(pUserAuthHash, p); } @@ -710,7 +734,8 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse for (int32_t i = 0; i < nvgs; ++i) { SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i); char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key); + int32_t len = userAuthToString(pUser->tbName.acctId, pUser->user, pUser->tbName.dbname, pUser->tbName.tname, + pUser->type, key); if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -908,33 +933,24 @@ static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseM return TSDB_CODE_OUT_OF_MEMORY; } } - bool pass = false; - return taosHashPut(pMetaCache->pUserAuth, pKey, len, &pass, sizeof(pass)); + return taosHashPut(pMetaCache->pUserAuth, pKey, len, &nullPointer, POINTER_BYTES); } -int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, +int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, SParseMetaCache* pMetaCache) { char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToString(acctId, pUser, pDb, type, key); - return reserveUserAuthInCacheImpl(key, len, pMetaCache); -} - -int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); - char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser, dbFName, type, key); + int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key); return reserveUserAuthInCacheImpl(key, len, pMetaCache); } -int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, - bool* pPass) { - char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser, pDbFName, type, key); - bool* pRes = NULL; - int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pRes); +int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes) { + char key[USER_AUTH_KEY_MAX_LEN] = {0}; + int32_t len = userAuthToString(pAuthReq->tbName.acctId, pAuthReq->user, pAuthReq->tbName.dbname, + pAuthReq->tbName.tname, pAuthReq->type, key); + SUserAuthRes* pAuth = NULL; + int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pAuth); if (TSDB_CODE_SUCCESS == code) { - *pPass = *pRes; + memcpy(pAuthRes, pAuth, sizeof(SUserAuthRes)); } return code; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ab0571206babbfcc08d6c0c904c71259cae957c3..a38e2368ddd23702b0e2b5a993b2e11da1fbc950 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 473 +#define YYNOCODE 476 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOperatorType yy62; - SToken yy63; - EOrder yy158; - SNodeList* yy222; - SAlterOption yy233; - int32_t yy332; - int64_t yy393; - EJoinType yy574; - bool yy669; - ENullOrder yy675; - SDataType yy686; - int8_t yy707; - EFillMode yy822; - SNode* yy828; + EFillMode yy46; + SAlterOption yy53; + SToken yy113; + EOperatorType yy156; + bool yy369; + SNodeList* yy432; + SNode* yy448; + int8_t yy551; + ENullOrder yy585; + EJoinType yy596; + EOrder yy666; + SDataType yy728; + STokenPair yy777; + int32_t yy788; + int64_t yy837; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +140,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 753 -#define YYNRULE 573 -#define YYNTOKEN 329 -#define YY_MAX_SHIFT 752 -#define YY_MIN_SHIFTREDUCE 1119 -#define YY_MAX_SHIFTREDUCE 1691 -#define YY_ERROR_ACTION 1692 -#define YY_ACCEPT_ACTION 1693 -#define YY_NO_ACTION 1694 -#define YY_MIN_REDUCE 1695 -#define YY_MAX_REDUCE 2267 +#define YYNSTATE 762 +#define YYNRULE 583 +#define YYNTOKEN 330 +#define YY_MAX_SHIFT 761 +#define YY_MIN_SHIFTREDUCE 1136 +#define YY_MAX_SHIFTREDUCE 1718 +#define YY_ERROR_ACTION 1719 +#define YY_ACCEPT_ACTION 1720 +#define YY_NO_ACTION 1721 +#define YY_MIN_REDUCE 1722 +#define YY_MAX_REDUCE 2304 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,773 +217,814 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2833) +#define YY_ACTTAB_COUNT (3030) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2065, 2243, 1976, 167, 2079, 2238, 491, 377, 492, 1731, - /* 10 */ 1803, 2061, 45, 43, 1619, 164, 1976, 1974, 629, 2065, - /* 20 */ 385, 2242, 1468, 617, 1859, 2239, 2241, 2079, 376, 1493, - /* 30 */ 2061, 1973, 629, 1549, 166, 1466, 1707, 2097, 1848, 44, - /* 40 */ 42, 41, 40, 39, 641, 620, 2057, 2063, 366, 2061, - /* 50 */ 2047, 500, 658, 492, 1731, 388, 139, 652, 1544, 176, - /* 60 */ 2097, 1493, 1910, 161, 18, 2057, 2063, 367, 659, 350, - /* 70 */ 364, 1474, 1859, 2047, 178, 658, 652, 497, 1908, 1908, - /* 80 */ 352, 1959, 2078, 493, 2057, 2063, 2114, 1897, 641, 109, - /* 90 */ 2080, 662, 2082, 2083, 657, 652, 652, 749, 35, 291, - /* 100 */ 14, 179, 617, 2167, 627, 2078, 430, 379, 2163, 2114, - /* 110 */ 45, 43, 168, 2080, 662, 2082, 2083, 657, 385, 652, - /* 120 */ 1468, 184, 270, 2175, 616, 1764, 133, 615, 2066, 2193, - /* 130 */ 2238, 1549, 1171, 1466, 1170, 139, 1551, 1552, 490, 2061, - /* 140 */ 605, 495, 1737, 1623, 2238, 604, 182, 222, 333, 1493, - /* 150 */ 2239, 606, 1695, 581, 2204, 1152, 1544, 1834, 141, 604, - /* 160 */ 182, 2138, 18, 1172, 2239, 606, 1524, 1534, 48, 1474, - /* 170 */ 1761, 509, 1550, 1553, 2057, 2063, 131, 130, 129, 128, - /* 180 */ 127, 126, 125, 124, 123, 652, 1469, 641, 1467, 176, - /* 190 */ 61, 545, 544, 543, 1154, 749, 1157, 1158, 14, 136, - /* 200 */ 539, 181, 2175, 2176, 538, 137, 2180, 1385, 1386, 537, - /* 210 */ 542, 1960, 468, 1472, 1473, 536, 1523, 1526, 1527, 1528, - /* 220 */ 1529, 1530, 1531, 1532, 1533, 654, 650, 1542, 1543, 1545, - /* 230 */ 1546, 1547, 1548, 2, 1551, 1552, 726, 725, 724, 723, - /* 240 */ 395, 600, 722, 721, 143, 716, 715, 714, 713, 712, - /* 250 */ 711, 710, 156, 706, 705, 704, 394, 393, 701, 700, - /* 260 */ 699, 698, 697, 1616, 1524, 1534, 595, 48, 696, 694, - /* 270 */ 1550, 1553, 122, 198, 197, 121, 120, 119, 118, 117, - /* 280 */ 116, 115, 114, 113, 1469, 694, 1467, 154, 153, 691, - /* 290 */ 690, 689, 151, 38, 37, 1525, 467, 44, 42, 41, - /* 300 */ 40, 39, 1492, 154, 153, 691, 690, 689, 151, 1693, - /* 310 */ 2182, 1472, 1473, 273, 1523, 1526, 1527, 1528, 1529, 1530, - /* 320 */ 1531, 1532, 1533, 654, 650, 1542, 1543, 1545, 1546, 1547, - /* 330 */ 1548, 2, 11, 45, 43, 390, 2179, 2097, 1903, 1905, - /* 340 */ 185, 385, 1296, 1468, 499, 599, 49, 495, 1737, 345, - /* 350 */ 601, 596, 589, 1940, 1549, 61, 1466, 92, 1287, 684, - /* 360 */ 683, 682, 1291, 681, 1293, 1294, 680, 677, 86, 1302, - /* 370 */ 674, 1304, 1305, 671, 668, 1696, 31, 398, 2079, 1544, - /* 380 */ 568, 397, 38, 37, 65, 18, 44, 42, 41, 40, - /* 390 */ 39, 598, 1474, 1853, 1578, 628, 122, 1494, 11, 121, - /* 400 */ 120, 119, 118, 117, 116, 115, 114, 113, 38, 37, - /* 410 */ 628, 2097, 44, 42, 41, 40, 39, 185, 749, 656, - /* 420 */ 1496, 14, 642, 580, 2047, 1474, 658, 2238, 11, 1910, - /* 430 */ 9, 45, 43, 1554, 628, 687, 363, 2015, 187, 385, - /* 440 */ 1250, 1468, 2244, 182, 507, 1908, 1969, 2239, 606, 1718, - /* 450 */ 1579, 1647, 1549, 1249, 1466, 1857, 2078, 1551, 1552, 626, - /* 460 */ 2114, 1969, 2243, 327, 2080, 662, 2082, 2083, 657, 655, - /* 470 */ 652, 643, 2132, 545, 544, 543, 1239, 1544, 617, 1495, - /* 480 */ 61, 136, 539, 637, 240, 1969, 538, 1524, 1534, 2242, - /* 490 */ 1474, 537, 542, 1550, 1553, 2047, 185, 536, 592, 591, - /* 500 */ 1645, 1646, 1648, 1649, 1650, 185, 708, 1469, 448, 1467, - /* 510 */ 1615, 139, 1241, 1330, 1331, 477, 749, 447, 152, 46, - /* 520 */ 541, 540, 34, 383, 1573, 1574, 1575, 1576, 1577, 1581, - /* 530 */ 1582, 1583, 1584, 1495, 1472, 1473, 575, 1523, 1526, 1527, - /* 540 */ 1528, 1529, 1530, 1531, 1532, 1533, 654, 650, 1542, 1543, - /* 550 */ 1545, 1546, 1547, 1548, 2, 1551, 1552, 38, 37, 1846, - /* 560 */ 1681, 44, 42, 41, 40, 39, 38, 37, 250, 1717, - /* 570 */ 44, 42, 41, 40, 39, 53, 619, 180, 2175, 2176, - /* 580 */ 2079, 137, 2180, 642, 605, 1524, 1534, 1708, 2238, 38, - /* 590 */ 37, 1550, 1553, 44, 42, 41, 40, 39, 2243, 132, - /* 600 */ 1842, 642, 2238, 604, 182, 1469, 530, 1467, 2239, 606, - /* 610 */ 1716, 1493, 644, 2097, 2139, 2047, 1857, 132, 2242, 239, - /* 620 */ 2068, 659, 2239, 2240, 535, 27, 2047, 272, 658, 646, - /* 630 */ 185, 2139, 1472, 1473, 1857, 1523, 1526, 1527, 1528, 1529, - /* 640 */ 1530, 1531, 1532, 1533, 654, 650, 1542, 1543, 1545, 1546, - /* 650 */ 1547, 1548, 2, 45, 43, 86, 2047, 2079, 2078, 1658, - /* 660 */ 424, 385, 2114, 1468, 423, 109, 2080, 662, 2082, 2083, - /* 670 */ 657, 354, 652, 2070, 1549, 142, 1466, 149, 2138, 2167, - /* 680 */ 1852, 272, 1580, 379, 2163, 1442, 1443, 1468, 38, 37, - /* 690 */ 2097, 61, 44, 42, 41, 40, 39, 1910, 659, 1544, - /* 700 */ 1466, 617, 2182, 2047, 373, 658, 580, 642, 33, 709, - /* 710 */ 2238, 1819, 1474, 1908, 38, 37, 1904, 1905, 44, 42, - /* 720 */ 41, 40, 39, 54, 1844, 2244, 182, 1688, 2178, 238, - /* 730 */ 2239, 606, 642, 237, 139, 2078, 1474, 642, 749, 2114, - /* 740 */ 1857, 46, 169, 2080, 662, 2082, 2083, 657, 621, 652, - /* 750 */ 357, 45, 43, 428, 32, 1171, 608, 1170, 388, 385, - /* 760 */ 1910, 1468, 749, 391, 1585, 1857, 164, 378, 282, 283, - /* 770 */ 1857, 164, 1549, 281, 1466, 1859, 1908, 1551, 1552, 1835, - /* 780 */ 1859, 1635, 1715, 38, 37, 1832, 1172, 44, 42, 41, - /* 790 */ 40, 39, 89, 340, 607, 2259, 362, 1544, 561, 2079, - /* 800 */ 183, 2175, 2176, 1714, 137, 2180, 106, 1524, 1534, 2182, - /* 810 */ 1474, 720, 718, 1550, 1553, 642, 358, 534, 356, 355, - /* 820 */ 417, 532, 140, 147, 1687, 534, 1910, 1469, 2047, 1467, - /* 830 */ 1849, 429, 2097, 389, 8, 2177, 749, 559, 533, 14, - /* 840 */ 659, 185, 1908, 419, 415, 2047, 533, 658, 1857, 2047, - /* 850 */ 557, 1469, 555, 1467, 1472, 1473, 1713, 1523, 1526, 1527, - /* 860 */ 1528, 1529, 1530, 1531, 1532, 1533, 654, 650, 1542, 1543, - /* 870 */ 1545, 1546, 1547, 1548, 2, 1551, 1552, 2078, 1472, 1473, - /* 880 */ 623, 2114, 13, 12, 110, 2080, 662, 2082, 2083, 657, - /* 890 */ 100, 652, 1496, 1559, 41, 40, 39, 694, 2167, 1493, - /* 900 */ 105, 1712, 2047, 2164, 164, 1524, 1534, 642, 642, 1494, - /* 910 */ 102, 1550, 1553, 1860, 1850, 154, 153, 691, 690, 689, - /* 920 */ 151, 1910, 1496, 438, 453, 1469, 580, 1467, 38, 37, - /* 930 */ 2238, 1804, 44, 42, 41, 40, 39, 1909, 1955, 688, - /* 940 */ 1857, 1857, 1901, 163, 1711, 2244, 182, 2047, 1833, 190, - /* 950 */ 2239, 606, 1472, 1473, 73, 1523, 1526, 1527, 1528, 1529, - /* 960 */ 1530, 1531, 1532, 1533, 654, 650, 1542, 1543, 1545, 1546, - /* 970 */ 1547, 1548, 2, 61, 336, 165, 1491, 2040, 382, 381, - /* 980 */ 311, 1955, 1592, 461, 1254, 550, 475, 2079, 1482, 474, - /* 990 */ 2047, 1955, 192, 305, 309, 72, 1887, 1253, 71, 1549, - /* 1000 */ 560, 1475, 196, 81, 444, 194, 476, 1710, 431, 446, - /* 1010 */ 1709, 696, 692, 609, 236, 1901, 205, 487, 485, 482, - /* 1020 */ 2097, 432, 1706, 580, 1544, 1402, 1403, 2238, 659, 553, - /* 1030 */ 1157, 1158, 1705, 2047, 547, 658, 1840, 1474, 642, 235, - /* 1040 */ 191, 1704, 2244, 182, 1612, 1525, 83, 2239, 606, 82, - /* 1050 */ 248, 1703, 1702, 2047, 625, 353, 2047, 61, 422, 479, - /* 1060 */ 421, 1401, 1404, 648, 1493, 2078, 612, 434, 2047, 2114, - /* 1070 */ 249, 1857, 109, 2080, 662, 2082, 2083, 657, 2047, 652, - /* 1080 */ 69, 1701, 2065, 68, 2142, 420, 2167, 2047, 1700, 693, - /* 1090 */ 379, 2163, 1901, 2061, 90, 108, 472, 2047, 2047, 466, - /* 1100 */ 465, 464, 463, 460, 459, 458, 457, 456, 452, 451, - /* 1110 */ 450, 449, 335, 441, 440, 439, 642, 436, 435, 351, - /* 1120 */ 686, 228, 642, 185, 226, 1699, 642, 2047, 2057, 2063, - /* 1130 */ 380, 642, 392, 2033, 2047, 80, 79, 427, 454, 652, - /* 1140 */ 189, 2079, 508, 2187, 1612, 642, 51, 1854, 3, 1857, - /* 1150 */ 1477, 145, 1483, 134, 1478, 1857, 649, 642, 245, 1857, - /* 1160 */ 334, 241, 1751, 413, 1857, 411, 407, 403, 400, 420, - /* 1170 */ 642, 2047, 2079, 576, 2097, 563, 653, 562, 1857, 1486, - /* 1180 */ 1488, 405, 620, 267, 546, 152, 286, 2047, 642, 658, - /* 1190 */ 1857, 1476, 650, 1542, 1543, 1545, 1546, 1547, 1548, 642, - /* 1200 */ 1698, 642, 1744, 1857, 639, 2097, 230, 185, 152, 229, - /* 1210 */ 1525, 232, 2207, 659, 231, 640, 63, 292, 2047, 2078, - /* 1220 */ 658, 1857, 2079, 2114, 548, 1742, 109, 2080, 662, 2082, - /* 1230 */ 2083, 657, 1857, 652, 1857, 63, 47, 254, 179, 2079, - /* 1240 */ 2167, 610, 1437, 579, 379, 2163, 2047, 551, 234, 152, - /* 1250 */ 2078, 233, 13, 12, 2114, 2097, 47, 109, 2080, 662, - /* 1260 */ 2082, 2083, 657, 659, 652, 1440, 2194, 1738, 2047, 2258, - /* 1270 */ 658, 2167, 2097, 1644, 593, 379, 2163, 279, 1690, 1691, - /* 1280 */ 659, 221, 70, 261, 150, 2047, 2201, 658, 2098, 152, - /* 1290 */ 1732, 63, 1643, 1535, 256, 396, 613, 47, 666, 2079, - /* 1300 */ 2078, 150, 152, 1480, 2114, 1570, 624, 168, 2080, 662, - /* 1310 */ 2082, 2083, 657, 1399, 652, 52, 744, 2078, 1964, 2079, - /* 1320 */ 1898, 2114, 135, 150, 109, 2080, 662, 2082, 2083, 657, - /* 1330 */ 2197, 652, 2097, 1201, 284, 702, 2258, 752, 2167, 634, - /* 1340 */ 659, 288, 379, 2163, 1479, 2047, 1280, 658, 1586, 2205, - /* 1350 */ 618, 298, 2097, 2214, 304, 1308, 703, 1220, 1312, 1319, - /* 1360 */ 659, 269, 266, 1, 4, 2047, 175, 658, 404, 1202, - /* 1370 */ 399, 349, 742, 738, 734, 730, 296, 2078, 1218, 1317, - /* 1380 */ 155, 2114, 1422, 299, 109, 2080, 662, 2082, 2083, 657, - /* 1390 */ 195, 652, 433, 1496, 1965, 437, 2258, 2078, 2167, 470, - /* 1400 */ 1491, 2114, 379, 2163, 109, 2080, 662, 2082, 2083, 657, - /* 1410 */ 442, 652, 455, 587, 1957, 462, 2258, 107, 2167, 469, - /* 1420 */ 289, 471, 379, 2163, 480, 481, 478, 199, 2079, 200, - /* 1430 */ 483, 484, 2041, 2232, 202, 223, 1497, 486, 488, 489, - /* 1440 */ 498, 1499, 501, 1494, 502, 208, 210, 1498, 2079, 503, - /* 1450 */ 171, 1500, 638, 504, 506, 1174, 526, 522, 518, 514, - /* 1460 */ 220, 2097, 510, 213, 215, 84, 85, 219, 527, 659, - /* 1470 */ 528, 529, 112, 339, 2047, 531, 658, 565, 580, 1847, - /* 1480 */ 567, 2097, 2238, 88, 148, 300, 570, 276, 225, 659, - /* 1490 */ 242, 1843, 275, 227, 2047, 157, 658, 2244, 182, 158, - /* 1500 */ 1845, 87, 2239, 606, 218, 1841, 2078, 2024, 1429, 2021, - /* 1510 */ 2114, 243, 569, 109, 2080, 662, 2082, 2083, 657, 159, - /* 1520 */ 652, 2020, 2079, 160, 246, 2258, 2078, 2167, 571, 574, - /* 1530 */ 2114, 379, 2163, 109, 2080, 662, 2082, 2083, 657, 244, - /* 1540 */ 652, 577, 2186, 2198, 584, 2140, 2079, 2167, 2208, 594, - /* 1550 */ 632, 379, 2163, 590, 2213, 2097, 252, 368, 255, 597, - /* 1560 */ 7, 603, 582, 659, 2212, 585, 611, 583, 2047, 265, - /* 1570 */ 658, 217, 211, 369, 614, 2261, 216, 1612, 505, 2097, - /* 1580 */ 138, 1495, 622, 2183, 372, 274, 1501, 659, 95, 301, - /* 1590 */ 1970, 630, 2047, 260, 658, 209, 2079, 631, 1984, 263, - /* 1600 */ 2078, 2189, 264, 172, 2114, 566, 1983, 109, 2080, 662, - /* 1610 */ 2082, 2083, 657, 2079, 652, 1982, 302, 2237, 268, 645, - /* 1620 */ 262, 2167, 635, 303, 2078, 379, 2163, 375, 2114, 2097, - /* 1630 */ 97, 110, 2080, 662, 2082, 2083, 657, 659, 652, 1858, - /* 1640 */ 636, 60, 2047, 2148, 658, 2167, 2097, 99, 101, 2166, - /* 1650 */ 2163, 580, 664, 306, 659, 2238, 295, 1820, 1902, 2047, - /* 1660 */ 745, 658, 746, 748, 341, 50, 330, 342, 308, 315, - /* 1670 */ 2244, 182, 310, 2079, 2078, 2239, 606, 2039, 2114, 2038, - /* 1680 */ 2037, 321, 2080, 662, 2082, 2083, 657, 77, 652, 329, - /* 1690 */ 2034, 2078, 319, 2079, 401, 2114, 402, 1459, 110, 2080, - /* 1700 */ 662, 2082, 2083, 657, 1460, 652, 2097, 188, 406, 2032, - /* 1710 */ 408, 409, 2167, 410, 659, 2031, 647, 2163, 412, 2047, - /* 1720 */ 2030, 658, 414, 2029, 416, 602, 2097, 2028, 418, 78, - /* 1730 */ 1425, 1424, 1996, 1995, 659, 1994, 425, 426, 1993, 2047, - /* 1740 */ 1992, 658, 1376, 1948, 1947, 1945, 144, 1944, 1943, 1946, - /* 1750 */ 1942, 660, 1941, 1939, 1938, 2114, 2079, 1937, 110, 2080, - /* 1760 */ 662, 2082, 2083, 657, 193, 652, 443, 1936, 445, 1950, - /* 1770 */ 1935, 2078, 2167, 1934, 1933, 2114, 344, 2163, 169, 2080, - /* 1780 */ 662, 2082, 2083, 657, 146, 652, 1932, 1931, 1930, 2097, - /* 1790 */ 1929, 1928, 1927, 1926, 374, 1925, 1924, 659, 1923, 1922, - /* 1800 */ 1921, 1920, 2047, 1919, 658, 1918, 1949, 1917, 1916, 1915, - /* 1810 */ 1914, 1378, 1913, 473, 2079, 1912, 1911, 337, 338, 1251, - /* 1820 */ 1767, 201, 1766, 203, 1255, 1765, 204, 1763, 1727, 1247, - /* 1830 */ 2067, 2260, 1160, 1159, 2078, 1726, 2011, 2003, 2114, 2079, - /* 1840 */ 1991, 328, 2080, 662, 2082, 2083, 657, 2097, 652, 1990, - /* 1850 */ 214, 494, 75, 206, 1968, 656, 1836, 1762, 177, 1760, - /* 1860 */ 2047, 207, 658, 76, 496, 212, 511, 1758, 513, 515, - /* 1870 */ 1756, 517, 2097, 519, 1754, 1741, 1740, 384, 512, 523, - /* 1880 */ 659, 516, 521, 1723, 1838, 2047, 525, 658, 1194, 520, - /* 1890 */ 62, 1323, 2078, 524, 1837, 1324, 2114, 1238, 1237, 327, - /* 1900 */ 2080, 662, 2082, 2083, 657, 1752, 652, 1236, 2133, 717, - /* 1910 */ 719, 1230, 2079, 1235, 1232, 1231, 1229, 2078, 359, 1745, - /* 1920 */ 360, 2114, 361, 549, 328, 2080, 662, 2082, 2083, 657, - /* 1930 */ 1743, 652, 552, 2079, 1722, 1721, 554, 556, 1720, 558, - /* 1940 */ 111, 1447, 1449, 1446, 2010, 2097, 224, 1451, 26, 1431, - /* 1950 */ 386, 66, 2002, 659, 1433, 162, 1989, 1987, 2047, 572, - /* 1960 */ 658, 2243, 2079, 259, 16, 19, 2097, 58, 59, 258, - /* 1970 */ 20, 30, 64, 2068, 659, 21, 1675, 28, 17, 2047, - /* 1980 */ 1674, 658, 251, 2079, 5, 586, 1660, 6, 588, 370, - /* 1990 */ 2078, 253, 57, 1642, 2114, 2097, 1634, 328, 2080, 662, - /* 2000 */ 2082, 2083, 657, 659, 652, 1680, 1679, 1678, 2047, 170, - /* 2010 */ 658, 564, 257, 29, 371, 2114, 2097, 271, 323, 2080, - /* 2020 */ 662, 2082, 2083, 657, 659, 652, 55, 578, 91, 2047, - /* 2030 */ 1681, 658, 573, 2079, 1609, 173, 1988, 1608, 247, 365, - /* 2040 */ 2078, 1986, 1985, 1967, 2114, 94, 93, 312, 2080, 662, - /* 2050 */ 2082, 2083, 657, 22, 652, 277, 278, 633, 2079, 96, - /* 2060 */ 1640, 2078, 280, 285, 67, 2114, 2097, 1966, 313, 2080, - /* 2070 */ 662, 2082, 2083, 657, 659, 652, 98, 102, 290, 2047, - /* 2080 */ 287, 658, 23, 1561, 1560, 12, 1484, 2117, 10, 663, - /* 2090 */ 1539, 2097, 651, 2079, 1537, 36, 174, 15, 56, 659, - /* 2100 */ 1536, 24, 186, 1508, 2047, 25, 658, 1516, 1309, 665, - /* 2110 */ 667, 2078, 1571, 387, 1306, 2114, 2079, 669, 314, 2080, - /* 2120 */ 662, 2082, 2083, 657, 670, 652, 2097, 1303, 672, 673, - /* 2130 */ 661, 1297, 675, 676, 659, 1295, 2078, 678, 1301, 2047, - /* 2140 */ 2114, 658, 1300, 320, 2080, 662, 2082, 2083, 657, 2097, - /* 2150 */ 652, 679, 1286, 103, 293, 1299, 685, 659, 1298, 1318, - /* 2160 */ 104, 74, 2047, 1314, 658, 1192, 1226, 695, 1225, 1224, - /* 2170 */ 1223, 2078, 1222, 1221, 2079, 2114, 1245, 1219, 324, 2080, - /* 2180 */ 662, 2082, 2083, 657, 1217, 652, 1216, 1215, 707, 294, - /* 2190 */ 1213, 1212, 1211, 2079, 2078, 1210, 1209, 1208, 2114, 1207, - /* 2200 */ 1242, 316, 2080, 662, 2082, 2083, 657, 2097, 652, 1240, - /* 2210 */ 1198, 1204, 1203, 1200, 1199, 659, 1197, 1759, 727, 728, - /* 2220 */ 2047, 729, 658, 1757, 731, 733, 2097, 1755, 732, 735, - /* 2230 */ 737, 1753, 736, 739, 659, 740, 741, 1739, 743, 2047, - /* 2240 */ 1149, 658, 1719, 297, 1470, 747, 751, 307, 1694, 750, - /* 2250 */ 1694, 1694, 2078, 1694, 2079, 1694, 2114, 1694, 1694, 325, - /* 2260 */ 2080, 662, 2082, 2083, 657, 1694, 652, 1694, 1694, 1694, - /* 2270 */ 1694, 2078, 1694, 1694, 1694, 2114, 1694, 1694, 317, 2080, - /* 2280 */ 662, 2082, 2083, 657, 1694, 652, 1694, 2097, 1694, 1694, - /* 2290 */ 1694, 1694, 1694, 1694, 1694, 659, 1694, 1694, 1694, 1694, - /* 2300 */ 2047, 1694, 658, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2310 */ 1694, 1694, 2079, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2320 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 2079, 1694, - /* 2330 */ 1694, 1694, 2078, 1694, 1694, 1694, 2114, 1694, 1694, 326, - /* 2340 */ 2080, 662, 2082, 2083, 657, 2097, 652, 1694, 1694, 1694, - /* 2350 */ 1694, 1694, 1694, 659, 1694, 1694, 1694, 1694, 2047, 1694, - /* 2360 */ 658, 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, - /* 2370 */ 1694, 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, - /* 2380 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2390 */ 2078, 2079, 1694, 1694, 2114, 1694, 1694, 318, 2080, 662, - /* 2400 */ 2082, 2083, 657, 1694, 652, 1694, 2078, 2079, 1694, 1694, - /* 2410 */ 2114, 1694, 1694, 331, 2080, 662, 2082, 2083, 657, 1694, - /* 2420 */ 652, 1694, 1694, 1694, 2097, 1694, 2079, 1694, 1694, 1694, - /* 2430 */ 1694, 1694, 659, 1694, 1694, 1694, 1694, 2047, 1694, 658, - /* 2440 */ 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, 1694, - /* 2450 */ 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, 2097, - /* 2460 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, 1694, 2078, - /* 2470 */ 1694, 1694, 2047, 2114, 658, 1694, 332, 2080, 662, 2082, - /* 2480 */ 2083, 657, 1694, 652, 1694, 2078, 1694, 1694, 1694, 2114, - /* 2490 */ 2079, 1694, 2091, 2080, 662, 2082, 2083, 657, 1694, 652, - /* 2500 */ 1694, 1694, 1694, 1694, 2078, 1694, 1694, 2079, 2114, 1694, - /* 2510 */ 1694, 2090, 2080, 662, 2082, 2083, 657, 1694, 652, 1694, - /* 2520 */ 1694, 1694, 1694, 2097, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2530 */ 1694, 659, 1694, 1694, 1694, 1694, 2047, 1694, 658, 1694, - /* 2540 */ 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, 1694, - /* 2550 */ 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, 1694, - /* 2560 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 2079, 2078, 1694, - /* 2570 */ 1694, 1694, 2114, 1694, 1694, 2089, 2080, 662, 2082, 2083, - /* 2580 */ 657, 1694, 652, 1694, 1694, 2078, 1694, 2079, 1694, 2114, - /* 2590 */ 1694, 1694, 346, 2080, 662, 2082, 2083, 657, 1694, 652, - /* 2600 */ 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, 1694, - /* 2610 */ 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, 1694, - /* 2620 */ 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, 1694, - /* 2630 */ 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, 1694, - /* 2640 */ 1694, 1694, 1694, 1694, 1694, 2078, 1694, 1694, 1694, 2114, - /* 2650 */ 1694, 1694, 347, 2080, 662, 2082, 2083, 657, 1694, 652, - /* 2660 */ 1694, 2079, 1694, 1694, 1694, 2078, 1694, 1694, 1694, 2114, - /* 2670 */ 1694, 1694, 343, 2080, 662, 2082, 2083, 657, 2079, 652, - /* 2680 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2690 */ 1694, 1694, 1694, 1694, 2097, 1694, 1694, 1694, 1694, 1694, - /* 2700 */ 1694, 1694, 659, 1694, 1694, 1694, 1694, 2047, 1694, 658, - /* 2710 */ 1694, 2097, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 659, - /* 2720 */ 1694, 1694, 1694, 1694, 2047, 1694, 658, 1694, 1694, 1694, - /* 2730 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 2078, - /* 2740 */ 2079, 1694, 1694, 2114, 1694, 1694, 348, 2080, 662, 2082, - /* 2750 */ 2083, 657, 1694, 652, 1694, 1694, 660, 1694, 1694, 1694, - /* 2760 */ 2114, 1694, 1694, 323, 2080, 662, 2082, 2083, 657, 1694, - /* 2770 */ 652, 1694, 1694, 2097, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2780 */ 1694, 659, 1694, 1694, 1694, 1694, 2047, 1694, 658, 1694, - /* 2790 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2800 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, - /* 2810 */ 1694, 1694, 1694, 1694, 1694, 1694, 1694, 1694, 2078, 1694, - /* 2820 */ 1694, 1694, 2114, 1694, 1694, 322, 2080, 662, 2082, 2083, - /* 2830 */ 657, 1694, 652, + /* 0 */ 2116, 209, 2010, 1881, 432, 502, 1883, 1765, 431, 2102, + /* 10 */ 670, 2010, 48, 46, 1646, 1722, 2098, 2008, 640, 2098, + /* 20 */ 393, 1522, 1495, 384, 41, 40, 2007, 640, 47, 45, + /* 30 */ 44, 43, 42, 1576, 1791, 1493, 2134, 180, 1520, 132, + /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 1931, + /* 50 */ 669, 591, 2094, 2100, 2275, 2094, 2100, 388, 398, 1571, + /* 60 */ 30, 1937, 1939, 663, 499, 19, 663, 500, 1758, 2281, + /* 70 */ 184, 168, 1501, 1734, 2276, 617, 1520, 628, 140, 1869, + /* 80 */ 507, 2115, 107, 500, 1758, 2151, 38, 296, 169, 2117, + /* 90 */ 673, 2119, 2120, 668, 655, 663, 2176, 758, 141, 9, + /* 100 */ 15, 735, 734, 733, 732, 403, 1884, 731, 730, 144, + /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, + /* 120 */ 713, 402, 401, 710, 709, 708, 707, 706, 592, 2241, + /* 130 */ 62, 2219, 1320, 1938, 1939, 123, 1578, 1579, 122, 121, + /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 695, 694, + /* 150 */ 693, 1315, 692, 1317, 1318, 691, 688, 2216, 1326, 685, + /* 160 */ 1328, 1329, 682, 679, 504, 652, 1551, 1561, 2280, 277, + /* 170 */ 501, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, + /* 180 */ 2213, 356, 138, 2217, 1409, 1410, 1496, 2279, 1494, 1942, + /* 190 */ 1720, 2276, 2278, 133, 287, 288, 1989, 41, 40, 286, + /* 200 */ 537, 47, 45, 44, 43, 42, 278, 52, 703, 155, + /* 210 */ 154, 700, 699, 698, 152, 1499, 1500, 1794, 1550, 1553, + /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 48, 46, 425, 1169, + /* 240 */ 192, 341, 62, 1518, 393, 51, 1495, 62, 87, 93, + /* 250 */ 469, 2116, 616, 483, 350, 2275, 482, 1576, 177, 1493, + /* 260 */ 406, 670, 427, 423, 405, 47, 45, 44, 43, 42, + /* 270 */ 615, 184, 452, 1888, 484, 2276, 617, 454, 1171, 1994, + /* 280 */ 1174, 1175, 187, 1571, 555, 554, 553, 2134, 1723, 19, + /* 290 */ 106, 545, 137, 549, 551, 550, 1501, 548, 1605, 2084, + /* 300 */ 103, 669, 547, 552, 366, 365, 1521, 591, 546, 123, + /* 310 */ 2275, 1639, 122, 121, 120, 119, 118, 117, 116, 115, + /* 320 */ 114, 758, 359, 101, 15, 2281, 184, 430, 2280, 429, + /* 330 */ 2276, 617, 2115, 1191, 442, 1190, 2151, 653, 1892, 110, + /* 340 */ 2117, 673, 2119, 2120, 668, 1522, 663, 1885, 226, 143, + /* 350 */ 438, 150, 2175, 2204, 1606, 133, 428, 387, 2200, 187, + /* 360 */ 1578, 1579, 542, 480, 652, 1192, 474, 473, 472, 471, + /* 370 */ 468, 467, 466, 465, 464, 460, 459, 458, 457, 340, + /* 380 */ 449, 448, 447, 177, 444, 443, 357, 653, 1892, 638, + /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 165, 187, + /* 400 */ 555, 554, 553, 358, 1993, 189, 1895, 545, 137, 549, + /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 2070, 547, 552, + /* 420 */ 366, 365, 1354, 1355, 546, 2279, 1708, 1264, 37, 391, + /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, + /* 440 */ 1500, 1974, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 450 */ 1560, 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, + /* 460 */ 12, 48, 46, 227, 1266, 1495, 413, 1521, 610, 393, + /* 470 */ 2116, 1495, 652, 196, 167, 1674, 516, 1520, 1493, 172, + /* 480 */ 631, 1833, 1576, 476, 1493, 533, 529, 525, 521, 224, + /* 490 */ 253, 41, 40, 277, 2134, 47, 45, 44, 43, 42, + /* 500 */ 612, 607, 600, 1523, 611, 66, 2134, 2103, 1571, 12, + /* 510 */ 1191, 10, 1190, 84, 19, 1501, 83, 2098, 2084, 639, + /* 520 */ 669, 1501, 603, 602, 1672, 1673, 1675, 1676, 1677, 88, + /* 530 */ 41, 40, 222, 91, 47, 45, 44, 43, 42, 1868, + /* 540 */ 758, 211, 1192, 200, 199, 502, 758, 1765, 609, 15, + /* 550 */ 1650, 2115, 51, 2094, 2100, 2151, 1520, 62, 110, 2117, + /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 2051, 166, 514, + /* 570 */ 181, 2003, 2204, 316, 41, 40, 387, 2200, 47, 45, + /* 580 */ 44, 43, 42, 1426, 1427, 1578, 1579, 314, 73, 186, + /* 590 */ 657, 72, 2176, 718, 560, 1854, 456, 2230, 1745, 221, + /* 600 */ 215, 62, 705, 705, 220, 455, 512, 14, 13, 570, + /* 610 */ 207, 495, 493, 490, 244, 1551, 1561, 396, 385, 1425, + /* 620 */ 1428, 1577, 1580, 240, 213, 162, 165, 653, 1892, 1496, + /* 630 */ 1177, 1494, 372, 1894, 1894, 1496, 1519, 1494, 616, 563, + /* 640 */ 1942, 2275, 653, 1892, 557, 57, 2084, 1519, 1944, 239, + /* 650 */ 62, 255, 44, 43, 42, 371, 615, 184, 1499, 1500, + /* 660 */ 436, 2276, 617, 1942, 1499, 1500, 1523, 1550, 1553, 1554, + /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, + /* 680 */ 1572, 1573, 1574, 1575, 2, 48, 46, 1581, 109, 70, + /* 690 */ 1944, 2116, 69, 393, 619, 1495, 1744, 381, 653, 1892, + /* 700 */ 1715, 631, 1552, 1221, 187, 1942, 1576, 1619, 1493, 187, + /* 710 */ 2219, 34, 729, 727, 628, 140, 437, 41, 40, 653, + /* 720 */ 1892, 47, 45, 44, 43, 42, 578, 2134, 81, 80, + /* 730 */ 435, 541, 1571, 191, 164, 540, 2215, 446, 639, 2084, + /* 740 */ 1222, 669, 41, 40, 2084, 1501, 47, 45, 44, 43, + /* 750 */ 42, 653, 1892, 187, 339, 142, 697, 421, 2175, 1935, + /* 760 */ 419, 415, 411, 408, 428, 12, 243, 653, 1892, 461, + /* 770 */ 758, 1501, 2115, 49, 653, 1892, 2151, 2116, 639, 110, + /* 780 */ 2117, 673, 2119, 2120, 668, 462, 663, 670, 637, 1767, + /* 790 */ 2003, 181, 515, 2204, 36, 396, 87, 387, 2200, 1714, + /* 800 */ 41, 40, 187, 165, 47, 45, 44, 43, 42, 1578, + /* 810 */ 1579, 1894, 360, 2134, 193, 275, 2212, 627, 2231, 134, + /* 820 */ 626, 1887, 2275, 628, 140, 2084, 696, 669, 648, 1944, + /* 830 */ 2003, 653, 1892, 1468, 1469, 1685, 386, 615, 184, 1551, + /* 840 */ 1561, 1989, 2276, 617, 1942, 1577, 1580, 41, 40, 1889, + /* 850 */ 717, 47, 45, 44, 43, 42, 1867, 590, 2115, 1496, + /* 860 */ 1275, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, + /* 870 */ 668, 1944, 663, 1274, 242, 1944, 1989, 2295, 241, 2204, + /* 880 */ 363, 245, 397, 387, 2200, 194, 1943, 634, 1499, 1500, + /* 890 */ 1942, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 48, + /* 910 */ 46, 2116, 1743, 1174, 1175, 653, 1892, 393, 1742, 1495, + /* 920 */ 198, 670, 1877, 2238, 183, 2212, 2213, 2067, 138, 2217, + /* 930 */ 1576, 2116, 1493, 587, 591, 485, 1523, 2275, 90, 345, + /* 940 */ 1662, 667, 370, 1643, 571, 1741, 364, 2134, 362, 361, + /* 950 */ 1740, 539, 2281, 184, 1879, 569, 1571, 2276, 617, 2084, + /* 960 */ 2084, 669, 653, 1892, 653, 1892, 2084, 2134, 567, 1501, + /* 970 */ 565, 701, 541, 1875, 1935, 702, 540, 55, 1935, 2084, + /* 980 */ 632, 669, 636, 2219, 1896, 703, 155, 154, 700, 699, + /* 990 */ 698, 152, 2115, 2084, 758, 2077, 2151, 49, 2084, 110, + /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 54, 2214, + /* 1010 */ 3, 2295, 2115, 2204, 1520, 670, 2151, 387, 2200, 332, + /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 653, + /* 1030 */ 1892, 41, 40, 1578, 1579, 47, 45, 44, 43, 42, + /* 1040 */ 399, 2134, 591, 1739, 249, 2275, 1586, 291, 165, 653, + /* 1050 */ 1892, 153, 1520, 2084, 591, 669, 1894, 2275, 653, 1892, + /* 1060 */ 2281, 184, 1279, 1551, 1561, 2276, 617, 650, 1607, 1577, + /* 1070 */ 1580, 232, 2281, 184, 230, 1278, 651, 2276, 617, 2224, + /* 1080 */ 1639, 653, 1892, 1496, 254, 1494, 2115, 1738, 653, 1892, + /* 1090 */ 2151, 2084, 439, 170, 2117, 673, 2119, 2120, 668, 297, + /* 1100 */ 663, 576, 1737, 628, 140, 440, 400, 1736, 56, 146, + /* 1110 */ 1733, 135, 1499, 1500, 1732, 1550, 1553, 1554, 1555, 1556, + /* 1120 */ 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, 1572, 1573, + /* 1130 */ 1574, 1575, 2, 48, 46, 2084, 420, 487, 1870, 1731, + /* 1140 */ 1730, 393, 35, 1495, 618, 2296, 1729, 2116, 591, 623, + /* 1150 */ 2084, 2275, 1612, 1728, 1576, 2084, 1493, 670, 2084, 2251, + /* 1160 */ 1552, 620, 2084, 1727, 1726, 1725, 2281, 184, 74, 543, + /* 1170 */ 310, 2276, 617, 1921, 2116, 234, 236, 1504, 233, 235, + /* 1180 */ 1571, 544, 148, 2134, 670, 1781, 598, 2084, 2084, 153, + /* 1190 */ 252, 1262, 1642, 1501, 2084, 2084, 238, 669, 1552, 237, + /* 1200 */ 573, 2084, 572, 1260, 185, 2212, 2213, 556, 138, 2217, + /* 1210 */ 2134, 2084, 2084, 2084, 153, 50, 50, 82, 758, 1774, + /* 1220 */ 259, 15, 2084, 1772, 669, 1717, 1718, 664, 2115, 1834, + /* 1230 */ 14, 13, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, + /* 1240 */ 668, 558, 663, 1503, 670, 561, 1463, 2295, 50, 2204, + /* 1250 */ 2105, 284, 660, 387, 2200, 2115, 71, 1578, 1579, 2151, + /* 1260 */ 151, 153, 110, 2117, 673, 2119, 2120, 668, 2244, 663, + /* 1270 */ 2134, 1466, 1671, 1670, 2295, 64, 2204, 261, 50, 1735, + /* 1280 */ 387, 2200, 2084, 225, 669, 272, 604, 1551, 1561, 2135, + /* 1290 */ 635, 266, 404, 1577, 1580, 1998, 703, 155, 154, 700, + /* 1300 */ 699, 698, 152, 2107, 1759, 1423, 1932, 1496, 289, 1494, + /* 1310 */ 2234, 50, 711, 645, 677, 2115, 151, 293, 1305, 2151, + /* 1320 */ 1768, 153, 169, 2117, 673, 2119, 2120, 668, 1764, 663, + /* 1330 */ 1507, 629, 1613, 136, 1240, 1562, 1499, 1500, 151, 1550, + /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 274, 271, 390, + /* 1360 */ 389, 712, 1, 2242, 5, 407, 412, 2116, 309, 1509, + /* 1370 */ 753, 1332, 354, 1336, 1446, 304, 197, 670, 1343, 2269, + /* 1380 */ 1576, 624, 1502, 1238, 441, 1523, 1999, 445, 450, 478, + /* 1390 */ 1341, 621, 1518, 463, 1991, 156, 1506, 2116, 470, 477, + /* 1400 */ 479, 488, 1597, 2134, 489, 486, 1571, 670, 201, 2223, + /* 1410 */ 202, 491, 492, 1524, 497, 2084, 204, 669, 494, 1501, + /* 1420 */ 496, 4, 498, 505, 506, 1526, 508, 212, 1521, 509, + /* 1430 */ 214, 1525, 1527, 2134, 510, 513, 511, 217, 1194, 517, + /* 1440 */ 112, 534, 535, 219, 659, 2084, 85, 669, 2115, 86, + /* 1450 */ 223, 536, 2151, 538, 344, 110, 2117, 673, 2119, 2120, + /* 1460 */ 668, 575, 663, 2060, 89, 2116, 577, 2295, 305, 2204, + /* 1470 */ 1882, 229, 1878, 387, 2200, 670, 231, 158, 2115, 246, + /* 1480 */ 159, 1880, 2151, 1876, 160, 110, 2117, 673, 2119, 2120, + /* 1490 */ 668, 161, 663, 581, 2057, 580, 250, 2295, 1453, 2204, + /* 1500 */ 588, 2134, 605, 387, 2200, 2056, 585, 595, 643, 601, + /* 1510 */ 2250, 2235, 582, 2084, 2249, 669, 248, 149, 2245, 8, + /* 1520 */ 376, 257, 608, 614, 586, 260, 596, 594, 2226, 593, + /* 1530 */ 267, 265, 377, 1510, 625, 1505, 2298, 173, 622, 1639, + /* 1540 */ 268, 139, 1522, 633, 380, 279, 2115, 96, 1528, 2004, + /* 1550 */ 2151, 270, 646, 110, 2117, 673, 2119, 2120, 668, 306, + /* 1560 */ 663, 641, 1513, 1515, 2220, 2179, 647, 2204, 98, 642, + /* 1570 */ 307, 387, 2200, 2116, 2018, 661, 1569, 1570, 1572, 1573, + /* 1580 */ 1574, 1575, 1893, 670, 100, 61, 269, 2017, 2185, 102, + /* 1590 */ 2016, 383, 308, 2116, 675, 1936, 1855, 300, 311, 2274, + /* 1600 */ 754, 757, 273, 670, 755, 53, 335, 320, 346, 2134, + /* 1610 */ 347, 313, 315, 2076, 334, 324, 2075, 2074, 78, 2071, + /* 1620 */ 409, 2084, 410, 669, 1486, 1487, 190, 414, 2069, 2134, + /* 1630 */ 416, 417, 418, 2068, 355, 2066, 422, 2065, 2064, 426, + /* 1640 */ 424, 2084, 79, 669, 1449, 1448, 2030, 2029, 2028, 433, + /* 1650 */ 434, 2027, 2026, 1400, 2115, 1982, 1981, 1979, 2151, 1978, + /* 1660 */ 1977, 110, 2117, 673, 2119, 2120, 668, 145, 663, 1980, + /* 1670 */ 1976, 1975, 1973, 2177, 2115, 2204, 195, 2116, 2151, 387, + /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 1972, + /* 1690 */ 1971, 1970, 453, 656, 1984, 2204, 451, 1969, 1968, 387, + /* 1700 */ 2200, 2116, 1967, 1966, 1965, 1964, 1963, 1962, 1961, 1960, + /* 1710 */ 1959, 670, 1958, 2134, 1957, 1956, 1955, 147, 1954, 1953, + /* 1720 */ 1952, 1983, 1951, 1950, 1949, 2084, 1948, 669, 1947, 1946, + /* 1730 */ 1945, 481, 342, 1402, 343, 1276, 1797, 2134, 203, 1280, + /* 1740 */ 1796, 205, 1795, 1793, 206, 1754, 2104, 179, 1176, 2084, + /* 1750 */ 218, 669, 1753, 2047, 2037, 2025, 2024, 2002, 2115, 1871, + /* 1760 */ 1272, 1214, 2151, 1792, 1790, 111, 2117, 673, 2119, 2120, + /* 1770 */ 668, 76, 663, 208, 503, 210, 77, 178, 2116, 2204, + /* 1780 */ 216, 518, 2115, 2203, 2200, 1788, 2151, 519, 670, 111, + /* 1790 */ 2117, 673, 2119, 2120, 668, 522, 663, 520, 523, 1786, + /* 1800 */ 524, 2116, 526, 2204, 528, 1784, 530, 658, 2200, 1771, + /* 1810 */ 1770, 670, 532, 1750, 2134, 527, 1873, 531, 1348, 1347, + /* 1820 */ 1872, 1263, 1261, 1259, 2116, 1258, 2084, 1257, 669, 1256, + /* 1830 */ 1255, 1252, 1782, 726, 670, 728, 1251, 2134, 1249, 1250, + /* 1840 */ 367, 1775, 1773, 368, 559, 369, 562, 2116, 63, 2084, + /* 1850 */ 1749, 669, 1748, 1747, 228, 568, 113, 670, 2046, 671, + /* 1860 */ 2134, 564, 566, 2151, 1473, 374, 111, 2117, 673, 2119, + /* 1870 */ 2120, 668, 2084, 663, 669, 1475, 1477, 29, 1472, 67, + /* 1880 */ 2204, 1459, 2115, 2134, 349, 2200, 2151, 1457, 375, 111, + /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1455, 2036, + /* 1900 */ 583, 2023, 2021, 2204, 20, 2115, 2280, 31, 2201, 2151, + /* 1910 */ 2116, 599, 333, 2117, 673, 2119, 2120, 668, 251, 663, + /* 1920 */ 670, 17, 6, 1687, 256, 7, 589, 264, 2115, 597, + /* 1930 */ 58, 258, 2151, 163, 21, 333, 2117, 673, 2119, 2120, + /* 1940 */ 668, 22, 663, 2105, 584, 1669, 2134, 171, 373, 262, + /* 1950 */ 263, 32, 33, 65, 24, 1661, 92, 23, 2084, 1702, + /* 1960 */ 669, 1701, 1707, 1708, 378, 1706, 1705, 379, 276, 1636, + /* 1970 */ 1635, 60, 174, 2022, 2020, 2116, 2019, 2001, 95, 94, + /* 1980 */ 282, 25, 2000, 283, 285, 670, 1667, 290, 68, 99, + /* 1990 */ 644, 2115, 97, 295, 292, 2151, 2116, 13, 326, 2117, + /* 2000 */ 673, 2119, 2120, 668, 103, 663, 670, 26, 1511, 2116, + /* 2010 */ 1588, 2134, 1587, 1598, 175, 11, 2154, 188, 1566, 667, + /* 2020 */ 662, 1543, 39, 2084, 59, 669, 674, 676, 18, 1564, + /* 2030 */ 1338, 672, 2134, 395, 298, 1563, 16, 382, 27, 680, + /* 2040 */ 28, 613, 1535, 1333, 2084, 2134, 669, 678, 683, 1330, + /* 2050 */ 1327, 681, 684, 686, 689, 1321, 2115, 2084, 687, 669, + /* 2060 */ 2151, 1319, 690, 170, 2117, 673, 2119, 2120, 668, 104, + /* 2070 */ 663, 105, 1342, 1246, 704, 75, 1212, 2115, 1325, 1324, + /* 2080 */ 1245, 2151, 1244, 2116, 333, 2117, 673, 2119, 2120, 668, + /* 2090 */ 2115, 663, 1243, 670, 2151, 1242, 1241, 332, 2117, 673, + /* 2100 */ 2119, 2120, 668, 2116, 663, 1323, 2170, 1322, 1239, 1237, + /* 2110 */ 1236, 1235, 1270, 670, 299, 2297, 716, 1233, 1232, 2134, + /* 2120 */ 1231, 1230, 1229, 1228, 392, 1227, 1267, 1265, 1224, 1223, + /* 2130 */ 1220, 2084, 1219, 669, 1218, 1217, 1789, 736, 737, 2134, + /* 2140 */ 738, 1787, 740, 742, 394, 741, 1785, 745, 744, 746, + /* 2150 */ 1783, 2084, 748, 669, 749, 750, 1769, 752, 1166, 1746, + /* 2160 */ 302, 756, 1721, 759, 2115, 1497, 312, 760, 2151, 1721, + /* 2170 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2180 */ 1721, 1721, 1721, 1721, 2115, 1721, 579, 1721, 2151, 1721, + /* 2190 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2200 */ 2116, 1721, 1721, 1721, 761, 1721, 1721, 1721, 1721, 1721, + /* 2210 */ 670, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 303, 1721, + /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2230 */ 1721, 1721, 1721, 1721, 176, 1721, 2134, 1721, 1721, 1721, + /* 2240 */ 751, 747, 743, 739, 301, 1721, 2116, 1721, 2084, 1721, + /* 2250 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2260 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, + /* 2270 */ 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, + /* 2280 */ 1721, 574, 2134, 1721, 108, 2151, 670, 294, 328, 2117, + /* 2290 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, + /* 2300 */ 1721, 2151, 1721, 1721, 317, 2117, 673, 2119, 2120, 668, + /* 2310 */ 1721, 663, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 649, + /* 2320 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2115, 1721, 1721, + /* 2330 */ 1721, 2151, 1721, 1721, 318, 2117, 673, 2119, 2120, 668, + /* 2340 */ 2116, 663, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2350 */ 670, 1721, 1721, 2116, 281, 1721, 1721, 2115, 1721, 280, + /* 2360 */ 1721, 2151, 1721, 670, 319, 2117, 673, 2119, 2120, 668, + /* 2370 */ 1721, 663, 1721, 1721, 1721, 1721, 2134, 1721, 1721, 247, + /* 2380 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2390 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2400 */ 1721, 2084, 1721, 669, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2420 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 670, 325, 2117, + /* 2430 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2440 */ 2134, 329, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2450 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2460 */ 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, + /* 2470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2480 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 670, 2151, + /* 2490 */ 1721, 1721, 321, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2500 */ 1721, 1721, 2151, 1721, 1721, 330, 2117, 673, 2119, 2120, + /* 2510 */ 668, 1721, 663, 1721, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, + /* 2530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, + /* 2540 */ 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2115, + /* 2560 */ 1721, 1721, 1721, 2151, 1721, 2134, 322, 2117, 673, 2119, + /* 2570 */ 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, 669, + /* 2580 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2590 */ 2116, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2610 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 331, 2117, 673, + /* 2620 */ 2119, 2120, 668, 1721, 663, 2115, 2134, 1721, 1721, 2151, + /* 2630 */ 1721, 1721, 323, 2117, 673, 2119, 2120, 668, 2084, 663, + /* 2640 */ 669, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 670, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, + /* 2670 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2134, 336, 2117, + /* 2680 */ 673, 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, + /* 2690 */ 1721, 669, 1721, 1721, 2134, 1721, 1721, 670, 1721, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, + /* 2710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2720 */ 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 337, + /* 2730 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1721, 2115, + /* 2740 */ 1721, 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, + /* 2750 */ 2120, 668, 1721, 663, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2760 */ 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 2115, 1721, + /* 2770 */ 1721, 1721, 2151, 1721, 2116, 2127, 2117, 673, 2119, 2120, + /* 2780 */ 668, 1721, 663, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2790 */ 1721, 2134, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, + /* 2800 */ 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, + /* 2810 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 2084, 2115, 669, 1721, 1721, + /* 2840 */ 2151, 1721, 1721, 2126, 2117, 673, 2119, 2120, 668, 1721, + /* 2850 */ 663, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 2860 */ 2116, 1721, 351, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2870 */ 670, 1721, 2151, 2116, 1721, 352, 2117, 673, 2119, 2120, + /* 2880 */ 668, 1721, 663, 670, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 670, 1721, + /* 2900 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2910 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2920 */ 1721, 2084, 1721, 669, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2930 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, + /* 2940 */ 1721, 2115, 1721, 1721, 670, 2151, 1721, 1721, 348, 2117, + /* 2950 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2960 */ 1721, 353, 2117, 673, 2119, 2120, 668, 1721, 663, 671, + /* 2970 */ 2134, 1721, 1721, 2151, 1721, 1721, 328, 2117, 673, 2119, + /* 2980 */ 2120, 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, + /* 2990 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3000 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3010 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 3020 */ 1721, 1721, 327, 2117, 673, 2119, 2120, 668, 1721, 663, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 367, 443, 380, 347, 332, 447, 336, 357, 338, 339, - /* 10 */ 354, 378, 12, 13, 14, 365, 380, 395, 396, 367, - /* 20 */ 20, 463, 22, 340, 374, 467, 468, 332, 392, 20, - /* 30 */ 378, 395, 396, 33, 331, 35, 333, 365, 367, 12, - /* 40 */ 13, 14, 15, 16, 20, 373, 413, 414, 415, 378, - /* 50 */ 378, 336, 380, 338, 339, 357, 373, 424, 58, 365, - /* 60 */ 365, 20, 365, 365, 64, 413, 414, 415, 373, 372, - /* 70 */ 372, 71, 374, 378, 364, 380, 424, 14, 381, 381, - /* 80 */ 386, 387, 410, 20, 413, 414, 414, 377, 20, 417, - /* 90 */ 418, 419, 420, 421, 422, 424, 424, 97, 432, 433, - /* 100 */ 100, 429, 340, 431, 20, 410, 340, 435, 436, 414, - /* 110 */ 12, 13, 417, 418, 419, 420, 421, 422, 20, 424, - /* 120 */ 22, 449, 439, 440, 441, 0, 443, 444, 367, 457, - /* 130 */ 447, 33, 20, 35, 22, 373, 136, 137, 337, 378, - /* 140 */ 443, 340, 341, 14, 447, 462, 463, 35, 382, 20, - /* 150 */ 467, 468, 0, 458, 459, 4, 58, 0, 427, 462, - /* 160 */ 463, 430, 64, 51, 467, 468, 166, 167, 100, 71, - /* 170 */ 0, 63, 172, 173, 413, 414, 24, 25, 26, 27, - /* 180 */ 28, 29, 30, 31, 32, 424, 186, 20, 188, 365, - /* 190 */ 100, 66, 67, 68, 43, 97, 45, 46, 100, 74, - /* 200 */ 75, 439, 440, 441, 79, 443, 444, 166, 167, 84, - /* 210 */ 85, 387, 80, 213, 214, 90, 216, 217, 218, 219, + /* 0 */ 333, 338, 383, 370, 400, 342, 371, 344, 404, 371, + /* 10 */ 343, 383, 12, 13, 14, 0, 381, 398, 399, 381, + /* 20 */ 20, 20, 22, 395, 8, 9, 398, 399, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 369, 368, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 380, + /* 50 */ 383, 447, 417, 418, 450, 417, 418, 419, 379, 59, + /* 60 */ 44, 382, 383, 428, 337, 65, 428, 340, 341, 465, + /* 70 */ 466, 332, 72, 334, 470, 471, 20, 342, 343, 0, + /* 80 */ 337, 414, 348, 340, 341, 418, 436, 437, 421, 422, + /* 90 */ 423, 424, 425, 426, 432, 428, 434, 97, 364, 39, + /* 100 */ 100, 67, 68, 69, 70, 71, 372, 73, 74, 75, + /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, + /* 130 */ 100, 420, 97, 382, 383, 21, 136, 137, 24, 25, + /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, + /* 150 */ 115, 116, 117, 118, 119, 120, 121, 446, 123, 124, + /* 160 */ 125, 126, 127, 128, 14, 20, 166, 167, 447, 168, + /* 170 */ 20, 450, 172, 173, 369, 342, 343, 442, 443, 444, + /* 180 */ 445, 376, 447, 448, 166, 167, 186, 466, 188, 384, + /* 190 */ 330, 470, 471, 360, 130, 131, 343, 8, 9, 135, + /* 200 */ 367, 12, 13, 14, 15, 16, 59, 100, 129, 130, + /* 210 */ 131, 132, 133, 134, 135, 215, 216, 0, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 233, 136, 137, 66, 67, 68, 69, - /* 240 */ 70, 20, 72, 73, 74, 75, 76, 77, 78, 79, - /* 250 */ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, - /* 260 */ 90, 91, 92, 4, 166, 167, 171, 100, 63, 112, - /* 270 */ 172, 173, 21, 141, 142, 24, 25, 26, 27, 28, - /* 280 */ 29, 30, 31, 32, 186, 112, 188, 130, 131, 132, - /* 290 */ 133, 134, 135, 8, 9, 166, 164, 12, 13, 14, - /* 300 */ 15, 16, 20, 130, 131, 132, 133, 134, 135, 329, - /* 310 */ 416, 213, 214, 58, 216, 217, 218, 219, 220, 221, - /* 320 */ 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, - /* 330 */ 232, 233, 234, 12, 13, 376, 442, 365, 379, 380, - /* 340 */ 250, 20, 97, 22, 337, 373, 100, 340, 341, 64, - /* 350 */ 255, 256, 257, 0, 33, 100, 35, 102, 113, 114, - /* 360 */ 115, 116, 117, 118, 119, 120, 121, 122, 346, 124, - /* 370 */ 125, 126, 127, 128, 129, 0, 2, 397, 332, 58, - /* 380 */ 111, 401, 8, 9, 4, 64, 12, 13, 14, 15, - /* 390 */ 16, 419, 71, 371, 109, 340, 21, 20, 234, 24, - /* 400 */ 25, 26, 27, 28, 29, 30, 31, 32, 8, 9, - /* 410 */ 340, 365, 12, 13, 14, 15, 16, 250, 97, 373, - /* 420 */ 20, 100, 340, 443, 378, 71, 380, 447, 234, 365, - /* 430 */ 236, 12, 13, 14, 340, 111, 372, 361, 356, 20, - /* 440 */ 22, 22, 462, 463, 389, 381, 391, 467, 468, 332, - /* 450 */ 165, 213, 33, 35, 35, 373, 410, 136, 137, 389, - /* 460 */ 414, 391, 3, 417, 418, 419, 420, 421, 422, 423, - /* 470 */ 424, 425, 426, 66, 67, 68, 35, 58, 340, 20, - /* 480 */ 100, 74, 75, 389, 408, 391, 79, 166, 167, 3, - /* 490 */ 71, 84, 85, 172, 173, 378, 250, 90, 260, 261, - /* 500 */ 262, 263, 264, 265, 266, 250, 71, 186, 155, 188, - /* 510 */ 251, 373, 71, 136, 137, 97, 97, 164, 44, 100, - /* 520 */ 351, 352, 237, 238, 239, 240, 241, 242, 243, 244, - /* 530 */ 245, 246, 247, 20, 213, 214, 402, 216, 217, 218, - /* 540 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 550 */ 229, 230, 231, 232, 233, 136, 137, 8, 9, 366, - /* 560 */ 101, 12, 13, 14, 15, 16, 8, 9, 168, 332, - /* 570 */ 12, 13, 14, 15, 16, 101, 438, 439, 440, 441, - /* 580 */ 332, 443, 444, 340, 443, 166, 167, 333, 447, 8, - /* 590 */ 9, 172, 173, 12, 13, 14, 15, 16, 443, 356, - /* 600 */ 366, 340, 447, 462, 463, 186, 363, 188, 467, 468, - /* 610 */ 332, 20, 428, 365, 430, 378, 373, 356, 463, 130, - /* 620 */ 47, 373, 467, 468, 363, 44, 378, 168, 380, 428, - /* 630 */ 250, 430, 213, 214, 373, 216, 217, 218, 219, 220, - /* 640 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 650 */ 231, 232, 233, 12, 13, 346, 378, 332, 410, 101, - /* 660 */ 397, 20, 414, 22, 401, 417, 418, 419, 420, 421, - /* 670 */ 422, 362, 424, 100, 33, 427, 35, 429, 430, 431, - /* 680 */ 371, 168, 165, 435, 436, 196, 197, 22, 8, 9, - /* 690 */ 365, 100, 12, 13, 14, 15, 16, 365, 373, 58, - /* 700 */ 35, 340, 416, 378, 372, 380, 443, 340, 2, 353, - /* 710 */ 447, 355, 71, 381, 8, 9, 379, 380, 12, 13, - /* 720 */ 14, 15, 16, 356, 366, 462, 463, 178, 442, 131, - /* 730 */ 467, 468, 340, 135, 373, 410, 71, 340, 97, 414, - /* 740 */ 373, 100, 417, 418, 419, 420, 421, 422, 356, 424, - /* 750 */ 37, 12, 13, 356, 237, 20, 270, 22, 357, 20, - /* 760 */ 365, 22, 97, 357, 247, 373, 365, 372, 130, 131, - /* 770 */ 373, 365, 33, 135, 35, 374, 381, 136, 137, 0, - /* 780 */ 374, 101, 332, 8, 9, 0, 51, 12, 13, 14, - /* 790 */ 15, 16, 194, 195, 469, 470, 198, 58, 200, 332, - /* 800 */ 439, 440, 441, 332, 443, 444, 344, 166, 167, 416, - /* 810 */ 71, 351, 352, 172, 173, 340, 103, 112, 105, 106, - /* 820 */ 181, 108, 360, 44, 275, 112, 365, 186, 378, 188, - /* 830 */ 368, 356, 365, 372, 39, 442, 97, 21, 133, 100, - /* 840 */ 373, 250, 381, 204, 205, 378, 133, 380, 373, 378, - /* 850 */ 34, 186, 36, 188, 213, 214, 332, 216, 217, 218, - /* 860 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 870 */ 229, 230, 231, 232, 233, 136, 137, 410, 213, 214, - /* 880 */ 397, 414, 1, 2, 417, 418, 419, 420, 421, 422, - /* 890 */ 344, 424, 20, 14, 14, 15, 16, 112, 431, 20, - /* 900 */ 100, 332, 378, 436, 365, 166, 167, 340, 340, 20, - /* 910 */ 110, 172, 173, 374, 368, 130, 131, 132, 133, 134, - /* 920 */ 135, 365, 20, 356, 356, 186, 443, 188, 8, 9, - /* 930 */ 447, 354, 12, 13, 14, 15, 16, 381, 373, 375, - /* 940 */ 373, 373, 378, 168, 332, 462, 463, 378, 0, 384, - /* 950 */ 467, 468, 213, 214, 111, 216, 217, 218, 219, 220, - /* 960 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 970 */ 231, 232, 233, 100, 18, 18, 20, 397, 12, 13, - /* 980 */ 23, 373, 101, 27, 22, 4, 30, 332, 22, 33, - /* 990 */ 378, 373, 384, 358, 37, 38, 361, 35, 41, 33, - /* 1000 */ 19, 35, 384, 160, 48, 58, 50, 332, 22, 53, - /* 1010 */ 332, 63, 375, 44, 33, 378, 59, 60, 61, 62, - /* 1020 */ 365, 35, 332, 443, 58, 136, 137, 447, 373, 48, - /* 1030 */ 45, 46, 332, 378, 53, 380, 366, 71, 340, 58, - /* 1040 */ 168, 332, 462, 463, 249, 166, 99, 467, 468, 102, - /* 1050 */ 58, 332, 332, 378, 356, 99, 378, 100, 185, 97, - /* 1060 */ 187, 172, 173, 97, 20, 410, 44, 111, 378, 414, - /* 1070 */ 168, 373, 417, 418, 419, 420, 421, 422, 378, 424, - /* 1080 */ 99, 332, 367, 102, 429, 212, 431, 378, 332, 375, - /* 1090 */ 435, 436, 378, 378, 102, 138, 140, 378, 378, 143, - /* 1100 */ 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - /* 1110 */ 154, 155, 156, 157, 158, 159, 340, 161, 162, 163, - /* 1120 */ 366, 104, 340, 250, 107, 332, 340, 378, 413, 414, - /* 1130 */ 415, 340, 356, 0, 378, 178, 179, 180, 356, 424, - /* 1140 */ 183, 332, 356, 248, 249, 340, 42, 356, 44, 373, - /* 1150 */ 35, 42, 186, 44, 188, 373, 64, 340, 366, 373, - /* 1160 */ 203, 356, 0, 206, 373, 208, 209, 210, 211, 212, - /* 1170 */ 340, 378, 332, 356, 365, 199, 366, 201, 373, 213, - /* 1180 */ 214, 48, 373, 471, 22, 44, 356, 378, 340, 380, - /* 1190 */ 373, 35, 226, 227, 228, 229, 230, 231, 232, 340, - /* 1200 */ 332, 340, 0, 373, 356, 365, 104, 250, 44, 107, - /* 1210 */ 166, 104, 388, 373, 107, 356, 44, 356, 378, 410, - /* 1220 */ 380, 373, 332, 414, 22, 0, 417, 418, 419, 420, - /* 1230 */ 421, 422, 373, 424, 373, 44, 44, 44, 429, 332, - /* 1240 */ 431, 272, 101, 96, 435, 436, 378, 22, 104, 44, - /* 1250 */ 410, 107, 1, 2, 414, 365, 44, 417, 418, 419, - /* 1260 */ 420, 421, 422, 373, 424, 101, 457, 0, 378, 429, - /* 1270 */ 380, 431, 365, 101, 460, 435, 436, 44, 136, 137, - /* 1280 */ 373, 342, 44, 454, 44, 378, 446, 380, 365, 44, - /* 1290 */ 339, 44, 101, 101, 101, 342, 274, 44, 44, 332, - /* 1300 */ 410, 44, 44, 188, 414, 213, 101, 417, 418, 419, - /* 1310 */ 420, 421, 422, 101, 424, 168, 49, 410, 388, 332, - /* 1320 */ 377, 414, 44, 44, 417, 418, 419, 420, 421, 422, - /* 1330 */ 388, 424, 365, 35, 101, 13, 429, 19, 431, 101, - /* 1340 */ 373, 101, 435, 436, 188, 378, 101, 380, 101, 459, - /* 1350 */ 445, 33, 365, 446, 101, 101, 13, 35, 101, 101, - /* 1360 */ 373, 464, 437, 448, 252, 378, 48, 380, 48, 71, - /* 1370 */ 412, 411, 54, 55, 56, 57, 58, 410, 35, 101, - /* 1380 */ 101, 414, 184, 399, 417, 418, 419, 420, 421, 422, - /* 1390 */ 42, 424, 385, 20, 388, 385, 429, 410, 431, 165, - /* 1400 */ 20, 414, 435, 436, 417, 418, 419, 420, 421, 422, - /* 1410 */ 383, 424, 340, 446, 340, 385, 429, 99, 431, 383, - /* 1420 */ 102, 383, 435, 436, 98, 350, 95, 349, 332, 340, - /* 1430 */ 94, 348, 397, 446, 340, 33, 20, 340, 340, 334, - /* 1440 */ 334, 20, 405, 20, 380, 346, 346, 20, 332, 341, - /* 1450 */ 48, 20, 134, 398, 341, 52, 54, 55, 56, 57, - /* 1460 */ 58, 365, 340, 346, 346, 346, 346, 346, 343, 373, - /* 1470 */ 343, 334, 340, 334, 378, 365, 380, 202, 443, 365, - /* 1480 */ 409, 365, 447, 100, 407, 405, 192, 169, 365, 373, - /* 1490 */ 344, 365, 174, 365, 378, 365, 380, 462, 463, 365, - /* 1500 */ 365, 99, 467, 468, 102, 365, 410, 378, 190, 378, - /* 1510 */ 414, 193, 191, 417, 418, 419, 420, 421, 422, 365, - /* 1520 */ 424, 378, 332, 365, 344, 429, 410, 431, 404, 380, - /* 1530 */ 414, 435, 436, 417, 418, 419, 420, 421, 422, 403, - /* 1540 */ 424, 340, 446, 388, 378, 429, 332, 431, 388, 259, - /* 1550 */ 258, 435, 436, 378, 453, 365, 393, 378, 393, 378, - /* 1560 */ 267, 177, 253, 373, 453, 269, 271, 268, 378, 412, - /* 1570 */ 380, 169, 170, 276, 273, 472, 174, 249, 176, 365, - /* 1580 */ 373, 20, 340, 416, 341, 344, 20, 373, 344, 393, - /* 1590 */ 391, 378, 378, 455, 380, 193, 332, 378, 378, 451, - /* 1600 */ 410, 456, 450, 453, 414, 397, 378, 417, 418, 419, - /* 1610 */ 420, 421, 422, 332, 424, 378, 393, 466, 465, 429, - /* 1620 */ 452, 431, 170, 361, 410, 435, 436, 378, 414, 365, - /* 1630 */ 344, 417, 418, 419, 420, 421, 422, 373, 424, 373, - /* 1640 */ 390, 100, 378, 434, 380, 431, 365, 344, 100, 435, - /* 1650 */ 436, 443, 369, 340, 373, 447, 344, 355, 378, 378, - /* 1660 */ 36, 380, 335, 334, 394, 400, 406, 394, 345, 359, - /* 1670 */ 462, 463, 330, 332, 410, 467, 468, 0, 414, 0, - /* 1680 */ 0, 417, 418, 419, 420, 421, 422, 42, 424, 359, - /* 1690 */ 0, 410, 359, 332, 35, 414, 207, 35, 417, 418, - /* 1700 */ 419, 420, 421, 422, 35, 424, 365, 35, 207, 0, - /* 1710 */ 35, 35, 431, 207, 373, 0, 435, 436, 207, 378, - /* 1720 */ 0, 380, 35, 0, 22, 461, 365, 0, 35, 194, - /* 1730 */ 188, 186, 0, 0, 373, 0, 182, 181, 0, 378, - /* 1740 */ 0, 380, 47, 0, 0, 0, 42, 0, 0, 0, - /* 1750 */ 0, 410, 0, 0, 0, 414, 332, 0, 417, 418, - /* 1760 */ 419, 420, 421, 422, 155, 424, 35, 0, 155, 0, - /* 1770 */ 0, 410, 431, 0, 0, 414, 435, 436, 417, 418, - /* 1780 */ 419, 420, 421, 422, 42, 424, 0, 0, 0, 365, - /* 1790 */ 0, 0, 0, 0, 370, 0, 0, 373, 0, 0, - /* 1800 */ 0, 0, 378, 0, 380, 0, 0, 0, 0, 0, - /* 1810 */ 0, 22, 0, 139, 332, 0, 0, 96, 96, 22, - /* 1820 */ 0, 58, 0, 58, 22, 0, 58, 0, 0, 35, - /* 1830 */ 47, 470, 14, 14, 410, 0, 0, 0, 414, 332, - /* 1840 */ 0, 417, 418, 419, 420, 421, 422, 365, 424, 0, - /* 1850 */ 177, 47, 39, 42, 0, 373, 0, 0, 44, 0, - /* 1860 */ 378, 40, 380, 39, 47, 39, 35, 0, 39, 35, - /* 1870 */ 0, 39, 365, 35, 0, 0, 0, 370, 48, 35, - /* 1880 */ 373, 48, 39, 0, 0, 378, 39, 380, 65, 48, - /* 1890 */ 109, 22, 410, 48, 0, 35, 414, 35, 35, 417, - /* 1900 */ 418, 419, 420, 421, 422, 0, 424, 35, 426, 44, - /* 1910 */ 44, 22, 332, 35, 35, 35, 35, 410, 22, 0, - /* 1920 */ 22, 414, 22, 50, 417, 418, 419, 420, 421, 422, - /* 1930 */ 0, 424, 35, 332, 0, 0, 35, 35, 0, 22, - /* 1940 */ 20, 35, 35, 35, 0, 365, 107, 101, 100, 35, - /* 1950 */ 370, 100, 0, 373, 22, 189, 0, 0, 378, 22, - /* 1960 */ 380, 3, 332, 47, 254, 44, 365, 44, 44, 44, - /* 1970 */ 254, 44, 3, 47, 373, 44, 35, 100, 254, 378, - /* 1980 */ 35, 380, 100, 332, 96, 98, 101, 96, 95, 35, - /* 1990 */ 410, 101, 44, 101, 414, 365, 101, 417, 418, 419, - /* 2000 */ 420, 421, 422, 373, 424, 101, 35, 35, 378, 100, - /* 2010 */ 380, 410, 100, 100, 35, 414, 365, 47, 417, 418, - /* 2020 */ 419, 420, 421, 422, 373, 424, 168, 175, 100, 378, - /* 2030 */ 101, 380, 168, 332, 101, 47, 0, 101, 170, 168, - /* 2040 */ 410, 0, 0, 0, 414, 39, 100, 417, 418, 419, - /* 2050 */ 420, 421, 422, 100, 424, 47, 101, 171, 332, 39, - /* 2060 */ 101, 410, 100, 100, 100, 414, 365, 0, 417, 418, - /* 2070 */ 419, 420, 421, 422, 373, 424, 100, 110, 47, 378, - /* 2080 */ 169, 380, 44, 98, 98, 2, 22, 100, 235, 111, - /* 2090 */ 101, 365, 100, 332, 101, 100, 47, 100, 248, 373, - /* 2100 */ 101, 100, 47, 101, 378, 100, 380, 22, 101, 35, - /* 2110 */ 100, 410, 213, 35, 101, 414, 332, 35, 417, 418, - /* 2120 */ 419, 420, 421, 422, 100, 424, 365, 101, 35, 100, - /* 2130 */ 215, 101, 35, 100, 373, 101, 410, 35, 123, 378, - /* 2140 */ 414, 380, 123, 417, 418, 419, 420, 421, 422, 365, - /* 2150 */ 424, 100, 22, 100, 44, 123, 112, 373, 123, 35, - /* 2160 */ 100, 100, 378, 22, 380, 65, 35, 64, 35, 35, - /* 2170 */ 35, 410, 35, 35, 332, 414, 71, 35, 417, 418, - /* 2180 */ 419, 420, 421, 422, 35, 424, 35, 35, 93, 44, - /* 2190 */ 35, 35, 35, 332, 410, 22, 35, 35, 414, 35, - /* 2200 */ 71, 417, 418, 419, 420, 421, 422, 365, 424, 35, - /* 2210 */ 22, 35, 35, 35, 35, 373, 35, 0, 35, 48, - /* 2220 */ 378, 39, 380, 0, 35, 39, 365, 0, 48, 35, - /* 2230 */ 39, 0, 48, 35, 373, 48, 39, 0, 35, 378, - /* 2240 */ 35, 380, 0, 22, 22, 21, 20, 22, 473, 21, - /* 2250 */ 473, 473, 410, 473, 332, 473, 414, 473, 473, 417, - /* 2260 */ 418, 419, 420, 421, 422, 473, 424, 473, 473, 473, - /* 2270 */ 473, 410, 473, 473, 473, 414, 473, 473, 417, 418, - /* 2280 */ 419, 420, 421, 422, 473, 424, 473, 365, 473, 473, - /* 2290 */ 473, 473, 473, 473, 473, 373, 473, 473, 473, 473, - /* 2300 */ 378, 473, 380, 473, 473, 473, 473, 473, 473, 473, - /* 2310 */ 473, 473, 332, 473, 473, 473, 473, 473, 473, 473, - /* 2320 */ 473, 473, 473, 473, 473, 473, 473, 473, 332, 473, - /* 2330 */ 473, 473, 410, 473, 473, 473, 414, 473, 473, 417, - /* 2340 */ 418, 419, 420, 421, 422, 365, 424, 473, 473, 473, - /* 2350 */ 473, 473, 473, 373, 473, 473, 473, 473, 378, 473, - /* 2360 */ 380, 365, 473, 473, 473, 473, 473, 473, 473, 373, - /* 2370 */ 473, 473, 473, 473, 378, 473, 380, 473, 473, 473, - /* 2380 */ 473, 473, 473, 473, 473, 473, 473, 473, 473, 473, - /* 2390 */ 410, 332, 473, 473, 414, 473, 473, 417, 418, 419, - /* 2400 */ 420, 421, 422, 473, 424, 473, 410, 332, 473, 473, - /* 2410 */ 414, 473, 473, 417, 418, 419, 420, 421, 422, 473, - /* 2420 */ 424, 473, 473, 473, 365, 473, 332, 473, 473, 473, - /* 2430 */ 473, 473, 373, 473, 473, 473, 473, 378, 473, 380, - /* 2440 */ 365, 473, 473, 473, 473, 473, 473, 473, 373, 473, - /* 2450 */ 473, 473, 473, 378, 473, 380, 473, 473, 473, 365, - /* 2460 */ 473, 473, 473, 473, 473, 473, 473, 373, 473, 410, - /* 2470 */ 473, 473, 378, 414, 380, 473, 417, 418, 419, 420, - /* 2480 */ 421, 422, 473, 424, 473, 410, 473, 473, 473, 414, - /* 2490 */ 332, 473, 417, 418, 419, 420, 421, 422, 473, 424, - /* 2500 */ 473, 473, 473, 473, 410, 473, 473, 332, 414, 473, - /* 2510 */ 473, 417, 418, 419, 420, 421, 422, 473, 424, 473, - /* 2520 */ 473, 473, 473, 365, 473, 473, 473, 473, 473, 473, - /* 2530 */ 473, 373, 473, 473, 473, 473, 378, 473, 380, 473, - /* 2540 */ 365, 473, 473, 473, 473, 473, 473, 473, 373, 473, - /* 2550 */ 473, 473, 473, 378, 473, 380, 473, 473, 473, 473, - /* 2560 */ 473, 473, 473, 473, 473, 473, 473, 332, 410, 473, - /* 2570 */ 473, 473, 414, 473, 473, 417, 418, 419, 420, 421, - /* 2580 */ 422, 473, 424, 473, 473, 410, 473, 332, 473, 414, - /* 2590 */ 473, 473, 417, 418, 419, 420, 421, 422, 473, 424, - /* 2600 */ 365, 473, 473, 473, 473, 473, 473, 473, 373, 473, - /* 2610 */ 473, 473, 473, 378, 473, 380, 473, 473, 473, 473, - /* 2620 */ 365, 473, 473, 473, 473, 473, 473, 473, 373, 473, - /* 2630 */ 473, 473, 473, 378, 473, 380, 473, 473, 473, 473, - /* 2640 */ 473, 473, 473, 473, 473, 410, 473, 473, 473, 414, - /* 2650 */ 473, 473, 417, 418, 419, 420, 421, 422, 473, 424, - /* 2660 */ 473, 332, 473, 473, 473, 410, 473, 473, 473, 414, - /* 2670 */ 473, 473, 417, 418, 419, 420, 421, 422, 332, 424, - /* 2680 */ 473, 473, 473, 473, 473, 473, 473, 473, 473, 473, - /* 2690 */ 473, 473, 473, 473, 365, 473, 473, 473, 473, 473, - /* 2700 */ 473, 473, 373, 473, 473, 473, 473, 378, 473, 380, - /* 2710 */ 473, 365, 473, 473, 473, 473, 473, 473, 473, 373, - /* 2720 */ 473, 473, 473, 473, 378, 473, 380, 473, 473, 473, - /* 2730 */ 473, 473, 473, 473, 473, 473, 473, 473, 473, 410, - /* 2740 */ 332, 473, 473, 414, 473, 473, 417, 418, 419, 420, - /* 2750 */ 421, 422, 473, 424, 473, 473, 410, 473, 473, 473, - /* 2760 */ 414, 473, 473, 417, 418, 419, 420, 421, 422, 473, - /* 2770 */ 424, 473, 473, 365, 473, 473, 473, 473, 473, 473, - /* 2780 */ 473, 373, 473, 473, 473, 473, 378, 473, 380, 473, - /* 2790 */ 473, 473, 473, 473, 473, 473, 473, 473, 473, 473, - /* 2800 */ 473, 473, 473, 473, 473, 473, 473, 473, 473, 473, - /* 2810 */ 473, 473, 473, 473, 473, 473, 473, 473, 410, 473, - /* 2820 */ 473, 473, 414, 473, 473, 417, 418, 419, 420, 421, - /* 2830 */ 422, 473, 424, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 181, 4, + /* 240 */ 387, 18, 100, 20, 20, 100, 22, 100, 350, 102, + /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 369, 35, + /* 260 */ 400, 343, 205, 206, 404, 12, 13, 14, 15, 16, + /* 270 */ 465, 466, 49, 375, 51, 470, 471, 54, 43, 390, + /* 280 */ 45, 46, 252, 59, 67, 68, 69, 369, 0, 65, + /* 290 */ 100, 74, 75, 76, 355, 356, 72, 80, 109, 381, + /* 300 */ 110, 383, 85, 86, 87, 88, 20, 447, 91, 21, + /* 310 */ 450, 251, 24, 25, 26, 27, 28, 29, 30, 31, + /* 320 */ 32, 97, 99, 348, 100, 465, 466, 185, 3, 187, + /* 330 */ 470, 471, 414, 20, 111, 22, 418, 342, 343, 421, + /* 340 */ 422, 423, 424, 425, 426, 20, 428, 372, 35, 431, + /* 350 */ 342, 433, 434, 435, 165, 360, 214, 439, 440, 252, + /* 360 */ 136, 137, 367, 140, 20, 52, 143, 144, 145, 146, + /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + /* 380 */ 157, 158, 159, 369, 161, 162, 163, 342, 343, 20, + /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 369, 252, + /* 400 */ 67, 68, 69, 389, 390, 360, 377, 74, 75, 76, + /* 410 */ 186, 466, 188, 80, 171, 470, 471, 0, 85, 86, + /* 420 */ 87, 88, 136, 137, 91, 3, 101, 35, 239, 240, + /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, + /* 440 */ 216, 0, 218, 219, 220, 221, 222, 223, 224, 225, + /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, + /* 460 */ 236, 12, 13, 33, 72, 22, 49, 20, 343, 20, + /* 470 */ 333, 22, 20, 59, 351, 215, 64, 20, 35, 49, + /* 480 */ 343, 358, 33, 81, 35, 55, 56, 57, 58, 59, + /* 490 */ 59, 8, 9, 168, 369, 12, 13, 14, 15, 16, + /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 236, + /* 510 */ 20, 238, 22, 99, 65, 72, 102, 381, 381, 342, + /* 520 */ 383, 72, 262, 263, 264, 265, 266, 267, 268, 99, + /* 530 */ 8, 9, 102, 102, 12, 13, 14, 15, 16, 0, + /* 540 */ 97, 338, 52, 141, 142, 342, 97, 344, 423, 100, + /* 550 */ 14, 414, 100, 417, 418, 418, 20, 100, 421, 422, + /* 560 */ 423, 424, 425, 426, 428, 428, 164, 365, 18, 392, + /* 570 */ 433, 394, 435, 23, 8, 9, 439, 440, 12, 13, + /* 580 */ 14, 15, 16, 136, 137, 136, 137, 37, 38, 452, + /* 590 */ 432, 41, 434, 357, 4, 359, 155, 460, 333, 169, + /* 600 */ 170, 100, 64, 64, 174, 164, 176, 1, 2, 19, + /* 610 */ 60, 61, 62, 63, 412, 166, 167, 361, 361, 172, + /* 620 */ 173, 172, 173, 33, 194, 369, 369, 342, 343, 186, + /* 630 */ 14, 188, 376, 377, 377, 186, 20, 188, 447, 49, + /* 640 */ 384, 450, 342, 343, 54, 360, 381, 20, 369, 59, + /* 650 */ 100, 168, 14, 15, 16, 376, 465, 466, 215, 216, + /* 660 */ 360, 470, 471, 384, 215, 216, 20, 218, 219, 220, + /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, + /* 690 */ 369, 333, 102, 20, 272, 22, 333, 376, 342, 343, + /* 700 */ 178, 343, 166, 35, 252, 384, 33, 101, 35, 252, + /* 710 */ 420, 2, 355, 356, 342, 343, 360, 8, 9, 342, + /* 720 */ 343, 12, 13, 14, 15, 16, 111, 369, 178, 179, + /* 730 */ 180, 129, 59, 183, 168, 133, 446, 360, 342, 381, + /* 740 */ 72, 383, 8, 9, 381, 72, 12, 13, 14, 15, + /* 750 */ 16, 342, 343, 252, 204, 431, 378, 207, 434, 381, + /* 760 */ 210, 211, 212, 213, 214, 236, 130, 342, 343, 360, + /* 770 */ 97, 72, 414, 100, 342, 343, 418, 333, 342, 421, + /* 780 */ 422, 423, 424, 425, 426, 360, 428, 343, 392, 345, + /* 790 */ 394, 433, 360, 435, 2, 361, 350, 439, 440, 277, + /* 800 */ 8, 9, 252, 369, 12, 13, 14, 15, 16, 136, + /* 810 */ 137, 377, 366, 369, 168, 443, 444, 445, 460, 447, + /* 820 */ 448, 375, 450, 342, 343, 381, 111, 383, 392, 369, + /* 830 */ 394, 342, 343, 197, 198, 101, 376, 465, 466, 166, + /* 840 */ 167, 343, 470, 471, 384, 172, 173, 8, 9, 360, + /* 850 */ 72, 12, 13, 14, 15, 16, 0, 48, 414, 186, + /* 860 */ 22, 188, 418, 342, 343, 421, 422, 423, 424, 425, + /* 870 */ 426, 369, 428, 35, 131, 369, 343, 433, 135, 435, + /* 880 */ 37, 360, 376, 439, 440, 387, 384, 400, 215, 216, + /* 890 */ 384, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, + /* 910 */ 13, 333, 333, 45, 46, 342, 343, 20, 333, 22, + /* 920 */ 387, 343, 370, 345, 443, 444, 445, 0, 447, 448, + /* 930 */ 33, 333, 35, 360, 447, 97, 20, 450, 195, 196, + /* 940 */ 101, 343, 199, 4, 201, 333, 103, 369, 105, 106, + /* 950 */ 333, 108, 465, 466, 370, 21, 59, 470, 471, 381, + /* 960 */ 381, 383, 342, 343, 342, 343, 381, 369, 34, 72, + /* 970 */ 36, 378, 129, 370, 381, 378, 133, 168, 381, 381, + /* 980 */ 360, 383, 360, 420, 370, 129, 130, 131, 132, 133, + /* 990 */ 134, 135, 414, 381, 97, 400, 418, 100, 381, 421, + /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 42, 446, + /* 1010 */ 44, 433, 414, 435, 20, 343, 418, 439, 440, 421, + /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 342, + /* 1030 */ 343, 8, 9, 136, 137, 12, 13, 14, 15, 16, + /* 1040 */ 361, 369, 447, 333, 370, 450, 14, 360, 369, 342, + /* 1050 */ 343, 44, 20, 381, 447, 383, 377, 450, 342, 343, + /* 1060 */ 465, 466, 22, 166, 167, 470, 471, 360, 165, 172, + /* 1070 */ 173, 104, 465, 466, 107, 35, 360, 470, 471, 250, + /* 1080 */ 251, 342, 343, 186, 168, 188, 414, 333, 342, 343, + /* 1090 */ 418, 381, 22, 421, 422, 423, 424, 425, 426, 360, + /* 1100 */ 428, 400, 333, 342, 343, 35, 360, 333, 101, 42, + /* 1110 */ 333, 44, 215, 216, 333, 218, 219, 220, 221, 222, + /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, + /* 1130 */ 233, 234, 235, 12, 13, 381, 209, 97, 0, 333, + /* 1140 */ 333, 20, 239, 22, 472, 473, 333, 333, 447, 44, + /* 1150 */ 381, 450, 249, 333, 33, 381, 35, 343, 381, 345, + /* 1160 */ 166, 44, 381, 333, 333, 333, 465, 466, 111, 13, + /* 1170 */ 362, 470, 471, 365, 333, 104, 104, 35, 107, 107, + /* 1180 */ 59, 13, 44, 369, 343, 0, 345, 381, 381, 44, + /* 1190 */ 405, 35, 253, 72, 381, 381, 104, 383, 166, 107, + /* 1200 */ 200, 381, 202, 35, 443, 444, 445, 22, 447, 448, + /* 1210 */ 369, 381, 381, 381, 44, 44, 44, 160, 97, 0, + /* 1220 */ 44, 100, 381, 0, 383, 136, 137, 370, 414, 358, + /* 1230 */ 1, 2, 418, 44, 333, 421, 422, 423, 424, 425, + /* 1240 */ 426, 22, 428, 35, 343, 22, 101, 433, 44, 435, + /* 1250 */ 47, 44, 65, 439, 440, 414, 44, 136, 137, 418, + /* 1260 */ 44, 44, 421, 422, 423, 424, 425, 426, 391, 428, + /* 1270 */ 369, 101, 101, 101, 433, 44, 435, 101, 44, 334, + /* 1280 */ 439, 440, 381, 346, 383, 474, 463, 166, 167, 369, + /* 1290 */ 101, 457, 346, 172, 173, 391, 129, 130, 131, 132, + /* 1300 */ 133, 134, 135, 100, 341, 101, 380, 186, 101, 188, + /* 1310 */ 391, 44, 13, 101, 44, 414, 44, 101, 101, 418, + /* 1320 */ 0, 44, 421, 422, 423, 424, 425, 426, 343, 428, + /* 1330 */ 188, 449, 101, 44, 35, 101, 215, 216, 44, 218, + /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 467, 441, 12, + /* 1360 */ 13, 13, 451, 462, 254, 416, 49, 333, 101, 22, + /* 1370 */ 50, 101, 415, 101, 184, 402, 42, 343, 101, 345, + /* 1380 */ 33, 276, 35, 35, 388, 20, 391, 388, 386, 165, + /* 1390 */ 101, 274, 20, 342, 342, 101, 188, 333, 388, 386, + /* 1400 */ 386, 98, 215, 369, 354, 96, 59, 343, 353, 345, + /* 1410 */ 342, 95, 352, 20, 335, 381, 342, 383, 342, 72, + /* 1420 */ 342, 48, 339, 335, 339, 20, 409, 350, 20, 383, + /* 1430 */ 350, 20, 20, 369, 344, 344, 401, 350, 53, 342, + /* 1440 */ 342, 347, 347, 350, 97, 381, 350, 383, 414, 350, + /* 1450 */ 350, 335, 418, 369, 335, 421, 422, 423, 424, 425, + /* 1460 */ 426, 203, 428, 381, 100, 333, 413, 433, 409, 435, + /* 1470 */ 369, 369, 369, 439, 440, 343, 369, 369, 414, 348, + /* 1480 */ 369, 369, 418, 369, 369, 421, 422, 423, 424, 425, + /* 1490 */ 426, 369, 428, 192, 381, 191, 348, 433, 190, 435, + /* 1500 */ 342, 369, 261, 439, 440, 381, 383, 381, 260, 381, + /* 1510 */ 456, 391, 408, 381, 456, 383, 407, 411, 391, 269, + /* 1520 */ 381, 396, 381, 177, 406, 396, 271, 270, 459, 255, + /* 1530 */ 455, 458, 278, 186, 275, 188, 475, 456, 273, 251, + /* 1540 */ 454, 343, 20, 342, 344, 348, 414, 348, 20, 394, + /* 1550 */ 418, 416, 170, 421, 422, 423, 424, 425, 426, 396, + /* 1560 */ 428, 381, 215, 216, 420, 433, 393, 435, 348, 381, + /* 1570 */ 396, 439, 440, 333, 381, 228, 229, 230, 231, 232, + /* 1580 */ 233, 234, 343, 343, 348, 100, 453, 381, 438, 100, + /* 1590 */ 381, 381, 365, 333, 373, 381, 359, 348, 342, 469, + /* 1600 */ 36, 335, 468, 343, 336, 403, 410, 363, 397, 369, + /* 1610 */ 397, 349, 331, 0, 363, 363, 0, 0, 42, 0, + /* 1620 */ 35, 381, 208, 383, 35, 35, 35, 208, 0, 369, + /* 1630 */ 35, 35, 208, 0, 208, 0, 35, 0, 0, 35, + /* 1640 */ 22, 381, 195, 383, 188, 186, 0, 0, 0, 182, + /* 1650 */ 181, 0, 0, 47, 414, 0, 0, 0, 418, 0, + /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 42, 428, 0, + /* 1670 */ 0, 0, 0, 433, 414, 435, 155, 333, 418, 439, + /* 1680 */ 440, 421, 422, 423, 424, 425, 426, 343, 428, 0, + /* 1690 */ 0, 0, 155, 433, 0, 435, 35, 0, 0, 439, + /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1710 */ 0, 343, 0, 369, 0, 0, 0, 42, 0, 0, + /* 1720 */ 0, 0, 0, 0, 0, 381, 0, 383, 0, 0, + /* 1730 */ 0, 139, 48, 22, 48, 22, 0, 369, 59, 22, + /* 1740 */ 0, 59, 0, 0, 59, 0, 47, 47, 14, 381, + /* 1750 */ 177, 383, 0, 0, 0, 0, 0, 0, 414, 0, + /* 1760 */ 35, 66, 418, 0, 0, 421, 422, 423, 424, 425, + /* 1770 */ 426, 39, 428, 42, 47, 40, 39, 44, 333, 435, + /* 1780 */ 39, 35, 414, 439, 440, 0, 418, 49, 343, 421, + /* 1790 */ 422, 423, 424, 425, 426, 35, 428, 39, 49, 0, + /* 1800 */ 39, 333, 35, 435, 39, 0, 35, 439, 440, 0, + /* 1810 */ 0, 343, 39, 0, 369, 49, 0, 49, 35, 22, + /* 1820 */ 0, 35, 35, 35, 333, 35, 381, 35, 383, 35, + /* 1830 */ 35, 35, 0, 44, 343, 44, 35, 369, 35, 22, + /* 1840 */ 22, 0, 0, 22, 51, 22, 35, 333, 109, 381, + /* 1850 */ 0, 383, 0, 0, 107, 22, 20, 343, 0, 414, + /* 1860 */ 369, 35, 35, 418, 35, 374, 421, 422, 423, 424, + /* 1870 */ 425, 426, 381, 428, 383, 35, 101, 100, 35, 100, + /* 1880 */ 435, 193, 414, 369, 439, 440, 418, 22, 374, 421, + /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 35, 0, + /* 1900 */ 22, 0, 0, 435, 44, 414, 3, 100, 440, 418, + /* 1910 */ 333, 96, 421, 422, 423, 424, 425, 426, 170, 428, + /* 1920 */ 343, 256, 48, 101, 100, 48, 175, 47, 414, 98, + /* 1930 */ 168, 101, 418, 189, 44, 421, 422, 423, 424, 425, + /* 1940 */ 426, 44, 428, 47, 168, 101, 369, 100, 168, 100, + /* 1950 */ 44, 100, 44, 3, 44, 101, 100, 256, 381, 35, + /* 1960 */ 383, 35, 101, 101, 35, 35, 35, 35, 47, 101, + /* 1970 */ 101, 44, 47, 0, 0, 333, 0, 0, 39, 100, + /* 1980 */ 47, 100, 0, 101, 100, 343, 101, 100, 100, 100, + /* 1990 */ 171, 414, 39, 47, 169, 418, 333, 2, 421, 422, + /* 2000 */ 423, 424, 425, 426, 110, 428, 343, 44, 22, 333, + /* 2010 */ 98, 369, 98, 215, 47, 237, 100, 47, 101, 343, + /* 2020 */ 100, 22, 100, 381, 250, 383, 111, 35, 256, 101, + /* 2030 */ 22, 217, 369, 35, 44, 101, 100, 374, 100, 35, + /* 2040 */ 100, 464, 101, 101, 381, 369, 383, 100, 35, 101, + /* 2050 */ 101, 100, 100, 35, 35, 101, 414, 381, 100, 383, + /* 2060 */ 418, 101, 100, 421, 422, 423, 424, 425, 426, 100, + /* 2070 */ 428, 100, 35, 35, 65, 100, 66, 414, 122, 122, + /* 2080 */ 35, 418, 35, 333, 421, 422, 423, 424, 425, 426, + /* 2090 */ 414, 428, 35, 343, 418, 35, 35, 421, 422, 423, + /* 2100 */ 424, 425, 426, 333, 428, 122, 430, 122, 35, 35, + /* 2110 */ 35, 35, 72, 343, 44, 473, 94, 35, 35, 369, + /* 2120 */ 35, 22, 35, 35, 374, 35, 72, 35, 35, 35, + /* 2130 */ 35, 381, 35, 383, 22, 35, 0, 35, 49, 369, + /* 2140 */ 39, 0, 35, 39, 374, 49, 0, 49, 35, 39, + /* 2150 */ 0, 381, 35, 383, 49, 39, 0, 35, 35, 0, + /* 2160 */ 22, 21, 476, 21, 414, 22, 22, 20, 418, 476, + /* 2170 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2180 */ 476, 476, 476, 476, 414, 476, 1, 476, 418, 476, + /* 2190 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2200 */ 333, 476, 476, 476, 19, 476, 476, 476, 476, 476, + /* 2210 */ 343, 476, 476, 476, 476, 476, 333, 476, 33, 476, + /* 2220 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2230 */ 476, 476, 476, 476, 49, 476, 369, 476, 476, 476, + /* 2240 */ 55, 56, 57, 58, 59, 476, 333, 476, 381, 476, + /* 2250 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, + /* 2260 */ 476, 476, 476, 476, 381, 476, 383, 476, 476, 476, + /* 2270 */ 476, 476, 476, 476, 476, 476, 333, 476, 476, 476, + /* 2280 */ 476, 414, 369, 476, 99, 418, 343, 102, 421, 422, + /* 2290 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, + /* 2300 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2310 */ 476, 428, 369, 476, 476, 476, 476, 476, 476, 134, + /* 2320 */ 476, 476, 476, 476, 381, 476, 383, 414, 476, 476, + /* 2330 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2340 */ 333, 428, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2350 */ 343, 476, 476, 333, 169, 476, 476, 414, 476, 174, + /* 2360 */ 476, 418, 476, 343, 421, 422, 423, 424, 425, 426, + /* 2370 */ 476, 428, 476, 476, 476, 476, 369, 476, 476, 194, + /* 2380 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2390 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2400 */ 476, 381, 476, 383, 333, 476, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 343, 476, 476, 333, 476, 476, + /* 2420 */ 476, 414, 476, 476, 476, 418, 476, 343, 421, 422, + /* 2430 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2440 */ 369, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2460 */ 476, 476, 476, 476, 476, 381, 476, 383, 476, 476, + /* 2470 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2480 */ 476, 476, 476, 476, 476, 414, 476, 476, 343, 418, + /* 2490 */ 476, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2500 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2510 */ 426, 476, 428, 476, 369, 476, 476, 476, 476, 476, + /* 2520 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 333, + /* 2530 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 343, + /* 2540 */ 476, 476, 476, 476, 333, 476, 476, 476, 476, 476, + /* 2550 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 414, + /* 2560 */ 476, 476, 476, 418, 476, 369, 421, 422, 423, 424, + /* 2570 */ 425, 426, 476, 428, 476, 476, 476, 381, 476, 383, + /* 2580 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2590 */ 333, 476, 381, 476, 383, 476, 476, 476, 476, 476, + /* 2600 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2610 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, + /* 2620 */ 424, 425, 426, 476, 428, 414, 369, 476, 476, 418, + /* 2630 */ 476, 476, 421, 422, 423, 424, 425, 426, 381, 428, + /* 2640 */ 383, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2670 */ 476, 414, 476, 476, 476, 418, 476, 369, 421, 422, + /* 2680 */ 423, 424, 425, 426, 476, 428, 476, 333, 476, 381, + /* 2690 */ 476, 383, 476, 476, 369, 476, 476, 343, 476, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 476, + /* 2710 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2720 */ 476, 476, 414, 369, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 381, 428, 383, 476, 414, + /* 2740 */ 476, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2750 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2760 */ 476, 476, 476, 476, 476, 343, 476, 476, 414, 476, + /* 2770 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2780 */ 426, 476, 428, 476, 343, 476, 476, 333, 476, 476, + /* 2790 */ 476, 369, 476, 476, 476, 476, 476, 343, 476, 476, + /* 2800 */ 476, 476, 476, 381, 476, 383, 476, 476, 476, 476, + /* 2810 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 381, 414, 383, 476, 476, + /* 2840 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2850 */ 428, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 2860 */ 333, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2870 */ 343, 476, 418, 333, 476, 421, 422, 423, 424, 425, + /* 2880 */ 426, 476, 428, 343, 476, 476, 476, 476, 333, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 369, 476, 343, 476, + /* 2900 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2910 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 381, 476, 383, 369, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, + /* 2940 */ 476, 414, 476, 476, 343, 418, 476, 476, 421, 422, + /* 2950 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2960 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, + /* 2970 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2980 */ 425, 426, 381, 428, 383, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 3020 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, }; -#define YY_SHIFT_COUNT (752) +#define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2242) +#define YY_SHIFT_MAX (2185) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 957, 0, 98, 0, 321, 321, 321, 321, 321, 321, - /* 10 */ 321, 321, 321, 321, 321, 419, 641, 641, 739, 641, - /* 20 */ 641, 641, 641, 641, 641, 641, 641, 641, 641, 641, - /* 30 */ 641, 641, 641, 641, 641, 641, 641, 641, 641, 641, - /* 40 */ 641, 641, 641, 641, 641, 641, 641, 641, 167, 591, - /* 50 */ 873, 68, 255, 90, 246, 90, 68, 68, 966, 966, - /* 60 */ 966, 90, 966, 966, 380, 90, 9, 889, 24, 24, - /* 70 */ 889, 151, 151, 41, 377, 63, 63, 24, 24, 24, - /* 80 */ 24, 24, 24, 24, 84, 24, 24, 108, 9, 24, - /* 90 */ 24, 221, 24, 9, 24, 84, 24, 84, 9, 24, - /* 100 */ 24, 9, 24, 9, 9, 9, 24, 205, 956, 285, - /* 110 */ 285, 251, 407, 665, 665, 665, 665, 665, 665, 665, - /* 120 */ 665, 665, 665, 665, 665, 665, 665, 665, 665, 665, - /* 130 */ 665, 665, 713, 459, 41, 377, 441, 513, 513, 513, - /* 140 */ 948, 194, 194, 441, 282, 282, 282, 108, 269, 164, - /* 150 */ 9, 354, 9, 354, 354, 324, 435, 245, 245, 245, - /* 160 */ 245, 245, 245, 245, 245, 1318, 375, 125, 400, 549, - /* 170 */ 238, 112, 95, 129, 879, 735, 872, 985, 705, 902, - /* 180 */ 895, 795, 486, 895, 1104, 259, 1044, 1112, 1320, 1198, - /* 190 */ 1348, 1373, 1348, 1234, 1380, 1380, 1348, 1234, 1234, 1326, - /* 200 */ 1331, 1380, 1336, 1380, 1380, 1380, 1416, 1416, 1421, 108, - /* 210 */ 1423, 108, 1427, 1431, 108, 1427, 108, 108, 108, 1380, - /* 220 */ 108, 1403, 1403, 1416, 9, 9, 9, 9, 9, 9, - /* 230 */ 9, 9, 9, 9, 9, 1380, 1416, 354, 354, 354, - /* 240 */ 1275, 1383, 1421, 205, 1294, 1321, 1423, 205, 1380, 1373, - /* 250 */ 1373, 354, 1290, 1292, 354, 1290, 1292, 354, 354, 9, - /* 260 */ 1293, 1384, 1290, 1296, 1299, 1309, 1112, 1297, 1301, 1295, - /* 270 */ 1328, 282, 1561, 1380, 1427, 205, 205, 1566, 1292, 354, - /* 280 */ 354, 354, 354, 354, 1292, 354, 1452, 205, 324, 205, - /* 290 */ 282, 1541, 1548, 354, 435, 1380, 205, 1624, 1416, 2833, - /* 300 */ 2833, 2833, 2833, 2833, 2833, 2833, 2833, 2833, 170, 1402, - /* 310 */ 152, 981, 558, 581, 680, 157, 374, 706, 775, 785, - /* 320 */ 920, 920, 920, 920, 920, 920, 920, 920, 920, 173, - /* 330 */ 598, 27, 27, 132, 639, 353, 947, 418, 962, 816, - /* 340 */ 489, 638, 638, 880, 881, 517, 880, 880, 880, 1133, - /* 350 */ 474, 986, 1109, 843, 779, 1017, 1102, 1107, 1144, 1162, - /* 360 */ 1202, 1225, 976, 1141, 1164, 992, 1172, 1191, 1193, 1142, - /* 370 */ 969, 1022, 1147, 1205, 1212, 1233, 1238, 1240, 1245, 1251, - /* 380 */ 1247, 1115, 1156, 1092, 1192, 573, 1253, 1254, 1257, 1258, - /* 390 */ 1278, 1279, 800, 1322, 1343, 1298, 1267, 1677, 1679, 1680, - /* 400 */ 1645, 1690, 1659, 1489, 1662, 1669, 1672, 1501, 1709, 1675, - /* 410 */ 1676, 1506, 1715, 1511, 1720, 1687, 1723, 1702, 1727, 1693, - /* 420 */ 1535, 1542, 1545, 1732, 1733, 1735, 1554, 1556, 1738, 1740, - /* 430 */ 1695, 1743, 1744, 1745, 1704, 1747, 1748, 1749, 1750, 1752, - /* 440 */ 1753, 1754, 1757, 1609, 1731, 1767, 1613, 1769, 1770, 1773, - /* 450 */ 1774, 1786, 1787, 1788, 1790, 1791, 1792, 1793, 1795, 1796, - /* 460 */ 1798, 1799, 1800, 1742, 1801, 1803, 1805, 1806, 1807, 1808, - /* 470 */ 1789, 1809, 1810, 1812, 1674, 1815, 1816, 1797, 1721, 1802, - /* 480 */ 1722, 1820, 1763, 1794, 1822, 1765, 1825, 1768, 1827, 1828, - /* 490 */ 1811, 1813, 1814, 1783, 1818, 1804, 1819, 1817, 1835, 1821, - /* 500 */ 1824, 1836, 1837, 1840, 1826, 1673, 1849, 1854, 1856, 1823, - /* 510 */ 1857, 1859, 1831, 1830, 1829, 1867, 1834, 1833, 1832, 1870, - /* 520 */ 1838, 1841, 1843, 1874, 1844, 1845, 1847, 1875, 1876, 1883, - /* 530 */ 1884, 1781, 1839, 1860, 1869, 1894, 1862, 1863, 1872, 1878, - /* 540 */ 1865, 1866, 1879, 1880, 1889, 1881, 1905, 1896, 1919, 1898, - /* 550 */ 1873, 1930, 1900, 1897, 1934, 1901, 1935, 1902, 1938, 1917, - /* 560 */ 1920, 1906, 1907, 1908, 1846, 1848, 1944, 1858, 1851, 1914, - /* 570 */ 1932, 1952, 1766, 1937, 1864, 1868, 1956, 1957, 1871, 1852, - /* 580 */ 1958, 1921, 1710, 1877, 1885, 1882, 1888, 1887, 1891, 1893, - /* 590 */ 1890, 1923, 1924, 1892, 1909, 1912, 1913, 1895, 1925, 1916, - /* 600 */ 1926, 1928, 1927, 1716, 1904, 1929, 1969, 1931, 1724, 1941, - /* 610 */ 1945, 1954, 1971, 1972, 1979, 1933, 1936, 1970, 1850, 1948, - /* 620 */ 1988, 2036, 2041, 2042, 2043, 1946, 2006, 1783, 2008, 1953, - /* 630 */ 1955, 1959, 1962, 1963, 1886, 1964, 2067, 2020, 1911, 1976, - /* 640 */ 1967, 1783, 2031, 2038, 1985, 1853, 1986, 2083, 2064, 1899, - /* 650 */ 1987, 1989, 1992, 1993, 1995, 1999, 2049, 1997, 2001, 2055, - /* 660 */ 2002, 2085, 1915, 2005, 1978, 2007, 2074, 2078, 2010, 2013, - /* 670 */ 2082, 2024, 2026, 2093, 2029, 2030, 2097, 2033, 2034, 2102, - /* 680 */ 2051, 2015, 2019, 2032, 2035, 2130, 2044, 2053, 2110, 2060, - /* 690 */ 2124, 2061, 2110, 2110, 2141, 2100, 2103, 2131, 2133, 2134, - /* 700 */ 2135, 2137, 2138, 2142, 2149, 2151, 2152, 2105, 2095, 2145, - /* 710 */ 2155, 2156, 2157, 2173, 2161, 2162, 2164, 2129, 1865, 2174, - /* 720 */ 1866, 2176, 2177, 2178, 2179, 2188, 2181, 2217, 2183, 2171, - /* 730 */ 2182, 2223, 2189, 2180, 2186, 2227, 2194, 2184, 2191, 2231, - /* 740 */ 2198, 2187, 2197, 2237, 2203, 2205, 2242, 2221, 2224, 2222, - /* 750 */ 2225, 2228, 2226, + /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, + /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, + /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 50 */ 897, 452, 457, 142, 145, 147, 30, 107, 30, 145, + /* 60 */ 145, 1347, 30, 1347, 1347, 501, 30, 56, 447, 344, + /* 70 */ 344, 447, 235, 235, 18, 286, 150, 150, 344, 344, + /* 80 */ 344, 344, 344, 344, 344, 369, 344, 344, 412, 56, + /* 90 */ 344, 344, 484, 344, 56, 344, 369, 344, 369, 56, + /* 100 */ 344, 344, 56, 344, 56, 56, 56, 344, 538, 223, + /* 110 */ 189, 189, 333, 114, 443, 443, 443, 443, 443, 443, + /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, + /* 130 */ 443, 443, 443, 843, 325, 18, 286, 392, 1, 1, + /* 140 */ 1, 539, 273, 273, 392, 627, 627, 627, 412, 615, + /* 150 */ 529, 56, 699, 56, 699, 699, 715, 778, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 2185, 217, 288, 483, + /* 170 */ 522, 260, 313, 243, 536, 1032, 490, 646, 868, 616, + /* 180 */ 602, 916, 829, 60, 422, 829, 966, 939, 994, 1110, + /* 190 */ 1317, 1190, 1334, 1365, 1334, 1224, 1372, 1372, 1334, 1224, + /* 200 */ 1224, 1303, 1309, 1372, 1316, 1372, 1372, 1372, 1393, 1373, + /* 210 */ 1393, 1373, 1405, 412, 1408, 412, 1411, 1412, 412, 1411, + /* 220 */ 412, 412, 412, 1372, 412, 1385, 1385, 1393, 56, 56, + /* 230 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 1372, + /* 240 */ 1393, 699, 699, 699, 1258, 1364, 1405, 538, 1301, 1304, + /* 250 */ 1408, 538, 1308, 1372, 1365, 1365, 699, 1241, 1248, 699, + /* 260 */ 1241, 1248, 699, 699, 56, 1250, 1346, 1241, 1255, 1257, + /* 270 */ 1274, 1110, 1254, 1259, 1265, 1288, 627, 1522, 1372, 1411, + /* 280 */ 538, 538, 1528, 1248, 699, 699, 699, 699, 699, 1248, + /* 290 */ 699, 1382, 538, 715, 538, 627, 1485, 1489, 699, 778, + /* 300 */ 1372, 538, 1564, 1393, 3030, 3030, 3030, 3030, 3030, 3030, + /* 310 */ 3030, 3030, 3030, 34, 430, 15, 590, 734, 16, 839, + /* 320 */ 79, 709, 792, 566, 856, 1023, 1023, 1023, 1023, 1023, + /* 330 */ 1023, 1023, 1023, 1023, 1167, 743, 253, 253, 402, 57, + /* 340 */ 441, 414, 838, 1040, 934, 636, 64, 64, 638, 606, + /* 350 */ 903, 638, 638, 638, 417, 927, 1007, 1070, 1067, 1057, + /* 360 */ 1138, 967, 1071, 1072, 1092, 1156, 1168, 1185, 1219, 1223, + /* 370 */ 1000, 1145, 1170, 431, 1171, 1172, 1176, 1089, 1117, 1105, + /* 380 */ 809, 1189, 1204, 1207, 1212, 1216, 1217, 1229, 1231, 1142, + /* 390 */ 1208, 1187, 1234, 1203, 1267, 1270, 1272, 1277, 1289, 1294, + /* 400 */ 190, 1299, 1348, 668, 1320, 1613, 1616, 1617, 1576, 1619, + /* 410 */ 1585, 1414, 1589, 1590, 1591, 1419, 1628, 1595, 1596, 1424, + /* 420 */ 1633, 1426, 1635, 1601, 1637, 1618, 1638, 1604, 1447, 1456, + /* 430 */ 1459, 1646, 1647, 1648, 1467, 1469, 1651, 1652, 1606, 1655, + /* 440 */ 1656, 1657, 1625, 1659, 1660, 1669, 1670, 1671, 1672, 1689, + /* 450 */ 1690, 1521, 1661, 1691, 1537, 1694, 1697, 1698, 1702, 1703, + /* 460 */ 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1712, 1714, 1715, + /* 470 */ 1716, 1675, 1718, 1719, 1720, 1721, 1722, 1723, 1711, 1724, + /* 480 */ 1726, 1728, 1592, 1729, 1730, 1713, 1684, 1717, 1686, 1736, + /* 490 */ 1679, 1725, 1740, 1682, 1742, 1685, 1743, 1745, 1731, 1732, + /* 500 */ 1733, 1699, 1700, 1734, 1727, 1752, 1735, 1737, 1753, 1754, + /* 510 */ 1755, 1741, 1573, 1756, 1757, 1759, 1695, 1763, 1764, 1746, + /* 520 */ 1738, 1758, 1785, 1760, 1749, 1761, 1799, 1767, 1766, 1765, + /* 530 */ 1805, 1771, 1768, 1773, 1809, 1810, 1813, 1816, 1739, 1747, + /* 540 */ 1783, 1797, 1820, 1786, 1787, 1788, 1790, 1792, 1794, 1795, + /* 550 */ 1789, 1791, 1796, 1801, 1817, 1803, 1832, 1818, 1841, 1821, + /* 560 */ 1793, 1842, 1823, 1811, 1850, 1826, 1852, 1827, 1853, 1833, + /* 570 */ 1836, 1829, 1840, 1843, 1775, 1777, 1858, 1762, 1779, 1688, + /* 580 */ 1863, 1865, 1899, 1744, 1878, 1776, 1748, 1901, 1902, 1780, + /* 590 */ 1751, 1903, 1860, 1665, 1807, 1822, 1824, 1874, 1831, 1877, + /* 600 */ 1815, 1830, 1890, 1897, 1844, 1847, 1849, 1851, 1854, 1906, + /* 610 */ 1880, 1896, 1856, 1908, 1701, 1861, 1862, 1950, 1910, 1772, + /* 620 */ 1924, 1926, 1929, 1930, 1931, 1932, 1868, 1869, 1921, 1774, + /* 630 */ 1927, 1925, 1973, 1974, 1976, 1977, 1879, 1939, 1699, 1933, + /* 640 */ 1881, 1882, 1885, 1884, 1887, 1819, 1888, 1982, 1953, 1825, + /* 650 */ 1889, 1894, 1699, 1946, 1963, 1912, 1778, 1914, 1995, 1986, + /* 660 */ 1798, 1916, 1917, 1920, 1928, 1922, 1934, 1967, 1936, 1938, + /* 670 */ 1970, 1941, 1999, 1814, 1940, 1915, 1942, 1992, 1998, 1947, + /* 680 */ 1948, 2004, 1951, 1949, 2013, 1952, 1954, 2018, 1958, 1960, + /* 690 */ 2019, 1962, 1956, 1957, 1983, 1985, 1969, 1990, 1971, 2037, + /* 700 */ 1975, 1990, 1990, 2008, 2010, 2009, 2038, 2045, 2047, 2057, + /* 710 */ 2060, 2061, 2073, 2074, 2075, 2076, 2040, 2022, 2070, 2082, + /* 720 */ 2083, 2085, 2099, 2087, 2088, 2090, 2054, 1789, 2092, 1791, + /* 730 */ 2093, 2094, 2095, 2097, 2112, 2100, 2136, 2102, 2089, 2101, + /* 740 */ 2141, 2107, 2096, 2104, 2146, 2113, 2098, 2110, 2150, 2117, + /* 750 */ 2105, 2116, 2156, 2122, 2123, 2159, 2138, 2140, 2143, 2144, + /* 760 */ 2142, 2147, }; -#define YY_REDUCE_COUNT (307) -#define YY_REDUCE_MIN (-442) -#define YY_REDUCE_MAX (2408) +#define YY_REDUCE_COUNT (312) +#define YY_REDUCE_MIN (-396) +#define YY_REDUCE_MAX (2601) static const short yy_reduce_ofst[] = { - /* 0 */ -20, -328, 248, 809, 840, 907, 967, 987, 1096, 655, - /* 10 */ 1116, 1190, 1214, 1281, 1341, 46, -305, 325, 467, 890, - /* 20 */ 1264, 1361, 1424, 1482, 1507, 1580, 1601, 1630, 1651, 1701, - /* 30 */ 1726, 1761, 1784, 1842, 1861, 1922, 1980, 1996, 2059, 2075, - /* 40 */ 2094, 2158, 2175, 2235, 2255, 2329, 2346, 2408, -317, -303, - /* 50 */ 263, 138, 483, 580, 1035, 1208, -238, 361, -367, -348, - /* 60 */ 715, 141, -329, -239, -442, 155, -302, -364, 243, 261, - /* 70 */ -378, -330, -285, -306, -41, -199, 7, 82, 367, 397, - /* 80 */ 475, 567, 568, 782, 55, 786, 791, 309, 64, 805, - /* 90 */ 817, -28, 392, 332, 698, 70, 830, 94, -350, 848, - /* 100 */ 859, 395, 861, 401, 461, 406, 776, 462, -234, -334, - /* 110 */ -334, -297, -344, 117, 237, 278, 450, 471, 524, 569, - /* 120 */ 612, 675, 678, 690, 700, 709, 719, 720, 749, 756, - /* 130 */ 793, 868, -290, -106, -176, 337, 169, -106, 286, 393, - /* 140 */ 546, 184, 201, 460, 565, 608, 618, 22, 76, -269, - /* 150 */ 539, 564, 556, 637, 714, 635, 356, 193, 234, 358, - /* 160 */ 670, 754, 792, 810, 754, 134, 254, 577, 824, 712, - /* 170 */ 814, 939, 829, 923, 923, 953, 930, 951, 943, 942, - /* 180 */ 905, 905, 897, 905, 925, 915, 923, 958, 960, 984, - /* 190 */ 1007, 1006, 1010, 1027, 1072, 1074, 1030, 1036, 1038, 1075, - /* 200 */ 1078, 1089, 1083, 1094, 1097, 1098, 1105, 1106, 1037, 1099, - /* 210 */ 1064, 1100, 1108, 1055, 1117, 1113, 1118, 1119, 1120, 1122, - /* 220 */ 1121, 1125, 1127, 1137, 1110, 1114, 1123, 1126, 1128, 1130, - /* 230 */ 1134, 1135, 1140, 1154, 1158, 1132, 1139, 1129, 1131, 1143, - /* 240 */ 1071, 1077, 1080, 1146, 1124, 1136, 1149, 1180, 1201, 1155, - /* 250 */ 1160, 1166, 1101, 1163, 1175, 1111, 1165, 1179, 1181, 923, - /* 260 */ 1145, 1138, 1150, 1168, 1148, 1152, 1157, 1103, 1151, 1153, - /* 270 */ 905, 1207, 1167, 1242, 1243, 1241, 1244, 1199, 1196, 1213, - /* 280 */ 1219, 1220, 1228, 1237, 1223, 1249, 1250, 1286, 1262, 1303, - /* 290 */ 1266, 1209, 1283, 1280, 1302, 1313, 1312, 1327, 1329, 1265, - /* 300 */ 1260, 1270, 1273, 1310, 1330, 1333, 1323, 1342, + /* 0 */ -140, 137, -82, 358, 444, 578, 814, 841, 1034, 1064, + /* 10 */ 1132, 1240, 1260, 1344, 1368, 1445, 598, -333, 672, 1468, + /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1750, 1770, 1867, + /* 30 */ 1883, 1913, 1943, 2007, 2020, 2071, 2084, 2145, 2196, 2211, + /* 40 */ 2257, 2308, 2325, 2354, 2422, 2441, 2454, 2527, 2540, 2555, + /* 50 */ 2601, 372, -195, -396, -265, 487, 595, 607, 701, 481, + /* 60 */ 761, -362, 191, -365, 136, -279, -55, 256, -372, -167, + /* 70 */ -5, -381, -273, -257, 14, -321, -337, 203, 45, 285, + /* 80 */ 300, 356, 377, 409, 425, 177, 432, 489, 446, 279, + /* 90 */ 521, 573, 125, 620, 321, 622, 396, 687, 436, 257, + /* 100 */ 707, 716, 460, 739, 434, 506, 679, 746, -266, 8, + /* 110 */ -350, -350, 123, -261, 265, 363, 579, 585, 612, 617, + /* 120 */ 710, 754, 769, 774, 777, 781, 806, 807, 813, 820, + /* 130 */ 830, 831, 832, -331, -289, -111, -249, -61, -289, 290, + /* 140 */ 563, -25, -338, 158, 357, -147, 498, 533, -102, 202, + /* 150 */ 324, 29, 378, 502, 593, 597, 808, 236, -367, 552, + /* 160 */ 584, 603, 614, 674, 857, 614, 785, 871, 945, 877, + /* 170 */ 811, 823, 937, 834, 920, 920, 946, 904, 963, 985, + /* 180 */ 926, 919, 882, 882, 890, 882, 917, 911, 920, 949, + /* 190 */ 957, 973, 996, 995, 999, 1002, 1051, 1052, 1010, 1013, + /* 200 */ 1014, 1050, 1055, 1068, 1060, 1074, 1076, 1078, 1079, 1083, + /* 210 */ 1088, 1085, 1017, 1077, 1046, 1080, 1090, 1035, 1087, 1091, + /* 220 */ 1093, 1096, 1099, 1097, 1100, 1094, 1095, 1116, 1084, 1101, + /* 230 */ 1102, 1103, 1107, 1108, 1111, 1112, 1114, 1115, 1122, 1098, + /* 240 */ 1119, 1082, 1113, 1124, 1053, 1106, 1059, 1131, 1104, 1109, + /* 250 */ 1123, 1148, 1118, 1158, 1120, 1127, 1126, 1054, 1125, 1128, + /* 260 */ 1058, 1129, 1139, 1141, 920, 1069, 1073, 1081, 1075, 1086, + /* 270 */ 1133, 1135, 1061, 1130, 1134, 882, 1198, 1144, 1201, 1200, + /* 280 */ 1197, 1199, 1155, 1163, 1180, 1188, 1193, 1206, 1209, 1174, + /* 290 */ 1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237, + /* 300 */ 1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251, + /* 310 */ 1252, 1262, 1281, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 10 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 20 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 30 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 40 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 50 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 60 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 70 */ 1692, 1692, 1692, 1958, 1692, 1692, 1692, 1692, 1692, 1692, - /* 80 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1771, 1692, 1692, - /* 90 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 100 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1769, 1951, 2169, - /* 110 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 120 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 130 */ 1692, 1692, 1692, 2181, 1692, 1692, 1692, 2181, 2181, 2181, - /* 140 */ 1769, 2141, 2141, 1692, 1692, 1692, 1692, 1771, 2014, 1692, - /* 150 */ 1692, 1692, 1692, 1692, 1692, 1886, 1692, 1692, 1692, 1692, - /* 160 */ 1692, 1910, 1692, 1692, 1692, 2004, 1692, 1692, 2206, 2262, - /* 170 */ 1692, 1692, 2209, 1692, 1692, 1692, 1963, 1692, 1839, 2196, - /* 180 */ 2173, 2187, 2246, 2174, 2171, 2190, 1692, 2200, 1692, 1997, - /* 190 */ 1956, 1692, 1956, 1953, 1692, 1692, 1956, 1953, 1953, 1828, - /* 200 */ 1824, 1692, 1822, 1692, 1692, 1692, 1692, 1692, 1692, 1771, - /* 210 */ 1692, 1771, 1692, 1692, 1771, 1692, 1771, 1771, 1771, 1692, - /* 220 */ 1771, 1749, 1749, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 230 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 240 */ 2026, 2012, 1692, 1769, 2008, 2006, 1692, 1769, 1692, 1692, - /* 250 */ 1692, 1692, 2217, 2215, 1692, 2217, 2215, 1692, 1692, 1692, - /* 260 */ 2231, 2227, 2217, 2235, 2233, 2202, 2200, 2265, 2252, 2248, - /* 270 */ 2187, 1692, 1692, 1692, 1692, 1769, 1769, 1692, 2215, 1692, - /* 280 */ 1692, 1692, 1692, 1692, 2215, 1692, 1692, 1769, 1692, 1769, - /* 290 */ 1692, 1692, 1855, 1692, 1692, 1692, 1769, 1724, 1692, 1999, - /* 300 */ 2017, 1981, 1981, 1889, 1889, 1889, 1772, 1697, 1692, 1692, - /* 310 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 320 */ 2230, 2229, 2096, 1692, 2145, 2144, 2143, 2134, 2095, 1851, - /* 330 */ 1692, 2094, 2093, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 340 */ 1692, 1972, 1971, 2087, 1692, 1692, 2088, 2086, 2085, 1692, - /* 350 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 360 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 370 */ 2249, 2253, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 2170, - /* 380 */ 1692, 1692, 1692, 1692, 1692, 2069, 1692, 1692, 1692, 1692, - /* 390 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 400 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 410 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 420 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 430 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 440 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 450 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 460 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 470 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 480 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 490 */ 1692, 1692, 1729, 2074, 1692, 1692, 1692, 1692, 1692, 1692, - /* 500 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 510 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 520 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 530 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 540 */ 1810, 1809, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 550 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 560 */ 1692, 1692, 1692, 1692, 2078, 1692, 1692, 1692, 1692, 1692, - /* 570 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 580 */ 2245, 2203, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 590 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 600 */ 2069, 1692, 2228, 1692, 1692, 2243, 1692, 2247, 1692, 1692, - /* 610 */ 1692, 1692, 1692, 1692, 1692, 2180, 2176, 1692, 1692, 2172, - /* 620 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 2077, 1692, 1692, - /* 630 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 640 */ 1692, 2068, 1692, 2131, 1692, 1692, 1692, 2165, 1692, 1692, - /* 650 */ 2116, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 660 */ 2078, 1692, 2081, 1692, 1692, 1692, 1692, 1692, 1883, 1692, - /* 670 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 680 */ 1692, 1868, 1866, 1865, 1864, 1692, 1861, 1692, 1896, 1692, - /* 690 */ 1692, 1692, 1892, 1891, 1692, 1692, 1692, 1692, 1692, 1692, - /* 700 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1790, - /* 710 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1782, 1692, - /* 720 */ 1781, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 730 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 740 */ 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, 1692, - /* 750 */ 1692, 1692, 1692, + /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 10 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 20 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 30 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 40 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 50 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 60 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 70 */ 1719, 1719, 1719, 1719, 1992, 1719, 1719, 1719, 1719, 1719, + /* 80 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1801, 1719, + /* 90 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 100 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1799, 1985, + /* 110 */ 2206, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 120 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 130 */ 1719, 1719, 1719, 1719, 2218, 1719, 1719, 1719, 2218, 2218, + /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, + /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, + /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, + /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1997, 1719, 1719, + /* 180 */ 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, 1719, 2237, + /* 190 */ 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, 1990, 1987, + /* 200 */ 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, 1719, 1766, + /* 210 */ 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, 1801, 1719, + /* 220 */ 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, 1719, 1719, + /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 240 */ 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, 2042, 2040, + /* 250 */ 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, 2252, 1719, + /* 260 */ 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, 2272, 2270, + /* 270 */ 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, 1719, 1719, + /* 280 */ 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, 1719, 2252, + /* 290 */ 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, 1719, 1719, + /* 300 */ 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, 1923, 1923, + /* 310 */ 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 320 */ 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, 1719, 2182, + /* 330 */ 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, 1719, 1719, + /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, 2124, 1719, + /* 350 */ 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, 1719, 1719, + /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2286, 2290, + /* 380 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, 1719, 1719, + /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, + /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 420 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 430 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 440 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 450 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 460 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 500 */ 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 550 */ 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 570 */ 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, 1719, 1719, + /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 590 */ 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 610 */ 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, 2284, 1719, + /* 620 */ 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, 1719, 1719, + /* 630 */ 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2114, 1719, + /* 640 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 650 */ 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, 2202, 1719, + /* 660 */ 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 670 */ 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, 1719, 1917, + /* 680 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 690 */ 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, 1719, 1719, + /* 700 */ 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, + /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, + /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 740 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 750 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 760 */ 1719, 1719, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1050,6 +1092,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* READ => nothing */ 0, /* WRITE => nothing */ 0, /* NK_DOT => nothing */ + 0, /* WITH => nothing */ 0, /* DNODE => nothing */ 0, /* PORT => nothing */ 0, /* DNODES => nothing */ @@ -1098,9 +1141,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_COLON => nothing */ 0, /* MAX_SPEED => nothing */ 0, /* START => nothing */ - 0, /* WITH => nothing */ 0, /* TIMESTAMP => nothing */ - 277, /* END => ABORT */ + 279, /* END => ABORT */ 0, /* TABLE => nothing */ 0, /* NK_LP => nothing */ 0, /* NK_RP => nothing */ @@ -1114,7 +1156,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_EQ => nothing */ 0, /* USING => nothing */ 0, /* TAGS => nothing */ - 0, /* COMMENT => nothing */ 0, /* BOOL => nothing */ 0, /* TINYINT => nothing */ 0, /* SMALLINT => nothing */ @@ -1132,6 +1173,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* BLOB => nothing */ 0, /* VARBINARY => nothing */ 0, /* DECIMAL => nothing */ + 0, /* COMMENT => nothing */ 0, /* MAX_DELAY => nothing */ 0, /* WATERMARK => nothing */ 0, /* ROLLUP => nothing */ @@ -1195,6 +1237,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* AGGREGATE => nothing */ 0, /* BUFSIZE => nothing */ 0, /* LANGUAGE => nothing */ + 0, /* REPLACE => nothing */ 0, /* STREAM => nothing */ 0, /* INTO => nothing */ 0, /* TRIGGER => nothing */ @@ -1210,6 +1253,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* TRANSACTION => nothing */ 0, /* BALANCE => nothing */ 0, /* VGROUP => nothing */ + 0, /* LEADER => nothing */ 0, /* MERGE => nothing */ 0, /* REDISTRIBUTE => nothing */ 0, /* SPLIT => nothing */ @@ -1280,57 +1324,56 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 277, /* AFTER => ABORT */ - 277, /* ATTACH => ABORT */ - 277, /* BEFORE => ABORT */ - 277, /* BEGIN => ABORT */ - 277, /* BITAND => ABORT */ - 277, /* BITNOT => ABORT */ - 277, /* BITOR => ABORT */ - 277, /* BLOCKS => ABORT */ - 277, /* CHANGE => ABORT */ - 277, /* COMMA => ABORT */ - 277, /* CONCAT => ABORT */ - 277, /* CONFLICT => ABORT */ - 277, /* COPY => ABORT */ - 277, /* DEFERRED => ABORT */ - 277, /* DELIMITERS => ABORT */ - 277, /* DETACH => ABORT */ - 277, /* DIVIDE => ABORT */ - 277, /* DOT => ABORT */ - 277, /* EACH => ABORT */ - 277, /* FAIL => ABORT */ - 277, /* FILE => ABORT */ - 277, /* FOR => ABORT */ - 277, /* GLOB => ABORT */ - 277, /* ID => ABORT */ - 277, /* IMMEDIATE => ABORT */ - 277, /* IMPORT => ABORT */ - 277, /* INITIALLY => ABORT */ - 277, /* INSTEAD => ABORT */ - 277, /* ISNULL => ABORT */ - 277, /* KEY => ABORT */ - 277, /* MODULES => ABORT */ - 277, /* NK_BITNOT => ABORT */ - 277, /* NK_SEMI => ABORT */ - 277, /* NOTNULL => ABORT */ - 277, /* OF => ABORT */ - 277, /* PLUS => ABORT */ - 277, /* PRIVILEGE => ABORT */ - 277, /* RAISE => ABORT */ - 277, /* REPLACE => ABORT */ - 277, /* RESTRICT => ABORT */ - 277, /* ROW => ABORT */ - 277, /* SEMI => ABORT */ - 277, /* STAR => ABORT */ - 277, /* STATEMENT => ABORT */ - 277, /* STRICT => ABORT */ - 277, /* STRING => ABORT */ - 277, /* TIMES => ABORT */ - 277, /* VALUES => ABORT */ - 277, /* VARIABLE => ABORT */ - 277, /* VIEW => ABORT */ - 277, /* WAL => ABORT */ + 279, /* AFTER => ABORT */ + 279, /* ATTACH => ABORT */ + 279, /* BEFORE => ABORT */ + 279, /* BEGIN => ABORT */ + 279, /* BITAND => ABORT */ + 279, /* BITNOT => ABORT */ + 279, /* BITOR => ABORT */ + 279, /* BLOCKS => ABORT */ + 279, /* CHANGE => ABORT */ + 279, /* COMMA => ABORT */ + 279, /* CONCAT => ABORT */ + 279, /* CONFLICT => ABORT */ + 279, /* COPY => ABORT */ + 279, /* DEFERRED => ABORT */ + 279, /* DELIMITERS => ABORT */ + 279, /* DETACH => ABORT */ + 279, /* DIVIDE => ABORT */ + 279, /* DOT => ABORT */ + 279, /* EACH => ABORT */ + 279, /* FAIL => ABORT */ + 279, /* FILE => ABORT */ + 279, /* FOR => ABORT */ + 279, /* GLOB => ABORT */ + 279, /* ID => ABORT */ + 279, /* IMMEDIATE => ABORT */ + 279, /* IMPORT => ABORT */ + 279, /* INITIALLY => ABORT */ + 279, /* INSTEAD => ABORT */ + 279, /* ISNULL => ABORT */ + 279, /* KEY => ABORT */ + 279, /* MODULES => ABORT */ + 279, /* NK_BITNOT => ABORT */ + 279, /* NK_SEMI => ABORT */ + 279, /* NOTNULL => ABORT */ + 279, /* OF => ABORT */ + 279, /* PLUS => ABORT */ + 279, /* PRIVILEGE => ABORT */ + 279, /* RAISE => ABORT */ + 279, /* RESTRICT => ABORT */ + 279, /* ROW => ABORT */ + 279, /* SEMI => ABORT */ + 279, /* STAR => ABORT */ + 279, /* STATEMENT => ABORT */ + 279, /* STRICT => ABORT */ + 279, /* STRING => ABORT */ + 279, /* TIMES => ABORT */ + 279, /* VALUES => ABORT */ + 279, /* VARIABLE => ABORT */ + 279, /* VIEW => ABORT */ + 279, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1466,55 +1509,55 @@ static const char *const yyTokenName[] = { /* 45 */ "READ", /* 46 */ "WRITE", /* 47 */ "NK_DOT", - /* 48 */ "DNODE", - /* 49 */ "PORT", - /* 50 */ "DNODES", - /* 51 */ "NK_IPTOKEN", - /* 52 */ "FORCE", - /* 53 */ "LOCAL", - /* 54 */ "QNODE", - /* 55 */ "BNODE", - /* 56 */ "SNODE", - /* 57 */ "MNODE", - /* 58 */ "DATABASE", - /* 59 */ "USE", - /* 60 */ "FLUSH", - /* 61 */ "TRIM", - /* 62 */ "COMPACT", - /* 63 */ "IF", - /* 64 */ "NOT", - /* 65 */ "EXISTS", - /* 66 */ "BUFFER", - /* 67 */ "CACHEMODEL", - /* 68 */ "CACHESIZE", - /* 69 */ "COMP", - /* 70 */ "DURATION", - /* 71 */ "NK_VARIABLE", - /* 72 */ "MAXROWS", - /* 73 */ "MINROWS", - /* 74 */ "KEEP", - /* 75 */ "PAGES", - /* 76 */ "PAGESIZE", - /* 77 */ "TSDB_PAGESIZE", - /* 78 */ "PRECISION", - /* 79 */ "REPLICA", - /* 80 */ "VGROUPS", - /* 81 */ "SINGLE_STABLE", - /* 82 */ "RETENTIONS", - /* 83 */ "SCHEMALESS", - /* 84 */ "WAL_LEVEL", - /* 85 */ "WAL_FSYNC_PERIOD", - /* 86 */ "WAL_RETENTION_PERIOD", - /* 87 */ "WAL_RETENTION_SIZE", - /* 88 */ "WAL_ROLL_PERIOD", - /* 89 */ "WAL_SEGMENT_SIZE", - /* 90 */ "STT_TRIGGER", - /* 91 */ "TABLE_PREFIX", - /* 92 */ "TABLE_SUFFIX", - /* 93 */ "NK_COLON", - /* 94 */ "MAX_SPEED", - /* 95 */ "START", - /* 96 */ "WITH", + /* 48 */ "WITH", + /* 49 */ "DNODE", + /* 50 */ "PORT", + /* 51 */ "DNODES", + /* 52 */ "NK_IPTOKEN", + /* 53 */ "FORCE", + /* 54 */ "LOCAL", + /* 55 */ "QNODE", + /* 56 */ "BNODE", + /* 57 */ "SNODE", + /* 58 */ "MNODE", + /* 59 */ "DATABASE", + /* 60 */ "USE", + /* 61 */ "FLUSH", + /* 62 */ "TRIM", + /* 63 */ "COMPACT", + /* 64 */ "IF", + /* 65 */ "NOT", + /* 66 */ "EXISTS", + /* 67 */ "BUFFER", + /* 68 */ "CACHEMODEL", + /* 69 */ "CACHESIZE", + /* 70 */ "COMP", + /* 71 */ "DURATION", + /* 72 */ "NK_VARIABLE", + /* 73 */ "MAXROWS", + /* 74 */ "MINROWS", + /* 75 */ "KEEP", + /* 76 */ "PAGES", + /* 77 */ "PAGESIZE", + /* 78 */ "TSDB_PAGESIZE", + /* 79 */ "PRECISION", + /* 80 */ "REPLICA", + /* 81 */ "VGROUPS", + /* 82 */ "SINGLE_STABLE", + /* 83 */ "RETENTIONS", + /* 84 */ "SCHEMALESS", + /* 85 */ "WAL_LEVEL", + /* 86 */ "WAL_FSYNC_PERIOD", + /* 87 */ "WAL_RETENTION_PERIOD", + /* 88 */ "WAL_RETENTION_SIZE", + /* 89 */ "WAL_ROLL_PERIOD", + /* 90 */ "WAL_SEGMENT_SIZE", + /* 91 */ "STT_TRIGGER", + /* 92 */ "TABLE_PREFIX", + /* 93 */ "TABLE_SUFFIX", + /* 94 */ "NK_COLON", + /* 95 */ "MAX_SPEED", + /* 96 */ "START", /* 97 */ "TIMESTAMP", /* 98 */ "END", /* 99 */ "TABLE", @@ -1530,24 +1573,24 @@ static const char *const yyTokenName[] = { /* 109 */ "NK_EQ", /* 110 */ "USING", /* 111 */ "TAGS", - /* 112 */ "COMMENT", - /* 113 */ "BOOL", - /* 114 */ "TINYINT", - /* 115 */ "SMALLINT", - /* 116 */ "INT", - /* 117 */ "INTEGER", - /* 118 */ "BIGINT", - /* 119 */ "FLOAT", - /* 120 */ "DOUBLE", - /* 121 */ "BINARY", - /* 122 */ "NCHAR", - /* 123 */ "UNSIGNED", - /* 124 */ "JSON", - /* 125 */ "VARCHAR", - /* 126 */ "MEDIUMBLOB", - /* 127 */ "BLOB", - /* 128 */ "VARBINARY", - /* 129 */ "DECIMAL", + /* 112 */ "BOOL", + /* 113 */ "TINYINT", + /* 114 */ "SMALLINT", + /* 115 */ "INT", + /* 116 */ "INTEGER", + /* 117 */ "BIGINT", + /* 118 */ "FLOAT", + /* 119 */ "DOUBLE", + /* 120 */ "BINARY", + /* 121 */ "NCHAR", + /* 122 */ "UNSIGNED", + /* 123 */ "JSON", + /* 124 */ "VARCHAR", + /* 125 */ "MEDIUMBLOB", + /* 126 */ "BLOB", + /* 127 */ "VARBINARY", + /* 128 */ "DECIMAL", + /* 129 */ "COMMENT", /* 130 */ "MAX_DELAY", /* 131 */ "WATERMARK", /* 132 */ "ROLLUP", @@ -1611,286 +1654,289 @@ static const char *const yyTokenName[] = { /* 190 */ "AGGREGATE", /* 191 */ "BUFSIZE", /* 192 */ "LANGUAGE", - /* 193 */ "STREAM", - /* 194 */ "INTO", - /* 195 */ "TRIGGER", - /* 196 */ "AT_ONCE", - /* 197 */ "WINDOW_CLOSE", - /* 198 */ "IGNORE", - /* 199 */ "EXPIRED", - /* 200 */ "FILL_HISTORY", - /* 201 */ "UPDATE", - /* 202 */ "SUBTABLE", - /* 203 */ "KILL", - /* 204 */ "CONNECTION", - /* 205 */ "TRANSACTION", - /* 206 */ "BALANCE", - /* 207 */ "VGROUP", - /* 208 */ "MERGE", - /* 209 */ "REDISTRIBUTE", - /* 210 */ "SPLIT", - /* 211 */ "DELETE", - /* 212 */ "INSERT", - /* 213 */ "NULL", - /* 214 */ "NK_QUESTION", - /* 215 */ "NK_ARROW", - /* 216 */ "ROWTS", - /* 217 */ "QSTART", - /* 218 */ "QEND", - /* 219 */ "QDURATION", - /* 220 */ "WSTART", - /* 221 */ "WEND", - /* 222 */ "WDURATION", - /* 223 */ "IROWTS", - /* 224 */ "ISFILLED", - /* 225 */ "CAST", - /* 226 */ "NOW", - /* 227 */ "TODAY", - /* 228 */ "TIMEZONE", - /* 229 */ "CLIENT_VERSION", - /* 230 */ "SERVER_VERSION", - /* 231 */ "SERVER_STATUS", - /* 232 */ "CURRENT_USER", - /* 233 */ "CASE", - /* 234 */ "WHEN", - /* 235 */ "THEN", - /* 236 */ "ELSE", - /* 237 */ "BETWEEN", - /* 238 */ "IS", - /* 239 */ "NK_LT", - /* 240 */ "NK_GT", - /* 241 */ "NK_LE", - /* 242 */ "NK_GE", - /* 243 */ "NK_NE", - /* 244 */ "MATCH", - /* 245 */ "NMATCH", - /* 246 */ "CONTAINS", - /* 247 */ "IN", - /* 248 */ "JOIN", - /* 249 */ "INNER", - /* 250 */ "SELECT", - /* 251 */ "DISTINCT", - /* 252 */ "WHERE", - /* 253 */ "PARTITION", - /* 254 */ "BY", - /* 255 */ "SESSION", - /* 256 */ "STATE_WINDOW", - /* 257 */ "EVENT_WINDOW", - /* 258 */ "SLIDING", - /* 259 */ "FILL", - /* 260 */ "VALUE", - /* 261 */ "VALUE_F", - /* 262 */ "NONE", - /* 263 */ "PREV", - /* 264 */ "NULL_F", - /* 265 */ "LINEAR", - /* 266 */ "NEXT", - /* 267 */ "HAVING", - /* 268 */ "RANGE", - /* 269 */ "EVERY", - /* 270 */ "ORDER", - /* 271 */ "SLIMIT", - /* 272 */ "SOFFSET", - /* 273 */ "LIMIT", - /* 274 */ "OFFSET", - /* 275 */ "ASC", - /* 276 */ "NULLS", - /* 277 */ "ABORT", - /* 278 */ "AFTER", - /* 279 */ "ATTACH", - /* 280 */ "BEFORE", - /* 281 */ "BEGIN", - /* 282 */ "BITAND", - /* 283 */ "BITNOT", - /* 284 */ "BITOR", - /* 285 */ "BLOCKS", - /* 286 */ "CHANGE", - /* 287 */ "COMMA", - /* 288 */ "CONCAT", - /* 289 */ "CONFLICT", - /* 290 */ "COPY", - /* 291 */ "DEFERRED", - /* 292 */ "DELIMITERS", - /* 293 */ "DETACH", - /* 294 */ "DIVIDE", - /* 295 */ "DOT", - /* 296 */ "EACH", - /* 297 */ "FAIL", - /* 298 */ "FILE", - /* 299 */ "FOR", - /* 300 */ "GLOB", - /* 301 */ "ID", - /* 302 */ "IMMEDIATE", - /* 303 */ "IMPORT", - /* 304 */ "INITIALLY", - /* 305 */ "INSTEAD", - /* 306 */ "ISNULL", - /* 307 */ "KEY", - /* 308 */ "MODULES", - /* 309 */ "NK_BITNOT", - /* 310 */ "NK_SEMI", - /* 311 */ "NOTNULL", - /* 312 */ "OF", - /* 313 */ "PLUS", - /* 314 */ "PRIVILEGE", - /* 315 */ "RAISE", - /* 316 */ "REPLACE", - /* 317 */ "RESTRICT", - /* 318 */ "ROW", - /* 319 */ "SEMI", - /* 320 */ "STAR", - /* 321 */ "STATEMENT", - /* 322 */ "STRICT", - /* 323 */ "STRING", - /* 324 */ "TIMES", - /* 325 */ "VALUES", - /* 326 */ "VARIABLE", - /* 327 */ "VIEW", - /* 328 */ "WAL", - /* 329 */ "cmd", - /* 330 */ "account_options", - /* 331 */ "alter_account_options", - /* 332 */ "literal", - /* 333 */ "alter_account_option", - /* 334 */ "user_name", - /* 335 */ "sysinfo_opt", - /* 336 */ "privileges", - /* 337 */ "priv_level", - /* 338 */ "priv_type_list", - /* 339 */ "priv_type", - /* 340 */ "db_name", - /* 341 */ "topic_name", - /* 342 */ "dnode_endpoint", - /* 343 */ "force_opt", - /* 344 */ "not_exists_opt", - /* 345 */ "db_options", - /* 346 */ "exists_opt", - /* 347 */ "alter_db_options", - /* 348 */ "speed_opt", - /* 349 */ "start_opt", - /* 350 */ "end_opt", - /* 351 */ "integer_list", - /* 352 */ "variable_list", - /* 353 */ "retention_list", - /* 354 */ "alter_db_option", - /* 355 */ "retention", - /* 356 */ "full_table_name", - /* 357 */ "column_def_list", - /* 358 */ "tags_def_opt", - /* 359 */ "table_options", - /* 360 */ "multi_create_clause", - /* 361 */ "tags_def", - /* 362 */ "multi_drop_clause", - /* 363 */ "alter_table_clause", - /* 364 */ "alter_table_options", - /* 365 */ "column_name", - /* 366 */ "type_name", - /* 367 */ "signed_literal", - /* 368 */ "create_subtable_clause", - /* 369 */ "specific_cols_opt", - /* 370 */ "expression_list", - /* 371 */ "drop_table_clause", - /* 372 */ "col_name_list", - /* 373 */ "table_name", - /* 374 */ "column_def", - /* 375 */ "duration_list", - /* 376 */ "rollup_func_list", - /* 377 */ "alter_table_option", - /* 378 */ "duration_literal", - /* 379 */ "rollup_func_name", - /* 380 */ "function_name", - /* 381 */ "col_name", - /* 382 */ "db_name_cond_opt", - /* 383 */ "like_pattern_opt", - /* 384 */ "table_name_cond", - /* 385 */ "from_db_opt", - /* 386 */ "tag_list_opt", - /* 387 */ "tag_item", - /* 388 */ "column_alias", - /* 389 */ "full_index_name", - /* 390 */ "index_options", - /* 391 */ "index_name", - /* 392 */ "func_list", - /* 393 */ "sliding_opt", - /* 394 */ "sma_stream_opt", - /* 395 */ "func", - /* 396 */ "sma_func_name", - /* 397 */ "query_or_subquery", - /* 398 */ "cgroup_name", - /* 399 */ "analyze_opt", - /* 400 */ "explain_options", - /* 401 */ "insert_query", - /* 402 */ "agg_func_opt", - /* 403 */ "bufsize_opt", - /* 404 */ "language_opt", - /* 405 */ "stream_name", - /* 406 */ "stream_options", - /* 407 */ "col_list_opt", - /* 408 */ "tag_def_or_ref_opt", - /* 409 */ "subtable_opt", - /* 410 */ "expression", - /* 411 */ "dnode_list", - /* 412 */ "where_clause_opt", - /* 413 */ "signed", - /* 414 */ "literal_func", - /* 415 */ "literal_list", - /* 416 */ "table_alias", - /* 417 */ "expr_or_subquery", - /* 418 */ "pseudo_column", - /* 419 */ "column_reference", - /* 420 */ "function_expression", - /* 421 */ "case_when_expression", - /* 422 */ "star_func", - /* 423 */ "star_func_para_list", - /* 424 */ "noarg_func", - /* 425 */ "other_para_list", - /* 426 */ "star_func_para", - /* 427 */ "when_then_list", - /* 428 */ "case_when_else_opt", - /* 429 */ "common_expression", - /* 430 */ "when_then_expr", - /* 431 */ "predicate", - /* 432 */ "compare_op", - /* 433 */ "in_op", - /* 434 */ "in_predicate_value", - /* 435 */ "boolean_value_expression", - /* 436 */ "boolean_primary", - /* 437 */ "from_clause_opt", - /* 438 */ "table_reference_list", - /* 439 */ "table_reference", - /* 440 */ "table_primary", - /* 441 */ "joined_table", - /* 442 */ "alias_opt", - /* 443 */ "subquery", - /* 444 */ "parenthesized_joined_table", - /* 445 */ "join_type", - /* 446 */ "search_condition", - /* 447 */ "query_specification", - /* 448 */ "set_quantifier_opt", - /* 449 */ "select_list", - /* 450 */ "partition_by_clause_opt", - /* 451 */ "range_opt", - /* 452 */ "every_opt", - /* 453 */ "fill_opt", - /* 454 */ "twindow_clause_opt", - /* 455 */ "group_by_clause_opt", - /* 456 */ "having_clause_opt", - /* 457 */ "select_item", - /* 458 */ "partition_list", - /* 459 */ "partition_item", - /* 460 */ "fill_mode", - /* 461 */ "group_by_list", - /* 462 */ "query_expression", - /* 463 */ "query_simple", - /* 464 */ "order_by_clause_opt", - /* 465 */ "slimit_clause_opt", - /* 466 */ "limit_clause_opt", - /* 467 */ "union_query_expression", - /* 468 */ "query_simple_or_subquery", - /* 469 */ "sort_specification_list", - /* 470 */ "sort_specification", - /* 471 */ "ordering_specification_opt", - /* 472 */ "null_ordering_opt", + /* 193 */ "REPLACE", + /* 194 */ "STREAM", + /* 195 */ "INTO", + /* 196 */ "TRIGGER", + /* 197 */ "AT_ONCE", + /* 198 */ "WINDOW_CLOSE", + /* 199 */ "IGNORE", + /* 200 */ "EXPIRED", + /* 201 */ "FILL_HISTORY", + /* 202 */ "UPDATE", + /* 203 */ "SUBTABLE", + /* 204 */ "KILL", + /* 205 */ "CONNECTION", + /* 206 */ "TRANSACTION", + /* 207 */ "BALANCE", + /* 208 */ "VGROUP", + /* 209 */ "LEADER", + /* 210 */ "MERGE", + /* 211 */ "REDISTRIBUTE", + /* 212 */ "SPLIT", + /* 213 */ "DELETE", + /* 214 */ "INSERT", + /* 215 */ "NULL", + /* 216 */ "NK_QUESTION", + /* 217 */ "NK_ARROW", + /* 218 */ "ROWTS", + /* 219 */ "QSTART", + /* 220 */ "QEND", + /* 221 */ "QDURATION", + /* 222 */ "WSTART", + /* 223 */ "WEND", + /* 224 */ "WDURATION", + /* 225 */ "IROWTS", + /* 226 */ "ISFILLED", + /* 227 */ "CAST", + /* 228 */ "NOW", + /* 229 */ "TODAY", + /* 230 */ "TIMEZONE", + /* 231 */ "CLIENT_VERSION", + /* 232 */ "SERVER_VERSION", + /* 233 */ "SERVER_STATUS", + /* 234 */ "CURRENT_USER", + /* 235 */ "CASE", + /* 236 */ "WHEN", + /* 237 */ "THEN", + /* 238 */ "ELSE", + /* 239 */ "BETWEEN", + /* 240 */ "IS", + /* 241 */ "NK_LT", + /* 242 */ "NK_GT", + /* 243 */ "NK_LE", + /* 244 */ "NK_GE", + /* 245 */ "NK_NE", + /* 246 */ "MATCH", + /* 247 */ "NMATCH", + /* 248 */ "CONTAINS", + /* 249 */ "IN", + /* 250 */ "JOIN", + /* 251 */ "INNER", + /* 252 */ "SELECT", + /* 253 */ "DISTINCT", + /* 254 */ "WHERE", + /* 255 */ "PARTITION", + /* 256 */ "BY", + /* 257 */ "SESSION", + /* 258 */ "STATE_WINDOW", + /* 259 */ "EVENT_WINDOW", + /* 260 */ "SLIDING", + /* 261 */ "FILL", + /* 262 */ "VALUE", + /* 263 */ "VALUE_F", + /* 264 */ "NONE", + /* 265 */ "PREV", + /* 266 */ "NULL_F", + /* 267 */ "LINEAR", + /* 268 */ "NEXT", + /* 269 */ "HAVING", + /* 270 */ "RANGE", + /* 271 */ "EVERY", + /* 272 */ "ORDER", + /* 273 */ "SLIMIT", + /* 274 */ "SOFFSET", + /* 275 */ "LIMIT", + /* 276 */ "OFFSET", + /* 277 */ "ASC", + /* 278 */ "NULLS", + /* 279 */ "ABORT", + /* 280 */ "AFTER", + /* 281 */ "ATTACH", + /* 282 */ "BEFORE", + /* 283 */ "BEGIN", + /* 284 */ "BITAND", + /* 285 */ "BITNOT", + /* 286 */ "BITOR", + /* 287 */ "BLOCKS", + /* 288 */ "CHANGE", + /* 289 */ "COMMA", + /* 290 */ "CONCAT", + /* 291 */ "CONFLICT", + /* 292 */ "COPY", + /* 293 */ "DEFERRED", + /* 294 */ "DELIMITERS", + /* 295 */ "DETACH", + /* 296 */ "DIVIDE", + /* 297 */ "DOT", + /* 298 */ "EACH", + /* 299 */ "FAIL", + /* 300 */ "FILE", + /* 301 */ "FOR", + /* 302 */ "GLOB", + /* 303 */ "ID", + /* 304 */ "IMMEDIATE", + /* 305 */ "IMPORT", + /* 306 */ "INITIALLY", + /* 307 */ "INSTEAD", + /* 308 */ "ISNULL", + /* 309 */ "KEY", + /* 310 */ "MODULES", + /* 311 */ "NK_BITNOT", + /* 312 */ "NK_SEMI", + /* 313 */ "NOTNULL", + /* 314 */ "OF", + /* 315 */ "PLUS", + /* 316 */ "PRIVILEGE", + /* 317 */ "RAISE", + /* 318 */ "RESTRICT", + /* 319 */ "ROW", + /* 320 */ "SEMI", + /* 321 */ "STAR", + /* 322 */ "STATEMENT", + /* 323 */ "STRICT", + /* 324 */ "STRING", + /* 325 */ "TIMES", + /* 326 */ "VALUES", + /* 327 */ "VARIABLE", + /* 328 */ "VIEW", + /* 329 */ "WAL", + /* 330 */ "cmd", + /* 331 */ "account_options", + /* 332 */ "alter_account_options", + /* 333 */ "literal", + /* 334 */ "alter_account_option", + /* 335 */ "user_name", + /* 336 */ "sysinfo_opt", + /* 337 */ "privileges", + /* 338 */ "priv_level", + /* 339 */ "with_opt", + /* 340 */ "priv_type_list", + /* 341 */ "priv_type", + /* 342 */ "db_name", + /* 343 */ "table_name", + /* 344 */ "topic_name", + /* 345 */ "search_condition", + /* 346 */ "dnode_endpoint", + /* 347 */ "force_opt", + /* 348 */ "not_exists_opt", + /* 349 */ "db_options", + /* 350 */ "exists_opt", + /* 351 */ "alter_db_options", + /* 352 */ "speed_opt", + /* 353 */ "start_opt", + /* 354 */ "end_opt", + /* 355 */ "integer_list", + /* 356 */ "variable_list", + /* 357 */ "retention_list", + /* 358 */ "alter_db_option", + /* 359 */ "retention", + /* 360 */ "full_table_name", + /* 361 */ "column_def_list", + /* 362 */ "tags_def_opt", + /* 363 */ "table_options", + /* 364 */ "multi_create_clause", + /* 365 */ "tags_def", + /* 366 */ "multi_drop_clause", + /* 367 */ "alter_table_clause", + /* 368 */ "alter_table_options", + /* 369 */ "column_name", + /* 370 */ "type_name", + /* 371 */ "signed_literal", + /* 372 */ "create_subtable_clause", + /* 373 */ "specific_cols_opt", + /* 374 */ "expression_list", + /* 375 */ "drop_table_clause", + /* 376 */ "col_name_list", + /* 377 */ "column_def", + /* 378 */ "duration_list", + /* 379 */ "rollup_func_list", + /* 380 */ "alter_table_option", + /* 381 */ "duration_literal", + /* 382 */ "rollup_func_name", + /* 383 */ "function_name", + /* 384 */ "col_name", + /* 385 */ "db_name_cond_opt", + /* 386 */ "like_pattern_opt", + /* 387 */ "table_name_cond", + /* 388 */ "from_db_opt", + /* 389 */ "tag_list_opt", + /* 390 */ "tag_item", + /* 391 */ "column_alias", + /* 392 */ "full_index_name", + /* 393 */ "index_options", + /* 394 */ "index_name", + /* 395 */ "func_list", + /* 396 */ "sliding_opt", + /* 397 */ "sma_stream_opt", + /* 398 */ "func", + /* 399 */ "sma_func_name", + /* 400 */ "query_or_subquery", + /* 401 */ "cgroup_name", + /* 402 */ "analyze_opt", + /* 403 */ "explain_options", + /* 404 */ "insert_query", + /* 405 */ "or_replace_opt", + /* 406 */ "agg_func_opt", + /* 407 */ "bufsize_opt", + /* 408 */ "language_opt", + /* 409 */ "stream_name", + /* 410 */ "stream_options", + /* 411 */ "col_list_opt", + /* 412 */ "tag_def_or_ref_opt", + /* 413 */ "subtable_opt", + /* 414 */ "expression", + /* 415 */ "dnode_list", + /* 416 */ "where_clause_opt", + /* 417 */ "signed", + /* 418 */ "literal_func", + /* 419 */ "literal_list", + /* 420 */ "table_alias", + /* 421 */ "expr_or_subquery", + /* 422 */ "pseudo_column", + /* 423 */ "column_reference", + /* 424 */ "function_expression", + /* 425 */ "case_when_expression", + /* 426 */ "star_func", + /* 427 */ "star_func_para_list", + /* 428 */ "noarg_func", + /* 429 */ "other_para_list", + /* 430 */ "star_func_para", + /* 431 */ "when_then_list", + /* 432 */ "case_when_else_opt", + /* 433 */ "common_expression", + /* 434 */ "when_then_expr", + /* 435 */ "predicate", + /* 436 */ "compare_op", + /* 437 */ "in_op", + /* 438 */ "in_predicate_value", + /* 439 */ "boolean_value_expression", + /* 440 */ "boolean_primary", + /* 441 */ "from_clause_opt", + /* 442 */ "table_reference_list", + /* 443 */ "table_reference", + /* 444 */ "table_primary", + /* 445 */ "joined_table", + /* 446 */ "alias_opt", + /* 447 */ "subquery", + /* 448 */ "parenthesized_joined_table", + /* 449 */ "join_type", + /* 450 */ "query_specification", + /* 451 */ "set_quantifier_opt", + /* 452 */ "select_list", + /* 453 */ "partition_by_clause_opt", + /* 454 */ "range_opt", + /* 455 */ "every_opt", + /* 456 */ "fill_opt", + /* 457 */ "twindow_clause_opt", + /* 458 */ "group_by_clause_opt", + /* 459 */ "having_clause_opt", + /* 460 */ "select_item", + /* 461 */ "partition_list", + /* 462 */ "partition_item", + /* 463 */ "fill_mode", + /* 464 */ "group_by_list", + /* 465 */ "query_expression", + /* 466 */ "query_simple", + /* 467 */ "order_by_clause_opt", + /* 468 */ "slimit_clause_opt", + /* 469 */ "limit_clause_opt", + /* 470 */ "union_query_expression", + /* 471 */ "query_simple_or_subquery", + /* 472 */ "sort_specification_list", + /* 473 */ "sort_specification", + /* 474 */ "ordering_specification_opt", + /* 475 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1929,8 +1975,8 @@ static const char *const yyRuleName[] = { /* 28 */ "cmd ::= DROP USER user_name", /* 29 */ "sysinfo_opt ::=", /* 30 */ "sysinfo_opt ::= SYSINFO NK_INTEGER", - /* 31 */ "cmd ::= GRANT privileges ON priv_level TO user_name", - /* 32 */ "cmd ::= REVOKE privileges ON priv_level FROM user_name", + /* 31 */ "cmd ::= GRANT privileges ON priv_level with_opt TO user_name", + /* 32 */ "cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name", /* 33 */ "privileges ::= ALL", /* 34 */ "privileges ::= priv_type_list", /* 35 */ "privileges ::= SUBSCRIBE", @@ -1940,537 +1986,547 @@ static const char *const yyRuleName[] = { /* 39 */ "priv_type ::= WRITE", /* 40 */ "priv_level ::= NK_STAR NK_DOT NK_STAR", /* 41 */ "priv_level ::= db_name NK_DOT NK_STAR", - /* 42 */ "priv_level ::= topic_name", - /* 43 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 44 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", - /* 45 */ "cmd ::= DROP DNODE NK_INTEGER force_opt", - /* 46 */ "cmd ::= DROP DNODE dnode_endpoint force_opt", - /* 47 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", - /* 48 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", - /* 49 */ "cmd ::= ALTER ALL DNODES NK_STRING", - /* 50 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", - /* 51 */ "dnode_endpoint ::= NK_STRING", - /* 52 */ "dnode_endpoint ::= NK_ID", - /* 53 */ "dnode_endpoint ::= NK_IPTOKEN", - /* 54 */ "force_opt ::=", - /* 55 */ "force_opt ::= FORCE", - /* 56 */ "cmd ::= ALTER LOCAL NK_STRING", - /* 57 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", - /* 58 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 59 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 60 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", - /* 61 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", - /* 62 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", - /* 63 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", - /* 64 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", - /* 65 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", - /* 66 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 67 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 68 */ "cmd ::= USE db_name", - /* 69 */ "cmd ::= ALTER DATABASE db_name alter_db_options", - /* 70 */ "cmd ::= FLUSH DATABASE db_name", - /* 71 */ "cmd ::= TRIM DATABASE db_name speed_opt", - /* 72 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", - /* 73 */ "not_exists_opt ::= IF NOT EXISTS", - /* 74 */ "not_exists_opt ::=", - /* 75 */ "exists_opt ::= IF EXISTS", - /* 76 */ "exists_opt ::=", - /* 77 */ "db_options ::=", - /* 78 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 79 */ "db_options ::= db_options CACHEMODEL NK_STRING", - /* 80 */ "db_options ::= db_options CACHESIZE NK_INTEGER", - /* 81 */ "db_options ::= db_options COMP NK_INTEGER", - /* 82 */ "db_options ::= db_options DURATION NK_INTEGER", - /* 83 */ "db_options ::= db_options DURATION NK_VARIABLE", - /* 84 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 85 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 86 */ "db_options ::= db_options KEEP integer_list", - /* 87 */ "db_options ::= db_options KEEP variable_list", - /* 88 */ "db_options ::= db_options PAGES NK_INTEGER", - /* 89 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 90 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", - /* 91 */ "db_options ::= db_options PRECISION NK_STRING", - /* 92 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 93 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 94 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 95 */ "db_options ::= db_options RETENTIONS retention_list", - /* 96 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 97 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 98 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 100 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 102 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 103 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 104 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 105 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 106 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 107 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", - /* 108 */ "alter_db_options ::= alter_db_option", - /* 109 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 110 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 111 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 112 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 113 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 114 */ "alter_db_option ::= KEEP integer_list", - /* 115 */ "alter_db_option ::= KEEP variable_list", - /* 116 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 117 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 118 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 119 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", - /* 120 */ "integer_list ::= NK_INTEGER", - /* 121 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 122 */ "variable_list ::= NK_VARIABLE", - /* 123 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 124 */ "retention_list ::= retention", - /* 125 */ "retention_list ::= retention_list NK_COMMA retention", - /* 126 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 127 */ "speed_opt ::=", - /* 128 */ "speed_opt ::= MAX_SPEED NK_INTEGER", - /* 129 */ "start_opt ::=", - /* 130 */ "start_opt ::= START WITH NK_INTEGER", - /* 131 */ "start_opt ::= START WITH NK_STRING", - /* 132 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", - /* 133 */ "end_opt ::=", - /* 134 */ "end_opt ::= END WITH NK_INTEGER", - /* 135 */ "end_opt ::= END WITH NK_STRING", - /* 136 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", - /* 137 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 138 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 139 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 140 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 141 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 142 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 143 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 144 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 145 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 146 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 147 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 148 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 149 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 150 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 151 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 152 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 153 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 154 */ "multi_create_clause ::= create_subtable_clause", - /* 155 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 156 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 157 */ "multi_drop_clause ::= drop_table_clause", - /* 158 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", - /* 159 */ "drop_table_clause ::= exists_opt full_table_name", - /* 160 */ "specific_cols_opt ::=", - /* 161 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 162 */ "full_table_name ::= table_name", - /* 163 */ "full_table_name ::= db_name NK_DOT table_name", - /* 164 */ "column_def_list ::= column_def", - /* 165 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 166 */ "column_def ::= column_name type_name", - /* 167 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 168 */ "type_name ::= BOOL", - /* 169 */ "type_name ::= TINYINT", - /* 170 */ "type_name ::= SMALLINT", - /* 171 */ "type_name ::= INT", - /* 172 */ "type_name ::= INTEGER", - /* 173 */ "type_name ::= BIGINT", - /* 174 */ "type_name ::= FLOAT", - /* 175 */ "type_name ::= DOUBLE", - /* 176 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 177 */ "type_name ::= TIMESTAMP", - /* 178 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 179 */ "type_name ::= TINYINT UNSIGNED", - /* 180 */ "type_name ::= SMALLINT UNSIGNED", - /* 181 */ "type_name ::= INT UNSIGNED", - /* 182 */ "type_name ::= BIGINT UNSIGNED", - /* 183 */ "type_name ::= JSON", - /* 184 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 185 */ "type_name ::= MEDIUMBLOB", - /* 186 */ "type_name ::= BLOB", - /* 187 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 188 */ "type_name ::= DECIMAL", - /* 189 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 190 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 191 */ "tags_def_opt ::=", - /* 192 */ "tags_def_opt ::= tags_def", - /* 193 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 194 */ "table_options ::=", - /* 195 */ "table_options ::= table_options COMMENT NK_STRING", - /* 196 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 197 */ "table_options ::= table_options WATERMARK duration_list", - /* 198 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 199 */ "table_options ::= table_options TTL NK_INTEGER", - /* 200 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 201 */ "table_options ::= table_options DELETE_MARK duration_list", - /* 202 */ "alter_table_options ::= alter_table_option", - /* 203 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 204 */ "alter_table_option ::= COMMENT NK_STRING", - /* 205 */ "alter_table_option ::= TTL NK_INTEGER", - /* 206 */ "duration_list ::= duration_literal", - /* 207 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 208 */ "rollup_func_list ::= rollup_func_name", - /* 209 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 210 */ "rollup_func_name ::= function_name", - /* 211 */ "rollup_func_name ::= FIRST", - /* 212 */ "rollup_func_name ::= LAST", - /* 213 */ "col_name_list ::= col_name", - /* 214 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 215 */ "col_name ::= column_name", - /* 216 */ "cmd ::= SHOW DNODES", - /* 217 */ "cmd ::= SHOW USERS", - /* 218 */ "cmd ::= SHOW USER PRIVILEGES", - /* 219 */ "cmd ::= SHOW DATABASES", - /* 220 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 221 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 222 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 223 */ "cmd ::= SHOW MNODES", - /* 224 */ "cmd ::= SHOW QNODES", - /* 225 */ "cmd ::= SHOW FUNCTIONS", - /* 226 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 227 */ "cmd ::= SHOW STREAMS", - /* 228 */ "cmd ::= SHOW ACCOUNTS", - /* 229 */ "cmd ::= SHOW APPS", - /* 230 */ "cmd ::= SHOW CONNECTIONS", - /* 231 */ "cmd ::= SHOW LICENCES", - /* 232 */ "cmd ::= SHOW GRANTS", - /* 233 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 234 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 235 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 236 */ "cmd ::= SHOW QUERIES", - /* 237 */ "cmd ::= SHOW SCORES", - /* 238 */ "cmd ::= SHOW TOPICS", - /* 239 */ "cmd ::= SHOW VARIABLES", - /* 240 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 241 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 242 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 243 */ "cmd ::= SHOW BNODES", - /* 244 */ "cmd ::= SHOW SNODES", - /* 245 */ "cmd ::= SHOW CLUSTER", - /* 246 */ "cmd ::= SHOW TRANSACTIONS", - /* 247 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 248 */ "cmd ::= SHOW CONSUMERS", - /* 249 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 250 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 251 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 252 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 253 */ "cmd ::= SHOW VNODES NK_STRING", - /* 254 */ "cmd ::= SHOW db_name_cond_opt ALIVE", - /* 255 */ "cmd ::= SHOW CLUSTER ALIVE", - /* 256 */ "db_name_cond_opt ::=", - /* 257 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 258 */ "like_pattern_opt ::=", - /* 259 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 260 */ "table_name_cond ::= table_name", - /* 261 */ "from_db_opt ::=", - /* 262 */ "from_db_opt ::= FROM db_name", - /* 263 */ "tag_list_opt ::=", - /* 264 */ "tag_list_opt ::= tag_item", - /* 265 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 266 */ "tag_item ::= TBNAME", - /* 267 */ "tag_item ::= QTAGS", - /* 268 */ "tag_item ::= column_name", - /* 269 */ "tag_item ::= column_name column_alias", - /* 270 */ "tag_item ::= column_name AS column_alias", - /* 271 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 272 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", - /* 273 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 274 */ "full_index_name ::= index_name", - /* 275 */ "full_index_name ::= db_name NK_DOT index_name", - /* 276 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 277 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 278 */ "func_list ::= func", - /* 279 */ "func_list ::= func_list NK_COMMA func", - /* 280 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 281 */ "sma_func_name ::= function_name", - /* 282 */ "sma_func_name ::= COUNT", - /* 283 */ "sma_func_name ::= FIRST", - /* 284 */ "sma_func_name ::= LAST", - /* 285 */ "sma_func_name ::= LAST_ROW", - /* 286 */ "sma_stream_opt ::=", - /* 287 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 288 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 289 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 290 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 291 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 292 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 293 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 294 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 295 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 296 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 297 */ "cmd ::= DESC full_table_name", - /* 298 */ "cmd ::= DESCRIBE full_table_name", - /* 299 */ "cmd ::= RESET QUERY CACHE", - /* 300 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 301 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", - /* 302 */ "analyze_opt ::=", - /* 303 */ "analyze_opt ::= ANALYZE", - /* 304 */ "explain_options ::=", - /* 305 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 306 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 307 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", - /* 308 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 309 */ "agg_func_opt ::=", - /* 310 */ "agg_func_opt ::= AGGREGATE", - /* 311 */ "bufsize_opt ::=", - /* 312 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 313 */ "language_opt ::=", - /* 314 */ "language_opt ::= LANGUAGE NK_STRING", - /* 315 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", - /* 316 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 317 */ "col_list_opt ::=", - /* 318 */ "col_list_opt ::= NK_LP col_name_list NK_RP", - /* 319 */ "tag_def_or_ref_opt ::=", - /* 320 */ "tag_def_or_ref_opt ::= tags_def", - /* 321 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", - /* 322 */ "stream_options ::=", - /* 323 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 324 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 325 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 326 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 327 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 328 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 329 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 330 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 331 */ "subtable_opt ::=", - /* 332 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 333 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 334 */ "cmd ::= KILL QUERY NK_STRING", - /* 335 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 336 */ "cmd ::= BALANCE VGROUP", - /* 337 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 338 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 339 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 340 */ "dnode_list ::= DNODE NK_INTEGER", - /* 341 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 342 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 343 */ "cmd ::= query_or_subquery", - /* 344 */ "cmd ::= insert_query", - /* 345 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 346 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", - /* 347 */ "literal ::= NK_INTEGER", - /* 348 */ "literal ::= NK_FLOAT", - /* 349 */ "literal ::= NK_STRING", - /* 350 */ "literal ::= NK_BOOL", - /* 351 */ "literal ::= TIMESTAMP NK_STRING", - /* 352 */ "literal ::= duration_literal", - /* 353 */ "literal ::= NULL", - /* 354 */ "literal ::= NK_QUESTION", - /* 355 */ "duration_literal ::= NK_VARIABLE", - /* 356 */ "signed ::= NK_INTEGER", - /* 357 */ "signed ::= NK_PLUS NK_INTEGER", - /* 358 */ "signed ::= NK_MINUS NK_INTEGER", - /* 359 */ "signed ::= NK_FLOAT", - /* 360 */ "signed ::= NK_PLUS NK_FLOAT", - /* 361 */ "signed ::= NK_MINUS NK_FLOAT", - /* 362 */ "signed_literal ::= signed", - /* 363 */ "signed_literal ::= NK_STRING", - /* 364 */ "signed_literal ::= NK_BOOL", - /* 365 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 366 */ "signed_literal ::= duration_literal", - /* 367 */ "signed_literal ::= NULL", - /* 368 */ "signed_literal ::= literal_func", - /* 369 */ "signed_literal ::= NK_QUESTION", - /* 370 */ "literal_list ::= signed_literal", - /* 371 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 372 */ "db_name ::= NK_ID", - /* 373 */ "table_name ::= NK_ID", - /* 374 */ "column_name ::= NK_ID", - /* 375 */ "function_name ::= NK_ID", - /* 376 */ "table_alias ::= NK_ID", - /* 377 */ "column_alias ::= NK_ID", - /* 378 */ "user_name ::= NK_ID", - /* 379 */ "topic_name ::= NK_ID", - /* 380 */ "stream_name ::= NK_ID", - /* 381 */ "cgroup_name ::= NK_ID", - /* 382 */ "index_name ::= NK_ID", - /* 383 */ "expr_or_subquery ::= expression", - /* 384 */ "expression ::= literal", - /* 385 */ "expression ::= pseudo_column", - /* 386 */ "expression ::= column_reference", - /* 387 */ "expression ::= function_expression", - /* 388 */ "expression ::= case_when_expression", - /* 389 */ "expression ::= NK_LP expression NK_RP", - /* 390 */ "expression ::= NK_PLUS expr_or_subquery", - /* 391 */ "expression ::= NK_MINUS expr_or_subquery", - /* 392 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 393 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 394 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 395 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 396 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 397 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 398 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 399 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 400 */ "expression_list ::= expr_or_subquery", - /* 401 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 402 */ "column_reference ::= column_name", - /* 403 */ "column_reference ::= table_name NK_DOT column_name", - /* 404 */ "pseudo_column ::= ROWTS", - /* 405 */ "pseudo_column ::= TBNAME", - /* 406 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 407 */ "pseudo_column ::= QSTART", - /* 408 */ "pseudo_column ::= QEND", - /* 409 */ "pseudo_column ::= QDURATION", - /* 410 */ "pseudo_column ::= WSTART", - /* 411 */ "pseudo_column ::= WEND", - /* 412 */ "pseudo_column ::= WDURATION", - /* 413 */ "pseudo_column ::= IROWTS", - /* 414 */ "pseudo_column ::= ISFILLED", - /* 415 */ "pseudo_column ::= QTAGS", - /* 416 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 417 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 418 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 419 */ "function_expression ::= literal_func", - /* 420 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 421 */ "literal_func ::= NOW", - /* 422 */ "noarg_func ::= NOW", - /* 423 */ "noarg_func ::= TODAY", - /* 424 */ "noarg_func ::= TIMEZONE", - /* 425 */ "noarg_func ::= DATABASE", - /* 426 */ "noarg_func ::= CLIENT_VERSION", - /* 427 */ "noarg_func ::= SERVER_VERSION", - /* 428 */ "noarg_func ::= SERVER_STATUS", - /* 429 */ "noarg_func ::= CURRENT_USER", - /* 430 */ "noarg_func ::= USER", - /* 431 */ "star_func ::= COUNT", - /* 432 */ "star_func ::= FIRST", - /* 433 */ "star_func ::= LAST", - /* 434 */ "star_func ::= LAST_ROW", - /* 435 */ "star_func_para_list ::= NK_STAR", - /* 436 */ "star_func_para_list ::= other_para_list", - /* 437 */ "other_para_list ::= star_func_para", - /* 438 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 439 */ "star_func_para ::= expr_or_subquery", - /* 440 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 441 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 442 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 443 */ "when_then_list ::= when_then_expr", - /* 444 */ "when_then_list ::= when_then_list when_then_expr", - /* 445 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 446 */ "case_when_else_opt ::=", - /* 447 */ "case_when_else_opt ::= ELSE common_expression", - /* 448 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 449 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 450 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 451 */ "predicate ::= expr_or_subquery IS NULL", - /* 452 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 453 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 454 */ "compare_op ::= NK_LT", - /* 455 */ "compare_op ::= NK_GT", - /* 456 */ "compare_op ::= NK_LE", - /* 457 */ "compare_op ::= NK_GE", - /* 458 */ "compare_op ::= NK_NE", - /* 459 */ "compare_op ::= NK_EQ", - /* 460 */ "compare_op ::= LIKE", - /* 461 */ "compare_op ::= NOT LIKE", - /* 462 */ "compare_op ::= MATCH", - /* 463 */ "compare_op ::= NMATCH", - /* 464 */ "compare_op ::= CONTAINS", - /* 465 */ "in_op ::= IN", - /* 466 */ "in_op ::= NOT IN", - /* 467 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 468 */ "boolean_value_expression ::= boolean_primary", - /* 469 */ "boolean_value_expression ::= NOT boolean_primary", - /* 470 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 471 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 472 */ "boolean_primary ::= predicate", - /* 473 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 474 */ "common_expression ::= expr_or_subquery", - /* 475 */ "common_expression ::= boolean_value_expression", - /* 476 */ "from_clause_opt ::=", - /* 477 */ "from_clause_opt ::= FROM table_reference_list", - /* 478 */ "table_reference_list ::= table_reference", - /* 479 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 480 */ "table_reference ::= table_primary", - /* 481 */ "table_reference ::= joined_table", - /* 482 */ "table_primary ::= table_name alias_opt", - /* 483 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 484 */ "table_primary ::= subquery alias_opt", - /* 485 */ "table_primary ::= parenthesized_joined_table", - /* 486 */ "alias_opt ::=", - /* 487 */ "alias_opt ::= table_alias", - /* 488 */ "alias_opt ::= AS table_alias", - /* 489 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 490 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 491 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 492 */ "join_type ::=", - /* 493 */ "join_type ::= INNER", - /* 494 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 495 */ "set_quantifier_opt ::=", - /* 496 */ "set_quantifier_opt ::= DISTINCT", - /* 497 */ "set_quantifier_opt ::= ALL", - /* 498 */ "select_list ::= select_item", - /* 499 */ "select_list ::= select_list NK_COMMA select_item", - /* 500 */ "select_item ::= NK_STAR", - /* 501 */ "select_item ::= common_expression", - /* 502 */ "select_item ::= common_expression column_alias", - /* 503 */ "select_item ::= common_expression AS column_alias", - /* 504 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 505 */ "where_clause_opt ::=", - /* 506 */ "where_clause_opt ::= WHERE search_condition", - /* 507 */ "partition_by_clause_opt ::=", - /* 508 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 509 */ "partition_list ::= partition_item", - /* 510 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 511 */ "partition_item ::= expr_or_subquery", - /* 512 */ "partition_item ::= expr_or_subquery column_alias", - /* 513 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 514 */ "twindow_clause_opt ::=", - /* 515 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 516 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 517 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 518 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 519 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 520 */ "sliding_opt ::=", - /* 521 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 522 */ "fill_opt ::=", - /* 523 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 524 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 525 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", - /* 526 */ "fill_mode ::= NONE", - /* 527 */ "fill_mode ::= PREV", - /* 528 */ "fill_mode ::= NULL", - /* 529 */ "fill_mode ::= NULL_F", - /* 530 */ "fill_mode ::= LINEAR", - /* 531 */ "fill_mode ::= NEXT", - /* 532 */ "group_by_clause_opt ::=", - /* 533 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 534 */ "group_by_list ::= expr_or_subquery", - /* 535 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 536 */ "having_clause_opt ::=", - /* 537 */ "having_clause_opt ::= HAVING search_condition", - /* 538 */ "range_opt ::=", - /* 539 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 540 */ "every_opt ::=", - /* 541 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 542 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 543 */ "query_simple ::= query_specification", - /* 544 */ "query_simple ::= union_query_expression", - /* 545 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 546 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 547 */ "query_simple_or_subquery ::= query_simple", - /* 548 */ "query_simple_or_subquery ::= subquery", - /* 549 */ "query_or_subquery ::= query_expression", - /* 550 */ "query_or_subquery ::= subquery", - /* 551 */ "order_by_clause_opt ::=", - /* 552 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 553 */ "slimit_clause_opt ::=", - /* 554 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 555 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 556 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 557 */ "limit_clause_opt ::=", - /* 558 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 559 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 560 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 561 */ "subquery ::= NK_LP query_expression NK_RP", - /* 562 */ "subquery ::= NK_LP subquery NK_RP", - /* 563 */ "search_condition ::= common_expression", - /* 564 */ "sort_specification_list ::= sort_specification", - /* 565 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 566 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 567 */ "ordering_specification_opt ::=", - /* 568 */ "ordering_specification_opt ::= ASC", - /* 569 */ "ordering_specification_opt ::= DESC", - /* 570 */ "null_ordering_opt ::=", - /* 571 */ "null_ordering_opt ::= NULLS FIRST", - /* 572 */ "null_ordering_opt ::= NULLS LAST", + /* 42 */ "priv_level ::= db_name NK_DOT table_name", + /* 43 */ "priv_level ::= topic_name", + /* 44 */ "with_opt ::=", + /* 45 */ "with_opt ::= WITH search_condition", + /* 46 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 47 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", + /* 48 */ "cmd ::= DROP DNODE NK_INTEGER force_opt", + /* 49 */ "cmd ::= DROP DNODE dnode_endpoint force_opt", + /* 50 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 51 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 52 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 53 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 54 */ "dnode_endpoint ::= NK_STRING", + /* 55 */ "dnode_endpoint ::= NK_ID", + /* 56 */ "dnode_endpoint ::= NK_IPTOKEN", + /* 57 */ "force_opt ::=", + /* 58 */ "force_opt ::= FORCE", + /* 59 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 60 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 61 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 62 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 63 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", + /* 64 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", + /* 65 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", + /* 66 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", + /* 67 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", + /* 68 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", + /* 69 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 70 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 71 */ "cmd ::= USE db_name", + /* 72 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 73 */ "cmd ::= FLUSH DATABASE db_name", + /* 74 */ "cmd ::= TRIM DATABASE db_name speed_opt", + /* 75 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", + /* 76 */ "not_exists_opt ::= IF NOT EXISTS", + /* 77 */ "not_exists_opt ::=", + /* 78 */ "exists_opt ::= IF EXISTS", + /* 79 */ "exists_opt ::=", + /* 80 */ "db_options ::=", + /* 81 */ "db_options ::= db_options BUFFER NK_INTEGER", + /* 82 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 83 */ "db_options ::= db_options CACHESIZE NK_INTEGER", + /* 84 */ "db_options ::= db_options COMP NK_INTEGER", + /* 85 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 86 */ "db_options ::= db_options DURATION NK_VARIABLE", + /* 87 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 88 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 89 */ "db_options ::= db_options KEEP integer_list", + /* 90 */ "db_options ::= db_options KEEP variable_list", + /* 91 */ "db_options ::= db_options PAGES NK_INTEGER", + /* 92 */ "db_options ::= db_options PAGESIZE NK_INTEGER", + /* 93 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", + /* 94 */ "db_options ::= db_options PRECISION NK_STRING", + /* 95 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 96 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 97 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 98 */ "db_options ::= db_options RETENTIONS retention_list", + /* 99 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 100 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 101 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 102 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 103 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 104 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 105 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 106 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 107 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 108 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 109 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", + /* 110 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 111 */ "alter_db_options ::= alter_db_option", + /* 112 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 113 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 114 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 115 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 116 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 117 */ "alter_db_option ::= KEEP integer_list", + /* 118 */ "alter_db_option ::= KEEP variable_list", + /* 119 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 120 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 121 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 122 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 123 */ "alter_db_option ::= MINROWS NK_INTEGER", + /* 124 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER", + /* 125 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 126 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER", + /* 127 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 128 */ "integer_list ::= NK_INTEGER", + /* 129 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 130 */ "variable_list ::= NK_VARIABLE", + /* 131 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 132 */ "retention_list ::= retention", + /* 133 */ "retention_list ::= retention_list NK_COMMA retention", + /* 134 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 135 */ "speed_opt ::=", + /* 136 */ "speed_opt ::= MAX_SPEED NK_INTEGER", + /* 137 */ "start_opt ::=", + /* 138 */ "start_opt ::= START WITH NK_INTEGER", + /* 139 */ "start_opt ::= START WITH NK_STRING", + /* 140 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", + /* 141 */ "end_opt ::=", + /* 142 */ "end_opt ::= END WITH NK_INTEGER", + /* 143 */ "end_opt ::= END WITH NK_STRING", + /* 144 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", + /* 145 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 146 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 147 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 148 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 149 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 150 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 151 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 152 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 153 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 154 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 155 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 156 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 157 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 158 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 159 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 160 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 161 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 162 */ "multi_create_clause ::= create_subtable_clause", + /* 163 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 164 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 165 */ "multi_drop_clause ::= drop_table_clause", + /* 166 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", + /* 167 */ "drop_table_clause ::= exists_opt full_table_name", + /* 168 */ "specific_cols_opt ::=", + /* 169 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 170 */ "full_table_name ::= table_name", + /* 171 */ "full_table_name ::= db_name NK_DOT table_name", + /* 172 */ "column_def_list ::= column_def", + /* 173 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 174 */ "column_def ::= column_name type_name", + /* 175 */ "type_name ::= BOOL", + /* 176 */ "type_name ::= TINYINT", + /* 177 */ "type_name ::= SMALLINT", + /* 178 */ "type_name ::= INT", + /* 179 */ "type_name ::= INTEGER", + /* 180 */ "type_name ::= BIGINT", + /* 181 */ "type_name ::= FLOAT", + /* 182 */ "type_name ::= DOUBLE", + /* 183 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 184 */ "type_name ::= TIMESTAMP", + /* 185 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 186 */ "type_name ::= TINYINT UNSIGNED", + /* 187 */ "type_name ::= SMALLINT UNSIGNED", + /* 188 */ "type_name ::= INT UNSIGNED", + /* 189 */ "type_name ::= BIGINT UNSIGNED", + /* 190 */ "type_name ::= JSON", + /* 191 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 192 */ "type_name ::= MEDIUMBLOB", + /* 193 */ "type_name ::= BLOB", + /* 194 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 195 */ "type_name ::= DECIMAL", + /* 196 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 197 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 198 */ "tags_def_opt ::=", + /* 199 */ "tags_def_opt ::= tags_def", + /* 200 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 201 */ "table_options ::=", + /* 202 */ "table_options ::= table_options COMMENT NK_STRING", + /* 203 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 204 */ "table_options ::= table_options WATERMARK duration_list", + /* 205 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 206 */ "table_options ::= table_options TTL NK_INTEGER", + /* 207 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 208 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 209 */ "alter_table_options ::= alter_table_option", + /* 210 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 211 */ "alter_table_option ::= COMMENT NK_STRING", + /* 212 */ "alter_table_option ::= TTL NK_INTEGER", + /* 213 */ "duration_list ::= duration_literal", + /* 214 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 215 */ "rollup_func_list ::= rollup_func_name", + /* 216 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 217 */ "rollup_func_name ::= function_name", + /* 218 */ "rollup_func_name ::= FIRST", + /* 219 */ "rollup_func_name ::= LAST", + /* 220 */ "col_name_list ::= col_name", + /* 221 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 222 */ "col_name ::= column_name", + /* 223 */ "cmd ::= SHOW DNODES", + /* 224 */ "cmd ::= SHOW USERS", + /* 225 */ "cmd ::= SHOW USER PRIVILEGES", + /* 226 */ "cmd ::= SHOW DATABASES", + /* 227 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 228 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 229 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 230 */ "cmd ::= SHOW MNODES", + /* 231 */ "cmd ::= SHOW QNODES", + /* 232 */ "cmd ::= SHOW FUNCTIONS", + /* 233 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 234 */ "cmd ::= SHOW STREAMS", + /* 235 */ "cmd ::= SHOW ACCOUNTS", + /* 236 */ "cmd ::= SHOW APPS", + /* 237 */ "cmd ::= SHOW CONNECTIONS", + /* 238 */ "cmd ::= SHOW LICENCES", + /* 239 */ "cmd ::= SHOW GRANTS", + /* 240 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 241 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 242 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 243 */ "cmd ::= SHOW QUERIES", + /* 244 */ "cmd ::= SHOW SCORES", + /* 245 */ "cmd ::= SHOW TOPICS", + /* 246 */ "cmd ::= SHOW VARIABLES", + /* 247 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 248 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 249 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 250 */ "cmd ::= SHOW BNODES", + /* 251 */ "cmd ::= SHOW SNODES", + /* 252 */ "cmd ::= SHOW CLUSTER", + /* 253 */ "cmd ::= SHOW TRANSACTIONS", + /* 254 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 255 */ "cmd ::= SHOW CONSUMERS", + /* 256 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 257 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 258 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 259 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 260 */ "cmd ::= SHOW VNODES NK_STRING", + /* 261 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 262 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 263 */ "db_name_cond_opt ::=", + /* 264 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 265 */ "like_pattern_opt ::=", + /* 266 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 267 */ "table_name_cond ::= table_name", + /* 268 */ "from_db_opt ::=", + /* 269 */ "from_db_opt ::= FROM db_name", + /* 270 */ "tag_list_opt ::=", + /* 271 */ "tag_list_opt ::= tag_item", + /* 272 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 273 */ "tag_item ::= TBNAME", + /* 274 */ "tag_item ::= QTAGS", + /* 275 */ "tag_item ::= column_name", + /* 276 */ "tag_item ::= column_name column_alias", + /* 277 */ "tag_item ::= column_name AS column_alias", + /* 278 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 279 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 280 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 281 */ "full_index_name ::= index_name", + /* 282 */ "full_index_name ::= db_name NK_DOT index_name", + /* 283 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 284 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 285 */ "func_list ::= func", + /* 286 */ "func_list ::= func_list NK_COMMA func", + /* 287 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 288 */ "sma_func_name ::= function_name", + /* 289 */ "sma_func_name ::= COUNT", + /* 290 */ "sma_func_name ::= FIRST", + /* 291 */ "sma_func_name ::= LAST", + /* 292 */ "sma_func_name ::= LAST_ROW", + /* 293 */ "sma_stream_opt ::=", + /* 294 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 295 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 296 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 297 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 298 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 299 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 302 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 303 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 304 */ "cmd ::= DESC full_table_name", + /* 305 */ "cmd ::= DESCRIBE full_table_name", + /* 306 */ "cmd ::= RESET QUERY CACHE", + /* 307 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 308 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 309 */ "analyze_opt ::=", + /* 310 */ "analyze_opt ::= ANALYZE", + /* 311 */ "explain_options ::=", + /* 312 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 313 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 314 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", + /* 315 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 316 */ "agg_func_opt ::=", + /* 317 */ "agg_func_opt ::= AGGREGATE", + /* 318 */ "bufsize_opt ::=", + /* 319 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 320 */ "language_opt ::=", + /* 321 */ "language_opt ::= LANGUAGE NK_STRING", + /* 322 */ "or_replace_opt ::=", + /* 323 */ "or_replace_opt ::= OR REPLACE", + /* 324 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 325 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 326 */ "col_list_opt ::=", + /* 327 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 328 */ "tag_def_or_ref_opt ::=", + /* 329 */ "tag_def_or_ref_opt ::= tags_def", + /* 330 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 331 */ "stream_options ::=", + /* 332 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 333 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 334 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 335 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 336 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 337 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 338 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 339 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 340 */ "subtable_opt ::=", + /* 341 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 342 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 343 */ "cmd ::= KILL QUERY NK_STRING", + /* 344 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 345 */ "cmd ::= BALANCE VGROUP", + /* 346 */ "cmd ::= BALANCE VGROUP LEADER", + /* 347 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 348 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 349 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 350 */ "dnode_list ::= DNODE NK_INTEGER", + /* 351 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 352 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 353 */ "cmd ::= query_or_subquery", + /* 354 */ "cmd ::= insert_query", + /* 355 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 356 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 357 */ "literal ::= NK_INTEGER", + /* 358 */ "literal ::= NK_FLOAT", + /* 359 */ "literal ::= NK_STRING", + /* 360 */ "literal ::= NK_BOOL", + /* 361 */ "literal ::= TIMESTAMP NK_STRING", + /* 362 */ "literal ::= duration_literal", + /* 363 */ "literal ::= NULL", + /* 364 */ "literal ::= NK_QUESTION", + /* 365 */ "duration_literal ::= NK_VARIABLE", + /* 366 */ "signed ::= NK_INTEGER", + /* 367 */ "signed ::= NK_PLUS NK_INTEGER", + /* 368 */ "signed ::= NK_MINUS NK_INTEGER", + /* 369 */ "signed ::= NK_FLOAT", + /* 370 */ "signed ::= NK_PLUS NK_FLOAT", + /* 371 */ "signed ::= NK_MINUS NK_FLOAT", + /* 372 */ "signed_literal ::= signed", + /* 373 */ "signed_literal ::= NK_STRING", + /* 374 */ "signed_literal ::= NK_BOOL", + /* 375 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 376 */ "signed_literal ::= duration_literal", + /* 377 */ "signed_literal ::= NULL", + /* 378 */ "signed_literal ::= literal_func", + /* 379 */ "signed_literal ::= NK_QUESTION", + /* 380 */ "literal_list ::= signed_literal", + /* 381 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 382 */ "db_name ::= NK_ID", + /* 383 */ "table_name ::= NK_ID", + /* 384 */ "column_name ::= NK_ID", + /* 385 */ "function_name ::= NK_ID", + /* 386 */ "table_alias ::= NK_ID", + /* 387 */ "column_alias ::= NK_ID", + /* 388 */ "user_name ::= NK_ID", + /* 389 */ "topic_name ::= NK_ID", + /* 390 */ "stream_name ::= NK_ID", + /* 391 */ "cgroup_name ::= NK_ID", + /* 392 */ "index_name ::= NK_ID", + /* 393 */ "expr_or_subquery ::= expression", + /* 394 */ "expression ::= literal", + /* 395 */ "expression ::= pseudo_column", + /* 396 */ "expression ::= column_reference", + /* 397 */ "expression ::= function_expression", + /* 398 */ "expression ::= case_when_expression", + /* 399 */ "expression ::= NK_LP expression NK_RP", + /* 400 */ "expression ::= NK_PLUS expr_or_subquery", + /* 401 */ "expression ::= NK_MINUS expr_or_subquery", + /* 402 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 403 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 404 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 405 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 406 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 407 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 408 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 409 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 410 */ "expression_list ::= expr_or_subquery", + /* 411 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 412 */ "column_reference ::= column_name", + /* 413 */ "column_reference ::= table_name NK_DOT column_name", + /* 414 */ "pseudo_column ::= ROWTS", + /* 415 */ "pseudo_column ::= TBNAME", + /* 416 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 417 */ "pseudo_column ::= QSTART", + /* 418 */ "pseudo_column ::= QEND", + /* 419 */ "pseudo_column ::= QDURATION", + /* 420 */ "pseudo_column ::= WSTART", + /* 421 */ "pseudo_column ::= WEND", + /* 422 */ "pseudo_column ::= WDURATION", + /* 423 */ "pseudo_column ::= IROWTS", + /* 424 */ "pseudo_column ::= ISFILLED", + /* 425 */ "pseudo_column ::= QTAGS", + /* 426 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 427 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 428 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 429 */ "function_expression ::= literal_func", + /* 430 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 431 */ "literal_func ::= NOW", + /* 432 */ "noarg_func ::= NOW", + /* 433 */ "noarg_func ::= TODAY", + /* 434 */ "noarg_func ::= TIMEZONE", + /* 435 */ "noarg_func ::= DATABASE", + /* 436 */ "noarg_func ::= CLIENT_VERSION", + /* 437 */ "noarg_func ::= SERVER_VERSION", + /* 438 */ "noarg_func ::= SERVER_STATUS", + /* 439 */ "noarg_func ::= CURRENT_USER", + /* 440 */ "noarg_func ::= USER", + /* 441 */ "star_func ::= COUNT", + /* 442 */ "star_func ::= FIRST", + /* 443 */ "star_func ::= LAST", + /* 444 */ "star_func ::= LAST_ROW", + /* 445 */ "star_func_para_list ::= NK_STAR", + /* 446 */ "star_func_para_list ::= other_para_list", + /* 447 */ "other_para_list ::= star_func_para", + /* 448 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 449 */ "star_func_para ::= expr_or_subquery", + /* 450 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 451 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 452 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 453 */ "when_then_list ::= when_then_expr", + /* 454 */ "when_then_list ::= when_then_list when_then_expr", + /* 455 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 456 */ "case_when_else_opt ::=", + /* 457 */ "case_when_else_opt ::= ELSE common_expression", + /* 458 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 459 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 460 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 461 */ "predicate ::= expr_or_subquery IS NULL", + /* 462 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 463 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 464 */ "compare_op ::= NK_LT", + /* 465 */ "compare_op ::= NK_GT", + /* 466 */ "compare_op ::= NK_LE", + /* 467 */ "compare_op ::= NK_GE", + /* 468 */ "compare_op ::= NK_NE", + /* 469 */ "compare_op ::= NK_EQ", + /* 470 */ "compare_op ::= LIKE", + /* 471 */ "compare_op ::= NOT LIKE", + /* 472 */ "compare_op ::= MATCH", + /* 473 */ "compare_op ::= NMATCH", + /* 474 */ "compare_op ::= CONTAINS", + /* 475 */ "in_op ::= IN", + /* 476 */ "in_op ::= NOT IN", + /* 477 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 478 */ "boolean_value_expression ::= boolean_primary", + /* 479 */ "boolean_value_expression ::= NOT boolean_primary", + /* 480 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 481 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 482 */ "boolean_primary ::= predicate", + /* 483 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 484 */ "common_expression ::= expr_or_subquery", + /* 485 */ "common_expression ::= boolean_value_expression", + /* 486 */ "from_clause_opt ::=", + /* 487 */ "from_clause_opt ::= FROM table_reference_list", + /* 488 */ "table_reference_list ::= table_reference", + /* 489 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 490 */ "table_reference ::= table_primary", + /* 491 */ "table_reference ::= joined_table", + /* 492 */ "table_primary ::= table_name alias_opt", + /* 493 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 494 */ "table_primary ::= subquery alias_opt", + /* 495 */ "table_primary ::= parenthesized_joined_table", + /* 496 */ "alias_opt ::=", + /* 497 */ "alias_opt ::= table_alias", + /* 498 */ "alias_opt ::= AS table_alias", + /* 499 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 500 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 501 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 502 */ "join_type ::=", + /* 503 */ "join_type ::= INNER", + /* 504 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 505 */ "set_quantifier_opt ::=", + /* 506 */ "set_quantifier_opt ::= DISTINCT", + /* 507 */ "set_quantifier_opt ::= ALL", + /* 508 */ "select_list ::= select_item", + /* 509 */ "select_list ::= select_list NK_COMMA select_item", + /* 510 */ "select_item ::= NK_STAR", + /* 511 */ "select_item ::= common_expression", + /* 512 */ "select_item ::= common_expression column_alias", + /* 513 */ "select_item ::= common_expression AS column_alias", + /* 514 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 515 */ "where_clause_opt ::=", + /* 516 */ "where_clause_opt ::= WHERE search_condition", + /* 517 */ "partition_by_clause_opt ::=", + /* 518 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 519 */ "partition_list ::= partition_item", + /* 520 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 521 */ "partition_item ::= expr_or_subquery", + /* 522 */ "partition_item ::= expr_or_subquery column_alias", + /* 523 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 524 */ "twindow_clause_opt ::=", + /* 525 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 526 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 527 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 528 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 529 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 530 */ "sliding_opt ::=", + /* 531 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 532 */ "fill_opt ::=", + /* 533 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", + /* 536 */ "fill_mode ::= NONE", + /* 537 */ "fill_mode ::= PREV", + /* 538 */ "fill_mode ::= NULL", + /* 539 */ "fill_mode ::= NULL_F", + /* 540 */ "fill_mode ::= LINEAR", + /* 541 */ "fill_mode ::= NEXT", + /* 542 */ "group_by_clause_opt ::=", + /* 543 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 544 */ "group_by_list ::= expr_or_subquery", + /* 545 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 546 */ "having_clause_opt ::=", + /* 547 */ "having_clause_opt ::= HAVING search_condition", + /* 548 */ "range_opt ::=", + /* 549 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 550 */ "every_opt ::=", + /* 551 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 552 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 553 */ "query_simple ::= query_specification", + /* 554 */ "query_simple ::= union_query_expression", + /* 555 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 556 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 557 */ "query_simple_or_subquery ::= query_simple", + /* 558 */ "query_simple_or_subquery ::= subquery", + /* 559 */ "query_or_subquery ::= query_expression", + /* 560 */ "query_or_subquery ::= subquery", + /* 561 */ "order_by_clause_opt ::=", + /* 562 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 563 */ "slimit_clause_opt ::=", + /* 564 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 565 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 566 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 567 */ "limit_clause_opt ::=", + /* 568 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 569 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 570 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 571 */ "subquery ::= NK_LP query_expression NK_RP", + /* 572 */ "subquery ::= NK_LP subquery NK_RP", + /* 573 */ "search_condition ::= common_expression", + /* 574 */ "sort_specification_list ::= sort_specification", + /* 575 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 576 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 577 */ "ordering_specification_opt ::=", + /* 578 */ "ordering_specification_opt ::= ASC", + /* 579 */ "ordering_specification_opt ::= DESC", + /* 580 */ "null_ordering_opt ::=", + /* 581 */ "null_ordering_opt ::= NULLS FIRST", + /* 582 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2597,202 +2653,208 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 329: /* cmd */ - case 332: /* literal */ - case 345: /* db_options */ - case 347: /* alter_db_options */ - case 349: /* start_opt */ - case 350: /* end_opt */ - case 355: /* retention */ - case 356: /* full_table_name */ - case 359: /* table_options */ - case 363: /* alter_table_clause */ - case 364: /* alter_table_options */ - case 367: /* signed_literal */ - case 368: /* create_subtable_clause */ - case 371: /* drop_table_clause */ - case 374: /* column_def */ - case 378: /* duration_literal */ - case 379: /* rollup_func_name */ - case 381: /* col_name */ - case 382: /* db_name_cond_opt */ - case 383: /* like_pattern_opt */ - case 384: /* table_name_cond */ - case 385: /* from_db_opt */ - case 387: /* tag_item */ - case 389: /* full_index_name */ - case 390: /* index_options */ - case 393: /* sliding_opt */ - case 394: /* sma_stream_opt */ - case 395: /* func */ - case 397: /* query_or_subquery */ - case 400: /* explain_options */ - case 401: /* insert_query */ - case 406: /* stream_options */ - case 409: /* subtable_opt */ - case 410: /* expression */ - case 412: /* where_clause_opt */ - case 413: /* signed */ - case 414: /* literal_func */ - case 417: /* expr_or_subquery */ - case 418: /* pseudo_column */ - case 419: /* column_reference */ - case 420: /* function_expression */ - case 421: /* case_when_expression */ - case 426: /* star_func_para */ - case 428: /* case_when_else_opt */ - case 429: /* common_expression */ - case 430: /* when_then_expr */ - case 431: /* predicate */ - case 434: /* in_predicate_value */ - case 435: /* boolean_value_expression */ - case 436: /* boolean_primary */ - case 437: /* from_clause_opt */ - case 438: /* table_reference_list */ - case 439: /* table_reference */ - case 440: /* table_primary */ - case 441: /* joined_table */ - case 443: /* subquery */ - case 444: /* parenthesized_joined_table */ - case 446: /* search_condition */ - case 447: /* query_specification */ - case 451: /* range_opt */ - case 452: /* every_opt */ - case 453: /* fill_opt */ - case 454: /* twindow_clause_opt */ - case 456: /* having_clause_opt */ - case 457: /* select_item */ - case 459: /* partition_item */ - case 462: /* query_expression */ - case 463: /* query_simple */ - case 465: /* slimit_clause_opt */ - case 466: /* limit_clause_opt */ - case 467: /* union_query_expression */ - case 468: /* query_simple_or_subquery */ - case 470: /* sort_specification */ + case 330: /* cmd */ + case 333: /* literal */ + case 339: /* with_opt */ + case 345: /* search_condition */ + case 349: /* db_options */ + case 351: /* alter_db_options */ + case 353: /* start_opt */ + case 354: /* end_opt */ + case 359: /* retention */ + case 360: /* full_table_name */ + case 363: /* table_options */ + case 367: /* alter_table_clause */ + case 368: /* alter_table_options */ + case 371: /* signed_literal */ + case 372: /* create_subtable_clause */ + case 375: /* drop_table_clause */ + case 377: /* column_def */ + case 381: /* duration_literal */ + case 382: /* rollup_func_name */ + case 384: /* col_name */ + case 385: /* db_name_cond_opt */ + case 386: /* like_pattern_opt */ + case 387: /* table_name_cond */ + case 388: /* from_db_opt */ + case 390: /* tag_item */ + case 392: /* full_index_name */ + case 393: /* index_options */ + case 396: /* sliding_opt */ + case 397: /* sma_stream_opt */ + case 398: /* func */ + case 400: /* query_or_subquery */ + case 403: /* explain_options */ + case 404: /* insert_query */ + case 410: /* stream_options */ + case 413: /* subtable_opt */ + case 414: /* expression */ + case 416: /* where_clause_opt */ + case 417: /* signed */ + case 418: /* literal_func */ + case 421: /* expr_or_subquery */ + case 422: /* pseudo_column */ + case 423: /* column_reference */ + case 424: /* function_expression */ + case 425: /* case_when_expression */ + case 430: /* star_func_para */ + case 432: /* case_when_else_opt */ + case 433: /* common_expression */ + case 434: /* when_then_expr */ + case 435: /* predicate */ + case 438: /* in_predicate_value */ + case 439: /* boolean_value_expression */ + case 440: /* boolean_primary */ + case 441: /* from_clause_opt */ + case 442: /* table_reference_list */ + case 443: /* table_reference */ + case 444: /* table_primary */ + case 445: /* joined_table */ + case 447: /* subquery */ + case 448: /* parenthesized_joined_table */ + case 450: /* query_specification */ + case 454: /* range_opt */ + case 455: /* every_opt */ + case 456: /* fill_opt */ + case 457: /* twindow_clause_opt */ + case 459: /* having_clause_opt */ + case 460: /* select_item */ + case 462: /* partition_item */ + case 465: /* query_expression */ + case 466: /* query_simple */ + case 468: /* slimit_clause_opt */ + case 469: /* limit_clause_opt */ + case 470: /* union_query_expression */ + case 471: /* query_simple_or_subquery */ + case 473: /* sort_specification */ +{ + nodesDestroyNode((yypminor->yy448)); +} + break; + case 331: /* account_options */ + case 332: /* alter_account_options */ + case 334: /* alter_account_option */ + case 352: /* speed_opt */ + case 407: /* bufsize_opt */ { - nodesDestroyNode((yypminor->yy828)); + } break; - case 330: /* account_options */ - case 331: /* alter_account_options */ - case 333: /* alter_account_option */ - case 348: /* speed_opt */ - case 403: /* bufsize_opt */ + case 335: /* user_name */ + case 342: /* db_name */ + case 343: /* table_name */ + case 344: /* topic_name */ + case 346: /* dnode_endpoint */ + case 369: /* column_name */ + case 383: /* function_name */ + case 391: /* column_alias */ + case 394: /* index_name */ + case 399: /* sma_func_name */ + case 401: /* cgroup_name */ + case 408: /* language_opt */ + case 409: /* stream_name */ + case 420: /* table_alias */ + case 426: /* star_func */ + case 428: /* noarg_func */ + case 446: /* alias_opt */ { } break; - case 334: /* user_name */ - case 337: /* priv_level */ - case 340: /* db_name */ - case 341: /* topic_name */ - case 342: /* dnode_endpoint */ - case 365: /* column_name */ - case 373: /* table_name */ - case 380: /* function_name */ - case 388: /* column_alias */ - case 391: /* index_name */ - case 396: /* sma_func_name */ - case 398: /* cgroup_name */ - case 404: /* language_opt */ - case 405: /* stream_name */ - case 416: /* table_alias */ - case 422: /* star_func */ - case 424: /* noarg_func */ - case 442: /* alias_opt */ + case 336: /* sysinfo_opt */ { } break; - case 335: /* sysinfo_opt */ + case 337: /* privileges */ + case 340: /* priv_type_list */ + case 341: /* priv_type */ { } break; - case 336: /* privileges */ - case 338: /* priv_type_list */ - case 339: /* priv_type */ + case 338: /* priv_level */ { } break; - case 343: /* force_opt */ - case 344: /* not_exists_opt */ - case 346: /* exists_opt */ - case 399: /* analyze_opt */ - case 402: /* agg_func_opt */ - case 448: /* set_quantifier_opt */ + case 347: /* force_opt */ + case 348: /* not_exists_opt */ + case 350: /* exists_opt */ + case 402: /* analyze_opt */ + case 405: /* or_replace_opt */ + case 406: /* agg_func_opt */ + case 451: /* set_quantifier_opt */ { } break; - case 351: /* integer_list */ - case 352: /* variable_list */ - case 353: /* retention_list */ - case 357: /* column_def_list */ - case 358: /* tags_def_opt */ - case 360: /* multi_create_clause */ - case 361: /* tags_def */ - case 362: /* multi_drop_clause */ - case 369: /* specific_cols_opt */ - case 370: /* expression_list */ - case 372: /* col_name_list */ - case 375: /* duration_list */ - case 376: /* rollup_func_list */ - case 386: /* tag_list_opt */ - case 392: /* func_list */ - case 407: /* col_list_opt */ - case 408: /* tag_def_or_ref_opt */ - case 411: /* dnode_list */ - case 415: /* literal_list */ - case 423: /* star_func_para_list */ - case 425: /* other_para_list */ - case 427: /* when_then_list */ - case 449: /* select_list */ - case 450: /* partition_by_clause_opt */ - case 455: /* group_by_clause_opt */ - case 458: /* partition_list */ - case 461: /* group_by_list */ - case 464: /* order_by_clause_opt */ - case 469: /* sort_specification_list */ + case 355: /* integer_list */ + case 356: /* variable_list */ + case 357: /* retention_list */ + case 361: /* column_def_list */ + case 362: /* tags_def_opt */ + case 364: /* multi_create_clause */ + case 365: /* tags_def */ + case 366: /* multi_drop_clause */ + case 373: /* specific_cols_opt */ + case 374: /* expression_list */ + case 376: /* col_name_list */ + case 378: /* duration_list */ + case 379: /* rollup_func_list */ + case 389: /* tag_list_opt */ + case 395: /* func_list */ + case 411: /* col_list_opt */ + case 412: /* tag_def_or_ref_opt */ + case 415: /* dnode_list */ + case 419: /* literal_list */ + case 427: /* star_func_para_list */ + case 429: /* other_para_list */ + case 431: /* when_then_list */ + case 452: /* select_list */ + case 453: /* partition_by_clause_opt */ + case 458: /* group_by_clause_opt */ + case 461: /* partition_list */ + case 464: /* group_by_list */ + case 467: /* order_by_clause_opt */ + case 472: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy222)); + nodesDestroyList((yypminor->yy432)); } break; - case 354: /* alter_db_option */ - case 377: /* alter_table_option */ + case 358: /* alter_db_option */ + case 380: /* alter_table_option */ { } break; - case 366: /* type_name */ + case 370: /* type_name */ { } break; - case 432: /* compare_op */ - case 433: /* in_op */ + case 436: /* compare_op */ + case 437: /* in_op */ { } break; - case 445: /* join_type */ + case 449: /* join_type */ { } break; - case 460: /* fill_mode */ + case 463: /* fill_mode */ { } break; - case 471: /* ordering_specification_opt */ + case 474: /* ordering_specification_opt */ { } break; - case 472: /* null_ordering_opt */ + case 475: /* null_ordering_opt */ { } @@ -3091,579 +3153,589 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 329, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 329, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 330, 0 }, /* (2) account_options ::= */ - { 330, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 330, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 330, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 330, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 330, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 330, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 330, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 330, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 330, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 331, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 331, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 333, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 333, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 333, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 333, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 333, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 333, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 333, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 333, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 333, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 333, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 329, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - { 329, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 329, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - { 329, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - { 329, -3 }, /* (28) cmd ::= DROP USER user_name */ - { 335, 0 }, /* (29) sysinfo_opt ::= */ - { 335, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 329, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 329, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 336, -1 }, /* (33) privileges ::= ALL */ - { 336, -1 }, /* (34) privileges ::= priv_type_list */ - { 336, -1 }, /* (35) privileges ::= SUBSCRIBE */ - { 338, -1 }, /* (36) priv_type_list ::= priv_type */ - { 338, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 339, -1 }, /* (38) priv_type ::= READ */ - { 339, -1 }, /* (39) priv_type ::= WRITE */ - { 337, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 337, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - { 337, -1 }, /* (42) priv_level ::= topic_name */ - { 329, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ - { 329, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 329, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ - { 329, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ - { 329, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 329, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 329, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ - { 329, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 342, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ - { 342, -1 }, /* (52) dnode_endpoint ::= NK_ID */ - { 342, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ - { 343, 0 }, /* (54) force_opt ::= */ - { 343, -1 }, /* (55) force_opt ::= FORCE */ - { 329, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ - { 329, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 329, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 329, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 329, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ - { 329, -2 }, /* (68) cmd ::= USE db_name */ - { 329, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 329, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ - { 329, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ - { 329, -5 }, /* (72) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ - { 344, -3 }, /* (73) not_exists_opt ::= IF NOT EXISTS */ - { 344, 0 }, /* (74) not_exists_opt ::= */ - { 346, -2 }, /* (75) exists_opt ::= IF EXISTS */ - { 346, 0 }, /* (76) exists_opt ::= */ - { 345, 0 }, /* (77) db_options ::= */ - { 345, -3 }, /* (78) db_options ::= db_options BUFFER NK_INTEGER */ - { 345, -3 }, /* (79) db_options ::= db_options CACHEMODEL NK_STRING */ - { 345, -3 }, /* (80) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 345, -3 }, /* (81) db_options ::= db_options COMP NK_INTEGER */ - { 345, -3 }, /* (82) db_options ::= db_options DURATION NK_INTEGER */ - { 345, -3 }, /* (83) db_options ::= db_options DURATION NK_VARIABLE */ - { 345, -3 }, /* (84) db_options ::= db_options MAXROWS NK_INTEGER */ - { 345, -3 }, /* (85) db_options ::= db_options MINROWS NK_INTEGER */ - { 345, -3 }, /* (86) db_options ::= db_options KEEP integer_list */ - { 345, -3 }, /* (87) db_options ::= db_options KEEP variable_list */ - { 345, -3 }, /* (88) db_options ::= db_options PAGES NK_INTEGER */ - { 345, -3 }, /* (89) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 345, -3 }, /* (90) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - { 345, -3 }, /* (91) db_options ::= db_options PRECISION NK_STRING */ - { 345, -3 }, /* (92) db_options ::= db_options REPLICA NK_INTEGER */ - { 345, -3 }, /* (93) db_options ::= db_options VGROUPS NK_INTEGER */ - { 345, -3 }, /* (94) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 345, -3 }, /* (95) db_options ::= db_options RETENTIONS retention_list */ - { 345, -3 }, /* (96) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 345, -3 }, /* (97) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 345, -3 }, /* (98) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 345, -3 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 345, -4 }, /* (100) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 345, -3 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 345, -4 }, /* (102) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 345, -3 }, /* (103) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 345, -3 }, /* (104) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 345, -3 }, /* (105) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - { 345, -3 }, /* (106) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ - { 345, -3 }, /* (107) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ - { 347, -1 }, /* (108) alter_db_options ::= alter_db_option */ - { 347, -2 }, /* (109) alter_db_options ::= alter_db_options alter_db_option */ - { 354, -2 }, /* (110) alter_db_option ::= BUFFER NK_INTEGER */ - { 354, -2 }, /* (111) alter_db_option ::= CACHEMODEL NK_STRING */ - { 354, -2 }, /* (112) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 354, -2 }, /* (113) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 354, -2 }, /* (114) alter_db_option ::= KEEP integer_list */ - { 354, -2 }, /* (115) alter_db_option ::= KEEP variable_list */ - { 354, -2 }, /* (116) alter_db_option ::= PAGES NK_INTEGER */ - { 354, -2 }, /* (117) alter_db_option ::= REPLICA NK_INTEGER */ - { 354, -2 }, /* (118) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 354, -2 }, /* (119) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - { 351, -1 }, /* (120) integer_list ::= NK_INTEGER */ - { 351, -3 }, /* (121) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 352, -1 }, /* (122) variable_list ::= NK_VARIABLE */ - { 352, -3 }, /* (123) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 353, -1 }, /* (124) retention_list ::= retention */ - { 353, -3 }, /* (125) retention_list ::= retention_list NK_COMMA retention */ - { 355, -3 }, /* (126) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 348, 0 }, /* (127) speed_opt ::= */ - { 348, -2 }, /* (128) speed_opt ::= MAX_SPEED NK_INTEGER */ - { 349, 0 }, /* (129) start_opt ::= */ - { 349, -3 }, /* (130) start_opt ::= START WITH NK_INTEGER */ - { 349, -3 }, /* (131) start_opt ::= START WITH NK_STRING */ - { 349, -4 }, /* (132) start_opt ::= START WITH TIMESTAMP NK_STRING */ - { 350, 0 }, /* (133) end_opt ::= */ - { 350, -3 }, /* (134) end_opt ::= END WITH NK_INTEGER */ - { 350, -3 }, /* (135) end_opt ::= END WITH NK_STRING */ - { 350, -4 }, /* (136) end_opt ::= END WITH TIMESTAMP NK_STRING */ - { 329, -9 }, /* (137) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 329, -3 }, /* (138) cmd ::= CREATE TABLE multi_create_clause */ - { 329, -9 }, /* (139) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 329, -3 }, /* (140) cmd ::= DROP TABLE multi_drop_clause */ - { 329, -4 }, /* (141) cmd ::= DROP STABLE exists_opt full_table_name */ - { 329, -3 }, /* (142) cmd ::= ALTER TABLE alter_table_clause */ - { 329, -3 }, /* (143) cmd ::= ALTER STABLE alter_table_clause */ - { 363, -2 }, /* (144) alter_table_clause ::= full_table_name alter_table_options */ - { 363, -5 }, /* (145) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 363, -4 }, /* (146) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 363, -5 }, /* (147) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 363, -5 }, /* (148) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 363, -5 }, /* (149) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 363, -4 }, /* (150) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 363, -5 }, /* (151) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 363, -5 }, /* (152) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 363, -6 }, /* (153) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 360, -1 }, /* (154) multi_create_clause ::= create_subtable_clause */ - { 360, -2 }, /* (155) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 368, -10 }, /* (156) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 362, -1 }, /* (157) multi_drop_clause ::= drop_table_clause */ - { 362, -3 }, /* (158) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ - { 371, -2 }, /* (159) drop_table_clause ::= exists_opt full_table_name */ - { 369, 0 }, /* (160) specific_cols_opt ::= */ - { 369, -3 }, /* (161) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 356, -1 }, /* (162) full_table_name ::= table_name */ - { 356, -3 }, /* (163) full_table_name ::= db_name NK_DOT table_name */ - { 357, -1 }, /* (164) column_def_list ::= column_def */ - { 357, -3 }, /* (165) column_def_list ::= column_def_list NK_COMMA column_def */ - { 374, -2 }, /* (166) column_def ::= column_name type_name */ - { 374, -4 }, /* (167) column_def ::= column_name type_name COMMENT NK_STRING */ - { 366, -1 }, /* (168) type_name ::= BOOL */ - { 366, -1 }, /* (169) type_name ::= TINYINT */ - { 366, -1 }, /* (170) type_name ::= SMALLINT */ - { 366, -1 }, /* (171) type_name ::= INT */ - { 366, -1 }, /* (172) type_name ::= INTEGER */ - { 366, -1 }, /* (173) type_name ::= BIGINT */ - { 366, -1 }, /* (174) type_name ::= FLOAT */ - { 366, -1 }, /* (175) type_name ::= DOUBLE */ - { 366, -4 }, /* (176) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 366, -1 }, /* (177) type_name ::= TIMESTAMP */ - { 366, -4 }, /* (178) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 366, -2 }, /* (179) type_name ::= TINYINT UNSIGNED */ - { 366, -2 }, /* (180) type_name ::= SMALLINT UNSIGNED */ - { 366, -2 }, /* (181) type_name ::= INT UNSIGNED */ - { 366, -2 }, /* (182) type_name ::= BIGINT UNSIGNED */ - { 366, -1 }, /* (183) type_name ::= JSON */ - { 366, -4 }, /* (184) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 366, -1 }, /* (185) type_name ::= MEDIUMBLOB */ - { 366, -1 }, /* (186) type_name ::= BLOB */ - { 366, -4 }, /* (187) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 366, -1 }, /* (188) type_name ::= DECIMAL */ - { 366, -4 }, /* (189) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 366, -6 }, /* (190) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 358, 0 }, /* (191) tags_def_opt ::= */ - { 358, -1 }, /* (192) tags_def_opt ::= tags_def */ - { 361, -4 }, /* (193) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 359, 0 }, /* (194) table_options ::= */ - { 359, -3 }, /* (195) table_options ::= table_options COMMENT NK_STRING */ - { 359, -3 }, /* (196) table_options ::= table_options MAX_DELAY duration_list */ - { 359, -3 }, /* (197) table_options ::= table_options WATERMARK duration_list */ - { 359, -5 }, /* (198) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 359, -3 }, /* (199) table_options ::= table_options TTL NK_INTEGER */ - { 359, -5 }, /* (200) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 359, -3 }, /* (201) table_options ::= table_options DELETE_MARK duration_list */ - { 364, -1 }, /* (202) alter_table_options ::= alter_table_option */ - { 364, -2 }, /* (203) alter_table_options ::= alter_table_options alter_table_option */ - { 377, -2 }, /* (204) alter_table_option ::= COMMENT NK_STRING */ - { 377, -2 }, /* (205) alter_table_option ::= TTL NK_INTEGER */ - { 375, -1 }, /* (206) duration_list ::= duration_literal */ - { 375, -3 }, /* (207) duration_list ::= duration_list NK_COMMA duration_literal */ - { 376, -1 }, /* (208) rollup_func_list ::= rollup_func_name */ - { 376, -3 }, /* (209) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 379, -1 }, /* (210) rollup_func_name ::= function_name */ - { 379, -1 }, /* (211) rollup_func_name ::= FIRST */ - { 379, -1 }, /* (212) rollup_func_name ::= LAST */ - { 372, -1 }, /* (213) col_name_list ::= col_name */ - { 372, -3 }, /* (214) col_name_list ::= col_name_list NK_COMMA col_name */ - { 381, -1 }, /* (215) col_name ::= column_name */ - { 329, -2 }, /* (216) cmd ::= SHOW DNODES */ - { 329, -2 }, /* (217) cmd ::= SHOW USERS */ - { 329, -3 }, /* (218) cmd ::= SHOW USER PRIVILEGES */ - { 329, -2 }, /* (219) cmd ::= SHOW DATABASES */ - { 329, -4 }, /* (220) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 329, -4 }, /* (221) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 329, -3 }, /* (222) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 329, -2 }, /* (223) cmd ::= SHOW MNODES */ - { 329, -2 }, /* (224) cmd ::= SHOW QNODES */ - { 329, -2 }, /* (225) cmd ::= SHOW FUNCTIONS */ - { 329, -5 }, /* (226) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 329, -2 }, /* (227) cmd ::= SHOW STREAMS */ - { 329, -2 }, /* (228) cmd ::= SHOW ACCOUNTS */ - { 329, -2 }, /* (229) cmd ::= SHOW APPS */ - { 329, -2 }, /* (230) cmd ::= SHOW CONNECTIONS */ - { 329, -2 }, /* (231) cmd ::= SHOW LICENCES */ - { 329, -2 }, /* (232) cmd ::= SHOW GRANTS */ - { 329, -4 }, /* (233) cmd ::= SHOW CREATE DATABASE db_name */ - { 329, -4 }, /* (234) cmd ::= SHOW CREATE TABLE full_table_name */ - { 329, -4 }, /* (235) cmd ::= SHOW CREATE STABLE full_table_name */ - { 329, -2 }, /* (236) cmd ::= SHOW QUERIES */ - { 329, -2 }, /* (237) cmd ::= SHOW SCORES */ - { 329, -2 }, /* (238) cmd ::= SHOW TOPICS */ - { 329, -2 }, /* (239) cmd ::= SHOW VARIABLES */ - { 329, -3 }, /* (240) cmd ::= SHOW CLUSTER VARIABLES */ - { 329, -3 }, /* (241) cmd ::= SHOW LOCAL VARIABLES */ - { 329, -5 }, /* (242) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 329, -2 }, /* (243) cmd ::= SHOW BNODES */ - { 329, -2 }, /* (244) cmd ::= SHOW SNODES */ - { 329, -2 }, /* (245) cmd ::= SHOW CLUSTER */ - { 329, -2 }, /* (246) cmd ::= SHOW TRANSACTIONS */ - { 329, -4 }, /* (247) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 329, -2 }, /* (248) cmd ::= SHOW CONSUMERS */ - { 329, -2 }, /* (249) cmd ::= SHOW SUBSCRIPTIONS */ - { 329, -5 }, /* (250) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 329, -7 }, /* (251) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 329, -3 }, /* (252) cmd ::= SHOW VNODES NK_INTEGER */ - { 329, -3 }, /* (253) cmd ::= SHOW VNODES NK_STRING */ - { 329, -3 }, /* (254) cmd ::= SHOW db_name_cond_opt ALIVE */ - { 329, -3 }, /* (255) cmd ::= SHOW CLUSTER ALIVE */ - { 382, 0 }, /* (256) db_name_cond_opt ::= */ - { 382, -2 }, /* (257) db_name_cond_opt ::= db_name NK_DOT */ - { 383, 0 }, /* (258) like_pattern_opt ::= */ - { 383, -2 }, /* (259) like_pattern_opt ::= LIKE NK_STRING */ - { 384, -1 }, /* (260) table_name_cond ::= table_name */ - { 385, 0 }, /* (261) from_db_opt ::= */ - { 385, -2 }, /* (262) from_db_opt ::= FROM db_name */ - { 386, 0 }, /* (263) tag_list_opt ::= */ - { 386, -1 }, /* (264) tag_list_opt ::= tag_item */ - { 386, -3 }, /* (265) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 387, -1 }, /* (266) tag_item ::= TBNAME */ - { 387, -1 }, /* (267) tag_item ::= QTAGS */ - { 387, -1 }, /* (268) tag_item ::= column_name */ - { 387, -2 }, /* (269) tag_item ::= column_name column_alias */ - { 387, -3 }, /* (270) tag_item ::= column_name AS column_alias */ - { 329, -8 }, /* (271) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - { 329, -9 }, /* (272) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - { 329, -4 }, /* (273) cmd ::= DROP INDEX exists_opt full_index_name */ - { 389, -1 }, /* (274) full_index_name ::= index_name */ - { 389, -3 }, /* (275) full_index_name ::= db_name NK_DOT index_name */ - { 390, -10 }, /* (276) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 390, -12 }, /* (277) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 392, -1 }, /* (278) func_list ::= func */ - { 392, -3 }, /* (279) func_list ::= func_list NK_COMMA func */ - { 395, -4 }, /* (280) func ::= sma_func_name NK_LP expression_list NK_RP */ - { 396, -1 }, /* (281) sma_func_name ::= function_name */ - { 396, -1 }, /* (282) sma_func_name ::= COUNT */ - { 396, -1 }, /* (283) sma_func_name ::= FIRST */ - { 396, -1 }, /* (284) sma_func_name ::= LAST */ - { 396, -1 }, /* (285) sma_func_name ::= LAST_ROW */ - { 394, 0 }, /* (286) sma_stream_opt ::= */ - { 394, -3 }, /* (287) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - { 394, -3 }, /* (288) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - { 394, -3 }, /* (289) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - { 329, -6 }, /* (290) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 329, -7 }, /* (291) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 329, -9 }, /* (292) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 329, -7 }, /* (293) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 329, -9 }, /* (294) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 329, -4 }, /* (295) cmd ::= DROP TOPIC exists_opt topic_name */ - { 329, -7 }, /* (296) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 329, -2 }, /* (297) cmd ::= DESC full_table_name */ - { 329, -2 }, /* (298) cmd ::= DESCRIBE full_table_name */ - { 329, -3 }, /* (299) cmd ::= RESET QUERY CACHE */ - { 329, -4 }, /* (300) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 329, -4 }, /* (301) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - { 399, 0 }, /* (302) analyze_opt ::= */ - { 399, -1 }, /* (303) analyze_opt ::= ANALYZE */ - { 400, 0 }, /* (304) explain_options ::= */ - { 400, -3 }, /* (305) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 400, -3 }, /* (306) explain_options ::= explain_options RATIO NK_FLOAT */ - { 329, -11 }, /* (307) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - { 329, -4 }, /* (308) cmd ::= DROP FUNCTION exists_opt function_name */ - { 402, 0 }, /* (309) agg_func_opt ::= */ - { 402, -1 }, /* (310) agg_func_opt ::= AGGREGATE */ - { 403, 0 }, /* (311) bufsize_opt ::= */ - { 403, -2 }, /* (312) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 404, 0 }, /* (313) language_opt ::= */ - { 404, -2 }, /* (314) language_opt ::= LANGUAGE NK_STRING */ - { 329, -12 }, /* (315) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - { 329, -4 }, /* (316) cmd ::= DROP STREAM exists_opt stream_name */ - { 407, 0 }, /* (317) col_list_opt ::= */ - { 407, -3 }, /* (318) col_list_opt ::= NK_LP col_name_list NK_RP */ - { 408, 0 }, /* (319) tag_def_or_ref_opt ::= */ - { 408, -1 }, /* (320) tag_def_or_ref_opt ::= tags_def */ - { 408, -4 }, /* (321) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - { 406, 0 }, /* (322) stream_options ::= */ - { 406, -3 }, /* (323) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 406, -3 }, /* (324) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 406, -4 }, /* (325) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 406, -3 }, /* (326) stream_options ::= stream_options WATERMARK duration_literal */ - { 406, -4 }, /* (327) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 406, -3 }, /* (328) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 406, -3 }, /* (329) stream_options ::= stream_options DELETE_MARK duration_literal */ - { 406, -4 }, /* (330) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - { 409, 0 }, /* (331) subtable_opt ::= */ - { 409, -4 }, /* (332) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 329, -3 }, /* (333) cmd ::= KILL CONNECTION NK_INTEGER */ - { 329, -3 }, /* (334) cmd ::= KILL QUERY NK_STRING */ - { 329, -3 }, /* (335) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 329, -2 }, /* (336) cmd ::= BALANCE VGROUP */ - { 329, -4 }, /* (337) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 329, -4 }, /* (338) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 329, -3 }, /* (339) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 411, -2 }, /* (340) dnode_list ::= DNODE NK_INTEGER */ - { 411, -3 }, /* (341) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 329, -4 }, /* (342) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 329, -1 }, /* (343) cmd ::= query_or_subquery */ - { 329, -1 }, /* (344) cmd ::= insert_query */ - { 401, -7 }, /* (345) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 401, -4 }, /* (346) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - { 332, -1 }, /* (347) literal ::= NK_INTEGER */ - { 332, -1 }, /* (348) literal ::= NK_FLOAT */ - { 332, -1 }, /* (349) literal ::= NK_STRING */ - { 332, -1 }, /* (350) literal ::= NK_BOOL */ - { 332, -2 }, /* (351) literal ::= TIMESTAMP NK_STRING */ - { 332, -1 }, /* (352) literal ::= duration_literal */ - { 332, -1 }, /* (353) literal ::= NULL */ - { 332, -1 }, /* (354) literal ::= NK_QUESTION */ - { 378, -1 }, /* (355) duration_literal ::= NK_VARIABLE */ - { 413, -1 }, /* (356) signed ::= NK_INTEGER */ - { 413, -2 }, /* (357) signed ::= NK_PLUS NK_INTEGER */ - { 413, -2 }, /* (358) signed ::= NK_MINUS NK_INTEGER */ - { 413, -1 }, /* (359) signed ::= NK_FLOAT */ - { 413, -2 }, /* (360) signed ::= NK_PLUS NK_FLOAT */ - { 413, -2 }, /* (361) signed ::= NK_MINUS NK_FLOAT */ - { 367, -1 }, /* (362) signed_literal ::= signed */ - { 367, -1 }, /* (363) signed_literal ::= NK_STRING */ - { 367, -1 }, /* (364) signed_literal ::= NK_BOOL */ - { 367, -2 }, /* (365) signed_literal ::= TIMESTAMP NK_STRING */ - { 367, -1 }, /* (366) signed_literal ::= duration_literal */ - { 367, -1 }, /* (367) signed_literal ::= NULL */ - { 367, -1 }, /* (368) signed_literal ::= literal_func */ - { 367, -1 }, /* (369) signed_literal ::= NK_QUESTION */ - { 415, -1 }, /* (370) literal_list ::= signed_literal */ - { 415, -3 }, /* (371) literal_list ::= literal_list NK_COMMA signed_literal */ - { 340, -1 }, /* (372) db_name ::= NK_ID */ - { 373, -1 }, /* (373) table_name ::= NK_ID */ - { 365, -1 }, /* (374) column_name ::= NK_ID */ - { 380, -1 }, /* (375) function_name ::= NK_ID */ - { 416, -1 }, /* (376) table_alias ::= NK_ID */ - { 388, -1 }, /* (377) column_alias ::= NK_ID */ - { 334, -1 }, /* (378) user_name ::= NK_ID */ - { 341, -1 }, /* (379) topic_name ::= NK_ID */ - { 405, -1 }, /* (380) stream_name ::= NK_ID */ - { 398, -1 }, /* (381) cgroup_name ::= NK_ID */ - { 391, -1 }, /* (382) index_name ::= NK_ID */ - { 417, -1 }, /* (383) expr_or_subquery ::= expression */ - { 410, -1 }, /* (384) expression ::= literal */ - { 410, -1 }, /* (385) expression ::= pseudo_column */ - { 410, -1 }, /* (386) expression ::= column_reference */ - { 410, -1 }, /* (387) expression ::= function_expression */ - { 410, -1 }, /* (388) expression ::= case_when_expression */ - { 410, -3 }, /* (389) expression ::= NK_LP expression NK_RP */ - { 410, -2 }, /* (390) expression ::= NK_PLUS expr_or_subquery */ - { 410, -2 }, /* (391) expression ::= NK_MINUS expr_or_subquery */ - { 410, -3 }, /* (392) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 410, -3 }, /* (393) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 410, -3 }, /* (394) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 410, -3 }, /* (395) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 410, -3 }, /* (396) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 410, -3 }, /* (397) expression ::= column_reference NK_ARROW NK_STRING */ - { 410, -3 }, /* (398) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 410, -3 }, /* (399) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 370, -1 }, /* (400) expression_list ::= expr_or_subquery */ - { 370, -3 }, /* (401) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 419, -1 }, /* (402) column_reference ::= column_name */ - { 419, -3 }, /* (403) column_reference ::= table_name NK_DOT column_name */ - { 418, -1 }, /* (404) pseudo_column ::= ROWTS */ - { 418, -1 }, /* (405) pseudo_column ::= TBNAME */ - { 418, -3 }, /* (406) pseudo_column ::= table_name NK_DOT TBNAME */ - { 418, -1 }, /* (407) pseudo_column ::= QSTART */ - { 418, -1 }, /* (408) pseudo_column ::= QEND */ - { 418, -1 }, /* (409) pseudo_column ::= QDURATION */ - { 418, -1 }, /* (410) pseudo_column ::= WSTART */ - { 418, -1 }, /* (411) pseudo_column ::= WEND */ - { 418, -1 }, /* (412) pseudo_column ::= WDURATION */ - { 418, -1 }, /* (413) pseudo_column ::= IROWTS */ - { 418, -1 }, /* (414) pseudo_column ::= ISFILLED */ - { 418, -1 }, /* (415) pseudo_column ::= QTAGS */ - { 420, -4 }, /* (416) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 420, -4 }, /* (417) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 420, -6 }, /* (418) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 420, -1 }, /* (419) function_expression ::= literal_func */ - { 414, -3 }, /* (420) literal_func ::= noarg_func NK_LP NK_RP */ - { 414, -1 }, /* (421) literal_func ::= NOW */ - { 424, -1 }, /* (422) noarg_func ::= NOW */ - { 424, -1 }, /* (423) noarg_func ::= TODAY */ - { 424, -1 }, /* (424) noarg_func ::= TIMEZONE */ - { 424, -1 }, /* (425) noarg_func ::= DATABASE */ - { 424, -1 }, /* (426) noarg_func ::= CLIENT_VERSION */ - { 424, -1 }, /* (427) noarg_func ::= SERVER_VERSION */ - { 424, -1 }, /* (428) noarg_func ::= SERVER_STATUS */ - { 424, -1 }, /* (429) noarg_func ::= CURRENT_USER */ - { 424, -1 }, /* (430) noarg_func ::= USER */ - { 422, -1 }, /* (431) star_func ::= COUNT */ - { 422, -1 }, /* (432) star_func ::= FIRST */ - { 422, -1 }, /* (433) star_func ::= LAST */ - { 422, -1 }, /* (434) star_func ::= LAST_ROW */ - { 423, -1 }, /* (435) star_func_para_list ::= NK_STAR */ - { 423, -1 }, /* (436) star_func_para_list ::= other_para_list */ - { 425, -1 }, /* (437) other_para_list ::= star_func_para */ - { 425, -3 }, /* (438) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 426, -1 }, /* (439) star_func_para ::= expr_or_subquery */ - { 426, -3 }, /* (440) star_func_para ::= table_name NK_DOT NK_STAR */ - { 421, -4 }, /* (441) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 421, -5 }, /* (442) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 427, -1 }, /* (443) when_then_list ::= when_then_expr */ - { 427, -2 }, /* (444) when_then_list ::= when_then_list when_then_expr */ - { 430, -4 }, /* (445) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 428, 0 }, /* (446) case_when_else_opt ::= */ - { 428, -2 }, /* (447) case_when_else_opt ::= ELSE common_expression */ - { 431, -3 }, /* (448) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 431, -5 }, /* (449) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 431, -6 }, /* (450) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 431, -3 }, /* (451) predicate ::= expr_or_subquery IS NULL */ - { 431, -4 }, /* (452) predicate ::= expr_or_subquery IS NOT NULL */ - { 431, -3 }, /* (453) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 432, -1 }, /* (454) compare_op ::= NK_LT */ - { 432, -1 }, /* (455) compare_op ::= NK_GT */ - { 432, -1 }, /* (456) compare_op ::= NK_LE */ - { 432, -1 }, /* (457) compare_op ::= NK_GE */ - { 432, -1 }, /* (458) compare_op ::= NK_NE */ - { 432, -1 }, /* (459) compare_op ::= NK_EQ */ - { 432, -1 }, /* (460) compare_op ::= LIKE */ - { 432, -2 }, /* (461) compare_op ::= NOT LIKE */ - { 432, -1 }, /* (462) compare_op ::= MATCH */ - { 432, -1 }, /* (463) compare_op ::= NMATCH */ - { 432, -1 }, /* (464) compare_op ::= CONTAINS */ - { 433, -1 }, /* (465) in_op ::= IN */ - { 433, -2 }, /* (466) in_op ::= NOT IN */ - { 434, -3 }, /* (467) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 435, -1 }, /* (468) boolean_value_expression ::= boolean_primary */ - { 435, -2 }, /* (469) boolean_value_expression ::= NOT boolean_primary */ - { 435, -3 }, /* (470) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 435, -3 }, /* (471) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 436, -1 }, /* (472) boolean_primary ::= predicate */ - { 436, -3 }, /* (473) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 429, -1 }, /* (474) common_expression ::= expr_or_subquery */ - { 429, -1 }, /* (475) common_expression ::= boolean_value_expression */ - { 437, 0 }, /* (476) from_clause_opt ::= */ - { 437, -2 }, /* (477) from_clause_opt ::= FROM table_reference_list */ - { 438, -1 }, /* (478) table_reference_list ::= table_reference */ - { 438, -3 }, /* (479) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 439, -1 }, /* (480) table_reference ::= table_primary */ - { 439, -1 }, /* (481) table_reference ::= joined_table */ - { 440, -2 }, /* (482) table_primary ::= table_name alias_opt */ - { 440, -4 }, /* (483) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 440, -2 }, /* (484) table_primary ::= subquery alias_opt */ - { 440, -1 }, /* (485) table_primary ::= parenthesized_joined_table */ - { 442, 0 }, /* (486) alias_opt ::= */ - { 442, -1 }, /* (487) alias_opt ::= table_alias */ - { 442, -2 }, /* (488) alias_opt ::= AS table_alias */ - { 444, -3 }, /* (489) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 444, -3 }, /* (490) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 441, -6 }, /* (491) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 445, 0 }, /* (492) join_type ::= */ - { 445, -1 }, /* (493) join_type ::= INNER */ - { 447, -12 }, /* (494) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 448, 0 }, /* (495) set_quantifier_opt ::= */ - { 448, -1 }, /* (496) set_quantifier_opt ::= DISTINCT */ - { 448, -1 }, /* (497) set_quantifier_opt ::= ALL */ - { 449, -1 }, /* (498) select_list ::= select_item */ - { 449, -3 }, /* (499) select_list ::= select_list NK_COMMA select_item */ - { 457, -1 }, /* (500) select_item ::= NK_STAR */ - { 457, -1 }, /* (501) select_item ::= common_expression */ - { 457, -2 }, /* (502) select_item ::= common_expression column_alias */ - { 457, -3 }, /* (503) select_item ::= common_expression AS column_alias */ - { 457, -3 }, /* (504) select_item ::= table_name NK_DOT NK_STAR */ - { 412, 0 }, /* (505) where_clause_opt ::= */ - { 412, -2 }, /* (506) where_clause_opt ::= WHERE search_condition */ - { 450, 0 }, /* (507) partition_by_clause_opt ::= */ - { 450, -3 }, /* (508) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 458, -1 }, /* (509) partition_list ::= partition_item */ - { 458, -3 }, /* (510) partition_list ::= partition_list NK_COMMA partition_item */ - { 459, -1 }, /* (511) partition_item ::= expr_or_subquery */ - { 459, -2 }, /* (512) partition_item ::= expr_or_subquery column_alias */ - { 459, -3 }, /* (513) partition_item ::= expr_or_subquery AS column_alias */ - { 454, 0 }, /* (514) twindow_clause_opt ::= */ - { 454, -6 }, /* (515) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 454, -4 }, /* (516) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 454, -6 }, /* (517) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 454, -8 }, /* (518) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 454, -7 }, /* (519) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - { 393, 0 }, /* (520) sliding_opt ::= */ - { 393, -4 }, /* (521) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 453, 0 }, /* (522) fill_opt ::= */ - { 453, -4 }, /* (523) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 453, -6 }, /* (524) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 453, -6 }, /* (525) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ - { 460, -1 }, /* (526) fill_mode ::= NONE */ - { 460, -1 }, /* (527) fill_mode ::= PREV */ - { 460, -1 }, /* (528) fill_mode ::= NULL */ - { 460, -1 }, /* (529) fill_mode ::= NULL_F */ - { 460, -1 }, /* (530) fill_mode ::= LINEAR */ - { 460, -1 }, /* (531) fill_mode ::= NEXT */ - { 455, 0 }, /* (532) group_by_clause_opt ::= */ - { 455, -3 }, /* (533) group_by_clause_opt ::= GROUP BY group_by_list */ - { 461, -1 }, /* (534) group_by_list ::= expr_or_subquery */ - { 461, -3 }, /* (535) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 456, 0 }, /* (536) having_clause_opt ::= */ - { 456, -2 }, /* (537) having_clause_opt ::= HAVING search_condition */ - { 451, 0 }, /* (538) range_opt ::= */ - { 451, -6 }, /* (539) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 452, 0 }, /* (540) every_opt ::= */ - { 452, -4 }, /* (541) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 462, -4 }, /* (542) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 463, -1 }, /* (543) query_simple ::= query_specification */ - { 463, -1 }, /* (544) query_simple ::= union_query_expression */ - { 467, -4 }, /* (545) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 467, -3 }, /* (546) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 468, -1 }, /* (547) query_simple_or_subquery ::= query_simple */ - { 468, -1 }, /* (548) query_simple_or_subquery ::= subquery */ - { 397, -1 }, /* (549) query_or_subquery ::= query_expression */ - { 397, -1 }, /* (550) query_or_subquery ::= subquery */ - { 464, 0 }, /* (551) order_by_clause_opt ::= */ - { 464, -3 }, /* (552) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 465, 0 }, /* (553) slimit_clause_opt ::= */ - { 465, -2 }, /* (554) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 465, -4 }, /* (555) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 465, -4 }, /* (556) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 466, 0 }, /* (557) limit_clause_opt ::= */ - { 466, -2 }, /* (558) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 466, -4 }, /* (559) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 466, -4 }, /* (560) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 443, -3 }, /* (561) subquery ::= NK_LP query_expression NK_RP */ - { 443, -3 }, /* (562) subquery ::= NK_LP subquery NK_RP */ - { 446, -1 }, /* (563) search_condition ::= common_expression */ - { 469, -1 }, /* (564) sort_specification_list ::= sort_specification */ - { 469, -3 }, /* (565) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 470, -3 }, /* (566) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 471, 0 }, /* (567) ordering_specification_opt ::= */ - { 471, -1 }, /* (568) ordering_specification_opt ::= ASC */ - { 471, -1 }, /* (569) ordering_specification_opt ::= DESC */ - { 472, 0 }, /* (570) null_ordering_opt ::= */ - { 472, -2 }, /* (571) null_ordering_opt ::= NULLS FIRST */ - { 472, -2 }, /* (572) null_ordering_opt ::= NULLS LAST */ + { 330, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 330, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 331, 0 }, /* (2) account_options ::= */ + { 331, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 331, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 331, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 331, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 331, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 331, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 331, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 331, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 331, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 332, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 332, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 334, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 334, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 334, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 334, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 334, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 334, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 334, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 334, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 334, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 334, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 330, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 330, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 330, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 330, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 330, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 336, 0 }, /* (29) sysinfo_opt ::= */ + { 336, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 330, -7 }, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ + { 330, -7 }, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ + { 337, -1 }, /* (33) privileges ::= ALL */ + { 337, -1 }, /* (34) privileges ::= priv_type_list */ + { 337, -1 }, /* (35) privileges ::= SUBSCRIBE */ + { 340, -1 }, /* (36) priv_type_list ::= priv_type */ + { 340, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 341, -1 }, /* (38) priv_type ::= READ */ + { 341, -1 }, /* (39) priv_type ::= WRITE */ + { 338, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 338, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + { 338, -3 }, /* (42) priv_level ::= db_name NK_DOT table_name */ + { 338, -1 }, /* (43) priv_level ::= topic_name */ + { 339, 0 }, /* (44) with_opt ::= */ + { 339, -2 }, /* (45) with_opt ::= WITH search_condition */ + { 330, -3 }, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ + { 330, -5 }, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 330, -4 }, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ + { 330, -4 }, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ + { 330, -4 }, /* (50) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 330, -5 }, /* (51) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 330, -4 }, /* (52) cmd ::= ALTER ALL DNODES NK_STRING */ + { 330, -5 }, /* (53) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 346, -1 }, /* (54) dnode_endpoint ::= NK_STRING */ + { 346, -1 }, /* (55) dnode_endpoint ::= NK_ID */ + { 346, -1 }, /* (56) dnode_endpoint ::= NK_IPTOKEN */ + { 347, 0 }, /* (57) force_opt ::= */ + { 347, -1 }, /* (58) force_opt ::= FORCE */ + { 330, -3 }, /* (59) cmd ::= ALTER LOCAL NK_STRING */ + { 330, -4 }, /* (60) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 330, -5 }, /* (61) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (62) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (63) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (64) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (65) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (66) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (67) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (68) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (69) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 330, -4 }, /* (70) cmd ::= DROP DATABASE exists_opt db_name */ + { 330, -2 }, /* (71) cmd ::= USE db_name */ + { 330, -4 }, /* (72) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 330, -3 }, /* (73) cmd ::= FLUSH DATABASE db_name */ + { 330, -4 }, /* (74) cmd ::= TRIM DATABASE db_name speed_opt */ + { 330, -5 }, /* (75) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ + { 348, -3 }, /* (76) not_exists_opt ::= IF NOT EXISTS */ + { 348, 0 }, /* (77) not_exists_opt ::= */ + { 350, -2 }, /* (78) exists_opt ::= IF EXISTS */ + { 350, 0 }, /* (79) exists_opt ::= */ + { 349, 0 }, /* (80) db_options ::= */ + { 349, -3 }, /* (81) db_options ::= db_options BUFFER NK_INTEGER */ + { 349, -3 }, /* (82) db_options ::= db_options CACHEMODEL NK_STRING */ + { 349, -3 }, /* (83) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 349, -3 }, /* (84) db_options ::= db_options COMP NK_INTEGER */ + { 349, -3 }, /* (85) db_options ::= db_options DURATION NK_INTEGER */ + { 349, -3 }, /* (86) db_options ::= db_options DURATION NK_VARIABLE */ + { 349, -3 }, /* (87) db_options ::= db_options MAXROWS NK_INTEGER */ + { 349, -3 }, /* (88) db_options ::= db_options MINROWS NK_INTEGER */ + { 349, -3 }, /* (89) db_options ::= db_options KEEP integer_list */ + { 349, -3 }, /* (90) db_options ::= db_options KEEP variable_list */ + { 349, -3 }, /* (91) db_options ::= db_options PAGES NK_INTEGER */ + { 349, -3 }, /* (92) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 349, -3 }, /* (93) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 349, -3 }, /* (94) db_options ::= db_options PRECISION NK_STRING */ + { 349, -3 }, /* (95) db_options ::= db_options REPLICA NK_INTEGER */ + { 349, -3 }, /* (96) db_options ::= db_options VGROUPS NK_INTEGER */ + { 349, -3 }, /* (97) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 349, -3 }, /* (98) db_options ::= db_options RETENTIONS retention_list */ + { 349, -3 }, /* (99) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 349, -3 }, /* (100) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 349, -3 }, /* (101) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 349, -3 }, /* (102) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 349, -4 }, /* (103) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 349, -3 }, /* (104) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 349, -4 }, /* (105) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 349, -3 }, /* (106) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 349, -3 }, /* (107) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 349, -3 }, /* (108) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 349, -3 }, /* (109) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 349, -3 }, /* (110) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 351, -1 }, /* (111) alter_db_options ::= alter_db_option */ + { 351, -2 }, /* (112) alter_db_options ::= alter_db_options alter_db_option */ + { 358, -2 }, /* (113) alter_db_option ::= BUFFER NK_INTEGER */ + { 358, -2 }, /* (114) alter_db_option ::= CACHEMODEL NK_STRING */ + { 358, -2 }, /* (115) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 358, -2 }, /* (116) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 358, -2 }, /* (117) alter_db_option ::= KEEP integer_list */ + { 358, -2 }, /* (118) alter_db_option ::= KEEP variable_list */ + { 358, -2 }, /* (119) alter_db_option ::= PAGES NK_INTEGER */ + { 358, -2 }, /* (120) alter_db_option ::= REPLICA NK_INTEGER */ + { 358, -2 }, /* (121) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 358, -2 }, /* (122) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 358, -2 }, /* (123) alter_db_option ::= MINROWS NK_INTEGER */ + { 358, -2 }, /* (124) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ + { 358, -3 }, /* (125) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 358, -2 }, /* (126) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ + { 358, -3 }, /* (127) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 355, -1 }, /* (128) integer_list ::= NK_INTEGER */ + { 355, -3 }, /* (129) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 356, -1 }, /* (130) variable_list ::= NK_VARIABLE */ + { 356, -3 }, /* (131) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 357, -1 }, /* (132) retention_list ::= retention */ + { 357, -3 }, /* (133) retention_list ::= retention_list NK_COMMA retention */ + { 359, -3 }, /* (134) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 352, 0 }, /* (135) speed_opt ::= */ + { 352, -2 }, /* (136) speed_opt ::= MAX_SPEED NK_INTEGER */ + { 353, 0 }, /* (137) start_opt ::= */ + { 353, -3 }, /* (138) start_opt ::= START WITH NK_INTEGER */ + { 353, -3 }, /* (139) start_opt ::= START WITH NK_STRING */ + { 353, -4 }, /* (140) start_opt ::= START WITH TIMESTAMP NK_STRING */ + { 354, 0 }, /* (141) end_opt ::= */ + { 354, -3 }, /* (142) end_opt ::= END WITH NK_INTEGER */ + { 354, -3 }, /* (143) end_opt ::= END WITH NK_STRING */ + { 354, -4 }, /* (144) end_opt ::= END WITH TIMESTAMP NK_STRING */ + { 330, -9 }, /* (145) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 330, -3 }, /* (146) cmd ::= CREATE TABLE multi_create_clause */ + { 330, -9 }, /* (147) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 330, -3 }, /* (148) cmd ::= DROP TABLE multi_drop_clause */ + { 330, -4 }, /* (149) cmd ::= DROP STABLE exists_opt full_table_name */ + { 330, -3 }, /* (150) cmd ::= ALTER TABLE alter_table_clause */ + { 330, -3 }, /* (151) cmd ::= ALTER STABLE alter_table_clause */ + { 367, -2 }, /* (152) alter_table_clause ::= full_table_name alter_table_options */ + { 367, -5 }, /* (153) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 367, -4 }, /* (154) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 367, -5 }, /* (155) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 367, -5 }, /* (156) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 367, -5 }, /* (157) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 367, -4 }, /* (158) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 367, -5 }, /* (159) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 367, -5 }, /* (160) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 367, -6 }, /* (161) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 364, -1 }, /* (162) multi_create_clause ::= create_subtable_clause */ + { 364, -2 }, /* (163) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 372, -10 }, /* (164) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 366, -1 }, /* (165) multi_drop_clause ::= drop_table_clause */ + { 366, -3 }, /* (166) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + { 375, -2 }, /* (167) drop_table_clause ::= exists_opt full_table_name */ + { 373, 0 }, /* (168) specific_cols_opt ::= */ + { 373, -3 }, /* (169) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 360, -1 }, /* (170) full_table_name ::= table_name */ + { 360, -3 }, /* (171) full_table_name ::= db_name NK_DOT table_name */ + { 361, -1 }, /* (172) column_def_list ::= column_def */ + { 361, -3 }, /* (173) column_def_list ::= column_def_list NK_COMMA column_def */ + { 377, -2 }, /* (174) column_def ::= column_name type_name */ + { 370, -1 }, /* (175) type_name ::= BOOL */ + { 370, -1 }, /* (176) type_name ::= TINYINT */ + { 370, -1 }, /* (177) type_name ::= SMALLINT */ + { 370, -1 }, /* (178) type_name ::= INT */ + { 370, -1 }, /* (179) type_name ::= INTEGER */ + { 370, -1 }, /* (180) type_name ::= BIGINT */ + { 370, -1 }, /* (181) type_name ::= FLOAT */ + { 370, -1 }, /* (182) type_name ::= DOUBLE */ + { 370, -4 }, /* (183) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (184) type_name ::= TIMESTAMP */ + { 370, -4 }, /* (185) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 370, -2 }, /* (186) type_name ::= TINYINT UNSIGNED */ + { 370, -2 }, /* (187) type_name ::= SMALLINT UNSIGNED */ + { 370, -2 }, /* (188) type_name ::= INT UNSIGNED */ + { 370, -2 }, /* (189) type_name ::= BIGINT UNSIGNED */ + { 370, -1 }, /* (190) type_name ::= JSON */ + { 370, -4 }, /* (191) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (192) type_name ::= MEDIUMBLOB */ + { 370, -1 }, /* (193) type_name ::= BLOB */ + { 370, -4 }, /* (194) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (195) type_name ::= DECIMAL */ + { 370, -4 }, /* (196) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 370, -6 }, /* (197) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 362, 0 }, /* (198) tags_def_opt ::= */ + { 362, -1 }, /* (199) tags_def_opt ::= tags_def */ + { 365, -4 }, /* (200) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 363, 0 }, /* (201) table_options ::= */ + { 363, -3 }, /* (202) table_options ::= table_options COMMENT NK_STRING */ + { 363, -3 }, /* (203) table_options ::= table_options MAX_DELAY duration_list */ + { 363, -3 }, /* (204) table_options ::= table_options WATERMARK duration_list */ + { 363, -5 }, /* (205) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 363, -3 }, /* (206) table_options ::= table_options TTL NK_INTEGER */ + { 363, -5 }, /* (207) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 363, -3 }, /* (208) table_options ::= table_options DELETE_MARK duration_list */ + { 368, -1 }, /* (209) alter_table_options ::= alter_table_option */ + { 368, -2 }, /* (210) alter_table_options ::= alter_table_options alter_table_option */ + { 380, -2 }, /* (211) alter_table_option ::= COMMENT NK_STRING */ + { 380, -2 }, /* (212) alter_table_option ::= TTL NK_INTEGER */ + { 378, -1 }, /* (213) duration_list ::= duration_literal */ + { 378, -3 }, /* (214) duration_list ::= duration_list NK_COMMA duration_literal */ + { 379, -1 }, /* (215) rollup_func_list ::= rollup_func_name */ + { 379, -3 }, /* (216) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 382, -1 }, /* (217) rollup_func_name ::= function_name */ + { 382, -1 }, /* (218) rollup_func_name ::= FIRST */ + { 382, -1 }, /* (219) rollup_func_name ::= LAST */ + { 376, -1 }, /* (220) col_name_list ::= col_name */ + { 376, -3 }, /* (221) col_name_list ::= col_name_list NK_COMMA col_name */ + { 384, -1 }, /* (222) col_name ::= column_name */ + { 330, -2 }, /* (223) cmd ::= SHOW DNODES */ + { 330, -2 }, /* (224) cmd ::= SHOW USERS */ + { 330, -3 }, /* (225) cmd ::= SHOW USER PRIVILEGES */ + { 330, -2 }, /* (226) cmd ::= SHOW DATABASES */ + { 330, -4 }, /* (227) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 330, -4 }, /* (228) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 330, -3 }, /* (229) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 330, -2 }, /* (230) cmd ::= SHOW MNODES */ + { 330, -2 }, /* (231) cmd ::= SHOW QNODES */ + { 330, -2 }, /* (232) cmd ::= SHOW FUNCTIONS */ + { 330, -5 }, /* (233) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 330, -2 }, /* (234) cmd ::= SHOW STREAMS */ + { 330, -2 }, /* (235) cmd ::= SHOW ACCOUNTS */ + { 330, -2 }, /* (236) cmd ::= SHOW APPS */ + { 330, -2 }, /* (237) cmd ::= SHOW CONNECTIONS */ + { 330, -2 }, /* (238) cmd ::= SHOW LICENCES */ + { 330, -2 }, /* (239) cmd ::= SHOW GRANTS */ + { 330, -4 }, /* (240) cmd ::= SHOW CREATE DATABASE db_name */ + { 330, -4 }, /* (241) cmd ::= SHOW CREATE TABLE full_table_name */ + { 330, -4 }, /* (242) cmd ::= SHOW CREATE STABLE full_table_name */ + { 330, -2 }, /* (243) cmd ::= SHOW QUERIES */ + { 330, -2 }, /* (244) cmd ::= SHOW SCORES */ + { 330, -2 }, /* (245) cmd ::= SHOW TOPICS */ + { 330, -2 }, /* (246) cmd ::= SHOW VARIABLES */ + { 330, -3 }, /* (247) cmd ::= SHOW CLUSTER VARIABLES */ + { 330, -3 }, /* (248) cmd ::= SHOW LOCAL VARIABLES */ + { 330, -5 }, /* (249) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 330, -2 }, /* (250) cmd ::= SHOW BNODES */ + { 330, -2 }, /* (251) cmd ::= SHOW SNODES */ + { 330, -2 }, /* (252) cmd ::= SHOW CLUSTER */ + { 330, -2 }, /* (253) cmd ::= SHOW TRANSACTIONS */ + { 330, -4 }, /* (254) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 330, -2 }, /* (255) cmd ::= SHOW CONSUMERS */ + { 330, -2 }, /* (256) cmd ::= SHOW SUBSCRIPTIONS */ + { 330, -5 }, /* (257) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 330, -7 }, /* (258) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 330, -3 }, /* (259) cmd ::= SHOW VNODES NK_INTEGER */ + { 330, -3 }, /* (260) cmd ::= SHOW VNODES NK_STRING */ + { 330, -3 }, /* (261) cmd ::= SHOW db_name_cond_opt ALIVE */ + { 330, -3 }, /* (262) cmd ::= SHOW CLUSTER ALIVE */ + { 385, 0 }, /* (263) db_name_cond_opt ::= */ + { 385, -2 }, /* (264) db_name_cond_opt ::= db_name NK_DOT */ + { 386, 0 }, /* (265) like_pattern_opt ::= */ + { 386, -2 }, /* (266) like_pattern_opt ::= LIKE NK_STRING */ + { 387, -1 }, /* (267) table_name_cond ::= table_name */ + { 388, 0 }, /* (268) from_db_opt ::= */ + { 388, -2 }, /* (269) from_db_opt ::= FROM db_name */ + { 389, 0 }, /* (270) tag_list_opt ::= */ + { 389, -1 }, /* (271) tag_list_opt ::= tag_item */ + { 389, -3 }, /* (272) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 390, -1 }, /* (273) tag_item ::= TBNAME */ + { 390, -1 }, /* (274) tag_item ::= QTAGS */ + { 390, -1 }, /* (275) tag_item ::= column_name */ + { 390, -2 }, /* (276) tag_item ::= column_name column_alias */ + { 390, -3 }, /* (277) tag_item ::= column_name AS column_alias */ + { 330, -8 }, /* (278) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + { 330, -9 }, /* (279) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + { 330, -4 }, /* (280) cmd ::= DROP INDEX exists_opt full_index_name */ + { 392, -1 }, /* (281) full_index_name ::= index_name */ + { 392, -3 }, /* (282) full_index_name ::= db_name NK_DOT index_name */ + { 393, -10 }, /* (283) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 393, -12 }, /* (284) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 395, -1 }, /* (285) func_list ::= func */ + { 395, -3 }, /* (286) func_list ::= func_list NK_COMMA func */ + { 398, -4 }, /* (287) func ::= sma_func_name NK_LP expression_list NK_RP */ + { 399, -1 }, /* (288) sma_func_name ::= function_name */ + { 399, -1 }, /* (289) sma_func_name ::= COUNT */ + { 399, -1 }, /* (290) sma_func_name ::= FIRST */ + { 399, -1 }, /* (291) sma_func_name ::= LAST */ + { 399, -1 }, /* (292) sma_func_name ::= LAST_ROW */ + { 397, 0 }, /* (293) sma_stream_opt ::= */ + { 397, -3 }, /* (294) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 397, -3 }, /* (295) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 397, -3 }, /* (296) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 330, -6 }, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 330, -7 }, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 330, -9 }, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 330, -7 }, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 330, -9 }, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 330, -4 }, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ + { 330, -7 }, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 330, -2 }, /* (304) cmd ::= DESC full_table_name */ + { 330, -2 }, /* (305) cmd ::= DESCRIBE full_table_name */ + { 330, -3 }, /* (306) cmd ::= RESET QUERY CACHE */ + { 330, -4 }, /* (307) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 330, -4 }, /* (308) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + { 402, 0 }, /* (309) analyze_opt ::= */ + { 402, -1 }, /* (310) analyze_opt ::= ANALYZE */ + { 403, 0 }, /* (311) explain_options ::= */ + { 403, -3 }, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 403, -3 }, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ + { 330, -12 }, /* (314) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + { 330, -4 }, /* (315) cmd ::= DROP FUNCTION exists_opt function_name */ + { 406, 0 }, /* (316) agg_func_opt ::= */ + { 406, -1 }, /* (317) agg_func_opt ::= AGGREGATE */ + { 407, 0 }, /* (318) bufsize_opt ::= */ + { 407, -2 }, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 408, 0 }, /* (320) language_opt ::= */ + { 408, -2 }, /* (321) language_opt ::= LANGUAGE NK_STRING */ + { 405, 0 }, /* (322) or_replace_opt ::= */ + { 405, -2 }, /* (323) or_replace_opt ::= OR REPLACE */ + { 330, -12 }, /* (324) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + { 330, -4 }, /* (325) cmd ::= DROP STREAM exists_opt stream_name */ + { 411, 0 }, /* (326) col_list_opt ::= */ + { 411, -3 }, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ + { 412, 0 }, /* (328) tag_def_or_ref_opt ::= */ + { 412, -1 }, /* (329) tag_def_or_ref_opt ::= tags_def */ + { 412, -4 }, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + { 410, 0 }, /* (331) stream_options ::= */ + { 410, -3 }, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 410, -3 }, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 410, -4 }, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 410, -3 }, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ + { 410, -4 }, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 410, -3 }, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 410, -3 }, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ + { 410, -4 }, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + { 413, 0 }, /* (340) subtable_opt ::= */ + { 413, -4 }, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 330, -3 }, /* (342) cmd ::= KILL CONNECTION NK_INTEGER */ + { 330, -3 }, /* (343) cmd ::= KILL QUERY NK_STRING */ + { 330, -3 }, /* (344) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 330, -2 }, /* (345) cmd ::= BALANCE VGROUP */ + { 330, -3 }, /* (346) cmd ::= BALANCE VGROUP LEADER */ + { 330, -4 }, /* (347) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 330, -4 }, /* (348) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 330, -3 }, /* (349) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 415, -2 }, /* (350) dnode_list ::= DNODE NK_INTEGER */ + { 415, -3 }, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 330, -4 }, /* (352) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 330, -1 }, /* (353) cmd ::= query_or_subquery */ + { 330, -1 }, /* (354) cmd ::= insert_query */ + { 404, -7 }, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 404, -4 }, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + { 333, -1 }, /* (357) literal ::= NK_INTEGER */ + { 333, -1 }, /* (358) literal ::= NK_FLOAT */ + { 333, -1 }, /* (359) literal ::= NK_STRING */ + { 333, -1 }, /* (360) literal ::= NK_BOOL */ + { 333, -2 }, /* (361) literal ::= TIMESTAMP NK_STRING */ + { 333, -1 }, /* (362) literal ::= duration_literal */ + { 333, -1 }, /* (363) literal ::= NULL */ + { 333, -1 }, /* (364) literal ::= NK_QUESTION */ + { 381, -1 }, /* (365) duration_literal ::= NK_VARIABLE */ + { 417, -1 }, /* (366) signed ::= NK_INTEGER */ + { 417, -2 }, /* (367) signed ::= NK_PLUS NK_INTEGER */ + { 417, -2 }, /* (368) signed ::= NK_MINUS NK_INTEGER */ + { 417, -1 }, /* (369) signed ::= NK_FLOAT */ + { 417, -2 }, /* (370) signed ::= NK_PLUS NK_FLOAT */ + { 417, -2 }, /* (371) signed ::= NK_MINUS NK_FLOAT */ + { 371, -1 }, /* (372) signed_literal ::= signed */ + { 371, -1 }, /* (373) signed_literal ::= NK_STRING */ + { 371, -1 }, /* (374) signed_literal ::= NK_BOOL */ + { 371, -2 }, /* (375) signed_literal ::= TIMESTAMP NK_STRING */ + { 371, -1 }, /* (376) signed_literal ::= duration_literal */ + { 371, -1 }, /* (377) signed_literal ::= NULL */ + { 371, -1 }, /* (378) signed_literal ::= literal_func */ + { 371, -1 }, /* (379) signed_literal ::= NK_QUESTION */ + { 419, -1 }, /* (380) literal_list ::= signed_literal */ + { 419, -3 }, /* (381) literal_list ::= literal_list NK_COMMA signed_literal */ + { 342, -1 }, /* (382) db_name ::= NK_ID */ + { 343, -1 }, /* (383) table_name ::= NK_ID */ + { 369, -1 }, /* (384) column_name ::= NK_ID */ + { 383, -1 }, /* (385) function_name ::= NK_ID */ + { 420, -1 }, /* (386) table_alias ::= NK_ID */ + { 391, -1 }, /* (387) column_alias ::= NK_ID */ + { 335, -1 }, /* (388) user_name ::= NK_ID */ + { 344, -1 }, /* (389) topic_name ::= NK_ID */ + { 409, -1 }, /* (390) stream_name ::= NK_ID */ + { 401, -1 }, /* (391) cgroup_name ::= NK_ID */ + { 394, -1 }, /* (392) index_name ::= NK_ID */ + { 421, -1 }, /* (393) expr_or_subquery ::= expression */ + { 414, -1 }, /* (394) expression ::= literal */ + { 414, -1 }, /* (395) expression ::= pseudo_column */ + { 414, -1 }, /* (396) expression ::= column_reference */ + { 414, -1 }, /* (397) expression ::= function_expression */ + { 414, -1 }, /* (398) expression ::= case_when_expression */ + { 414, -3 }, /* (399) expression ::= NK_LP expression NK_RP */ + { 414, -2 }, /* (400) expression ::= NK_PLUS expr_or_subquery */ + { 414, -2 }, /* (401) expression ::= NK_MINUS expr_or_subquery */ + { 414, -3 }, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 414, -3 }, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 414, -3 }, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 414, -3 }, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 414, -3 }, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 414, -3 }, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ + { 414, -3 }, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 414, -3 }, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 374, -1 }, /* (410) expression_list ::= expr_or_subquery */ + { 374, -3 }, /* (411) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 423, -1 }, /* (412) column_reference ::= column_name */ + { 423, -3 }, /* (413) column_reference ::= table_name NK_DOT column_name */ + { 422, -1 }, /* (414) pseudo_column ::= ROWTS */ + { 422, -1 }, /* (415) pseudo_column ::= TBNAME */ + { 422, -3 }, /* (416) pseudo_column ::= table_name NK_DOT TBNAME */ + { 422, -1 }, /* (417) pseudo_column ::= QSTART */ + { 422, -1 }, /* (418) pseudo_column ::= QEND */ + { 422, -1 }, /* (419) pseudo_column ::= QDURATION */ + { 422, -1 }, /* (420) pseudo_column ::= WSTART */ + { 422, -1 }, /* (421) pseudo_column ::= WEND */ + { 422, -1 }, /* (422) pseudo_column ::= WDURATION */ + { 422, -1 }, /* (423) pseudo_column ::= IROWTS */ + { 422, -1 }, /* (424) pseudo_column ::= ISFILLED */ + { 422, -1 }, /* (425) pseudo_column ::= QTAGS */ + { 424, -4 }, /* (426) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 424, -4 }, /* (427) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 424, -6 }, /* (428) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 424, -1 }, /* (429) function_expression ::= literal_func */ + { 418, -3 }, /* (430) literal_func ::= noarg_func NK_LP NK_RP */ + { 418, -1 }, /* (431) literal_func ::= NOW */ + { 428, -1 }, /* (432) noarg_func ::= NOW */ + { 428, -1 }, /* (433) noarg_func ::= TODAY */ + { 428, -1 }, /* (434) noarg_func ::= TIMEZONE */ + { 428, -1 }, /* (435) noarg_func ::= DATABASE */ + { 428, -1 }, /* (436) noarg_func ::= CLIENT_VERSION */ + { 428, -1 }, /* (437) noarg_func ::= SERVER_VERSION */ + { 428, -1 }, /* (438) noarg_func ::= SERVER_STATUS */ + { 428, -1 }, /* (439) noarg_func ::= CURRENT_USER */ + { 428, -1 }, /* (440) noarg_func ::= USER */ + { 426, -1 }, /* (441) star_func ::= COUNT */ + { 426, -1 }, /* (442) star_func ::= FIRST */ + { 426, -1 }, /* (443) star_func ::= LAST */ + { 426, -1 }, /* (444) star_func ::= LAST_ROW */ + { 427, -1 }, /* (445) star_func_para_list ::= NK_STAR */ + { 427, -1 }, /* (446) star_func_para_list ::= other_para_list */ + { 429, -1 }, /* (447) other_para_list ::= star_func_para */ + { 429, -3 }, /* (448) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 430, -1 }, /* (449) star_func_para ::= expr_or_subquery */ + { 430, -3 }, /* (450) star_func_para ::= table_name NK_DOT NK_STAR */ + { 425, -4 }, /* (451) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 425, -5 }, /* (452) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 431, -1 }, /* (453) when_then_list ::= when_then_expr */ + { 431, -2 }, /* (454) when_then_list ::= when_then_list when_then_expr */ + { 434, -4 }, /* (455) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 432, 0 }, /* (456) case_when_else_opt ::= */ + { 432, -2 }, /* (457) case_when_else_opt ::= ELSE common_expression */ + { 435, -3 }, /* (458) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 435, -5 }, /* (459) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 435, -6 }, /* (460) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 435, -3 }, /* (461) predicate ::= expr_or_subquery IS NULL */ + { 435, -4 }, /* (462) predicate ::= expr_or_subquery IS NOT NULL */ + { 435, -3 }, /* (463) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 436, -1 }, /* (464) compare_op ::= NK_LT */ + { 436, -1 }, /* (465) compare_op ::= NK_GT */ + { 436, -1 }, /* (466) compare_op ::= NK_LE */ + { 436, -1 }, /* (467) compare_op ::= NK_GE */ + { 436, -1 }, /* (468) compare_op ::= NK_NE */ + { 436, -1 }, /* (469) compare_op ::= NK_EQ */ + { 436, -1 }, /* (470) compare_op ::= LIKE */ + { 436, -2 }, /* (471) compare_op ::= NOT LIKE */ + { 436, -1 }, /* (472) compare_op ::= MATCH */ + { 436, -1 }, /* (473) compare_op ::= NMATCH */ + { 436, -1 }, /* (474) compare_op ::= CONTAINS */ + { 437, -1 }, /* (475) in_op ::= IN */ + { 437, -2 }, /* (476) in_op ::= NOT IN */ + { 438, -3 }, /* (477) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 439, -1 }, /* (478) boolean_value_expression ::= boolean_primary */ + { 439, -2 }, /* (479) boolean_value_expression ::= NOT boolean_primary */ + { 439, -3 }, /* (480) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 439, -3 }, /* (481) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 440, -1 }, /* (482) boolean_primary ::= predicate */ + { 440, -3 }, /* (483) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 433, -1 }, /* (484) common_expression ::= expr_or_subquery */ + { 433, -1 }, /* (485) common_expression ::= boolean_value_expression */ + { 441, 0 }, /* (486) from_clause_opt ::= */ + { 441, -2 }, /* (487) from_clause_opt ::= FROM table_reference_list */ + { 442, -1 }, /* (488) table_reference_list ::= table_reference */ + { 442, -3 }, /* (489) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 443, -1 }, /* (490) table_reference ::= table_primary */ + { 443, -1 }, /* (491) table_reference ::= joined_table */ + { 444, -2 }, /* (492) table_primary ::= table_name alias_opt */ + { 444, -4 }, /* (493) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 444, -2 }, /* (494) table_primary ::= subquery alias_opt */ + { 444, -1 }, /* (495) table_primary ::= parenthesized_joined_table */ + { 446, 0 }, /* (496) alias_opt ::= */ + { 446, -1 }, /* (497) alias_opt ::= table_alias */ + { 446, -2 }, /* (498) alias_opt ::= AS table_alias */ + { 448, -3 }, /* (499) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 448, -3 }, /* (500) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 445, -6 }, /* (501) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 449, 0 }, /* (502) join_type ::= */ + { 449, -1 }, /* (503) join_type ::= INNER */ + { 450, -12 }, /* (504) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 451, 0 }, /* (505) set_quantifier_opt ::= */ + { 451, -1 }, /* (506) set_quantifier_opt ::= DISTINCT */ + { 451, -1 }, /* (507) set_quantifier_opt ::= ALL */ + { 452, -1 }, /* (508) select_list ::= select_item */ + { 452, -3 }, /* (509) select_list ::= select_list NK_COMMA select_item */ + { 460, -1 }, /* (510) select_item ::= NK_STAR */ + { 460, -1 }, /* (511) select_item ::= common_expression */ + { 460, -2 }, /* (512) select_item ::= common_expression column_alias */ + { 460, -3 }, /* (513) select_item ::= common_expression AS column_alias */ + { 460, -3 }, /* (514) select_item ::= table_name NK_DOT NK_STAR */ + { 416, 0 }, /* (515) where_clause_opt ::= */ + { 416, -2 }, /* (516) where_clause_opt ::= WHERE search_condition */ + { 453, 0 }, /* (517) partition_by_clause_opt ::= */ + { 453, -3 }, /* (518) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 461, -1 }, /* (519) partition_list ::= partition_item */ + { 461, -3 }, /* (520) partition_list ::= partition_list NK_COMMA partition_item */ + { 462, -1 }, /* (521) partition_item ::= expr_or_subquery */ + { 462, -2 }, /* (522) partition_item ::= expr_or_subquery column_alias */ + { 462, -3 }, /* (523) partition_item ::= expr_or_subquery AS column_alias */ + { 457, 0 }, /* (524) twindow_clause_opt ::= */ + { 457, -6 }, /* (525) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 457, -4 }, /* (526) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 457, -6 }, /* (527) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 457, -8 }, /* (528) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 457, -7 }, /* (529) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 396, 0 }, /* (530) sliding_opt ::= */ + { 396, -4 }, /* (531) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 456, 0 }, /* (532) fill_opt ::= */ + { 456, -4 }, /* (533) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + { 463, -1 }, /* (536) fill_mode ::= NONE */ + { 463, -1 }, /* (537) fill_mode ::= PREV */ + { 463, -1 }, /* (538) fill_mode ::= NULL */ + { 463, -1 }, /* (539) fill_mode ::= NULL_F */ + { 463, -1 }, /* (540) fill_mode ::= LINEAR */ + { 463, -1 }, /* (541) fill_mode ::= NEXT */ + { 458, 0 }, /* (542) group_by_clause_opt ::= */ + { 458, -3 }, /* (543) group_by_clause_opt ::= GROUP BY group_by_list */ + { 464, -1 }, /* (544) group_by_list ::= expr_or_subquery */ + { 464, -3 }, /* (545) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 459, 0 }, /* (546) having_clause_opt ::= */ + { 459, -2 }, /* (547) having_clause_opt ::= HAVING search_condition */ + { 454, 0 }, /* (548) range_opt ::= */ + { 454, -6 }, /* (549) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 455, 0 }, /* (550) every_opt ::= */ + { 455, -4 }, /* (551) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 465, -4 }, /* (552) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 466, -1 }, /* (553) query_simple ::= query_specification */ + { 466, -1 }, /* (554) query_simple ::= union_query_expression */ + { 470, -4 }, /* (555) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 470, -3 }, /* (556) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 471, -1 }, /* (557) query_simple_or_subquery ::= query_simple */ + { 471, -1 }, /* (558) query_simple_or_subquery ::= subquery */ + { 400, -1 }, /* (559) query_or_subquery ::= query_expression */ + { 400, -1 }, /* (560) query_or_subquery ::= subquery */ + { 467, 0 }, /* (561) order_by_clause_opt ::= */ + { 467, -3 }, /* (562) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 468, 0 }, /* (563) slimit_clause_opt ::= */ + { 468, -2 }, /* (564) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 468, -4 }, /* (565) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 468, -4 }, /* (566) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 469, 0 }, /* (567) limit_clause_opt ::= */ + { 469, -2 }, /* (568) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 469, -4 }, /* (569) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 469, -4 }, /* (570) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 447, -3 }, /* (571) subquery ::= NK_LP query_expression NK_RP */ + { 447, -3 }, /* (572) subquery ::= NK_LP subquery NK_RP */ + { 345, -1 }, /* (573) search_condition ::= common_expression */ + { 472, -1 }, /* (574) sort_specification_list ::= sort_specification */ + { 472, -3 }, /* (575) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 473, -3 }, /* (576) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 474, 0 }, /* (577) ordering_specification_opt ::= */ + { 474, -1 }, /* (578) ordering_specification_opt ::= ASC */ + { 474, -1 }, /* (579) ordering_specification_opt ::= DESC */ + { 475, 0 }, /* (580) null_ordering_opt ::= */ + { 475, -2 }, /* (581) null_ordering_opt ::= NULLS FIRST */ + { 475, -2 }, /* (582) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3752,11 +3824,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,330,&yymsp[0].minor); + yy_destructor(yypParser,331,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,331,&yymsp[0].minor); + yy_destructor(yypParser,332,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3770,20 +3842,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,330,&yymsp[-2].minor); +{ yy_destructor(yypParser,331,&yymsp[-2].minor); { } - yy_destructor(yypParser,332,&yymsp[0].minor); + yy_destructor(yypParser,333,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,333,&yymsp[0].minor); +{ yy_destructor(yypParser,334,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,331,&yymsp[-1].minor); +{ yy_destructor(yypParser,332,&yymsp[-1].minor); { } - yy_destructor(yypParser,333,&yymsp[0].minor); + yy_destructor(yypParser,334,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3797,1567 +3869,1600 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,332,&yymsp[0].minor); + yy_destructor(yypParser,333,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy63, &yymsp[-1].minor.yy0, yymsp[0].minor.yy707); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy0, yymsp[0].minor.yy551); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy63, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy63, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy63, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy63); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy113); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy707 = 1; } +{ yymsp[1].minor.yy551 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy707 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy551 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy393, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63); } + case 31: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy837, &yymsp[-3].minor.yy777, &yymsp[0].minor.yy113, yymsp[-2].minor.yy448); } break; - case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy393, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63); } + case 32: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy837, &yymsp[-3].minor.yy777, &yymsp[0].minor.yy113, yymsp[-2].minor.yy448); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy393 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy393 = yymsp[0].minor.yy393; } - yymsp[0].minor.yy393 = yylhsminor.yy393; +{ yylhsminor.yy837 = yymsp[0].minor.yy837; } + yymsp[0].minor.yy837 = yylhsminor.yy837; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy393 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy393 = yymsp[-2].minor.yy393 | yymsp[0].minor.yy393; } - yymsp[-2].minor.yy393 = yylhsminor.yy393; +{ yylhsminor.yy837 = yymsp[-2].minor.yy837 | yymsp[0].minor.yy837; } + yymsp[-2].minor.yy837 = yylhsminor.yy837; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy393 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy393 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy63 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy63 = yylhsminor.yy63; +{ yylhsminor.yy777.first = yymsp[-2].minor.yy0; yylhsminor.yy777.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy63 = yymsp[-2].minor.yy63; } - yymsp[-2].minor.yy63 = yylhsminor.yy63; - break; - case 42: /* priv_level ::= topic_name */ - case 281: /* sma_func_name ::= function_name */ yytestcase(yyruleno==281); - case 487: /* alias_opt ::= table_alias */ yytestcase(yyruleno==487); -{ yylhsminor.yy63 = yymsp[0].minor.yy63; } - yymsp[0].minor.yy63 = yylhsminor.yy63; - break; - case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy63, NULL); } - break; - case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy0); } - break; - case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy669); } - break; - case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy63, yymsp[0].minor.yy669); } - break; - case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ +{ yylhsminor.yy777.first = yymsp[-2].minor.yy113; yylhsminor.yy777.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; + break; + case 42: /* priv_level ::= db_name NK_DOT table_name */ +{ yylhsminor.yy777.first = yymsp[-2].minor.yy113; yylhsminor.yy777.second = yymsp[0].minor.yy113; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; + break; + case 43: /* priv_level ::= topic_name */ +{ yylhsminor.yy777.first = yymsp[0].minor.yy113; yylhsminor.yy777.second = nil_token; } + yymsp[0].minor.yy777 = yylhsminor.yy777; + break; + case 44: /* with_opt ::= */ + case 137: /* start_opt ::= */ yytestcase(yyruleno==137); + case 141: /* end_opt ::= */ yytestcase(yyruleno==141); + case 265: /* like_pattern_opt ::= */ yytestcase(yyruleno==265); + case 340: /* subtable_opt ::= */ yytestcase(yyruleno==340); + case 456: /* case_when_else_opt ::= */ yytestcase(yyruleno==456); + case 486: /* from_clause_opt ::= */ yytestcase(yyruleno==486); + case 515: /* where_clause_opt ::= */ yytestcase(yyruleno==515); + case 524: /* twindow_clause_opt ::= */ yytestcase(yyruleno==524); + case 530: /* sliding_opt ::= */ yytestcase(yyruleno==530); + case 532: /* fill_opt ::= */ yytestcase(yyruleno==532); + case 546: /* having_clause_opt ::= */ yytestcase(yyruleno==546); + case 548: /* range_opt ::= */ yytestcase(yyruleno==548); + case 550: /* every_opt ::= */ yytestcase(yyruleno==550); + case 563: /* slimit_clause_opt ::= */ yytestcase(yyruleno==563); + case 567: /* limit_clause_opt ::= */ yytestcase(yyruleno==567); +{ yymsp[1].minor.yy448 = NULL; } + break; + case 45: /* with_opt ::= WITH search_condition */ + case 487: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==487); + case 516: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==516); + case 547: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==547); +{ yymsp[-1].minor.yy448 = yymsp[0].minor.yy448; } + break; + case 46: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy113, NULL); } + break; + case 47: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } + break; + case 48: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy369); } + break; + case 49: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy369); } + break; + case 50: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 48: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + case 51: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 49: /* cmd ::= ALTER ALL DNODES NK_STRING */ + case 52: /* cmd ::= ALTER ALL DNODES NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } break; - case 50: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + case 53: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 51: /* dnode_endpoint ::= NK_STRING */ - case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); - case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 282: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==282); - case 283: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==283); - case 284: /* sma_func_name ::= LAST */ yytestcase(yyruleno==284); - case 285: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==285); - case 372: /* db_name ::= NK_ID */ yytestcase(yyruleno==372); - case 373: /* table_name ::= NK_ID */ yytestcase(yyruleno==373); - case 374: /* column_name ::= NK_ID */ yytestcase(yyruleno==374); - case 375: /* function_name ::= NK_ID */ yytestcase(yyruleno==375); - case 376: /* table_alias ::= NK_ID */ yytestcase(yyruleno==376); - case 377: /* column_alias ::= NK_ID */ yytestcase(yyruleno==377); - case 378: /* user_name ::= NK_ID */ yytestcase(yyruleno==378); - case 379: /* topic_name ::= NK_ID */ yytestcase(yyruleno==379); - case 380: /* stream_name ::= NK_ID */ yytestcase(yyruleno==380); - case 381: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==381); - case 382: /* index_name ::= NK_ID */ yytestcase(yyruleno==382); - case 422: /* noarg_func ::= NOW */ yytestcase(yyruleno==422); - case 423: /* noarg_func ::= TODAY */ yytestcase(yyruleno==423); - case 424: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==424); - case 425: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==425); - case 426: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==426); - case 427: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==427); - case 428: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==428); - case 429: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==429); - case 430: /* noarg_func ::= USER */ yytestcase(yyruleno==430); - case 431: /* star_func ::= COUNT */ yytestcase(yyruleno==431); - case 432: /* star_func ::= FIRST */ yytestcase(yyruleno==432); - case 433: /* star_func ::= LAST */ yytestcase(yyruleno==433); - case 434: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==434); -{ yylhsminor.yy63 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy63 = yylhsminor.yy63; - break; - case 54: /* force_opt ::= */ - case 74: /* not_exists_opt ::= */ yytestcase(yyruleno==74); - case 76: /* exists_opt ::= */ yytestcase(yyruleno==76); - case 302: /* analyze_opt ::= */ yytestcase(yyruleno==302); - case 309: /* agg_func_opt ::= */ yytestcase(yyruleno==309); - case 495: /* set_quantifier_opt ::= */ yytestcase(yyruleno==495); -{ yymsp[1].minor.yy669 = false; } - break; - case 55: /* force_opt ::= FORCE */ - case 303: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==303); - case 310: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==310); - case 496: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==496); -{ yymsp[0].minor.yy669 = true; } - break; - case 56: /* cmd ::= ALTER LOCAL NK_STRING */ + case 54: /* dnode_endpoint ::= NK_STRING */ + case 55: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==55); + case 56: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==56); + case 289: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==289); + case 290: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==290); + case 291: /* sma_func_name ::= LAST */ yytestcase(yyruleno==291); + case 292: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==292); + case 382: /* db_name ::= NK_ID */ yytestcase(yyruleno==382); + case 383: /* table_name ::= NK_ID */ yytestcase(yyruleno==383); + case 384: /* column_name ::= NK_ID */ yytestcase(yyruleno==384); + case 385: /* function_name ::= NK_ID */ yytestcase(yyruleno==385); + case 386: /* table_alias ::= NK_ID */ yytestcase(yyruleno==386); + case 387: /* column_alias ::= NK_ID */ yytestcase(yyruleno==387); + case 388: /* user_name ::= NK_ID */ yytestcase(yyruleno==388); + case 389: /* topic_name ::= NK_ID */ yytestcase(yyruleno==389); + case 390: /* stream_name ::= NK_ID */ yytestcase(yyruleno==390); + case 391: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==391); + case 392: /* index_name ::= NK_ID */ yytestcase(yyruleno==392); + case 432: /* noarg_func ::= NOW */ yytestcase(yyruleno==432); + case 433: /* noarg_func ::= TODAY */ yytestcase(yyruleno==433); + case 434: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==434); + case 435: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==435); + case 436: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==436); + case 437: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==437); + case 438: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==438); + case 439: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==439); + case 440: /* noarg_func ::= USER */ yytestcase(yyruleno==440); + case 441: /* star_func ::= COUNT */ yytestcase(yyruleno==441); + case 442: /* star_func ::= FIRST */ yytestcase(yyruleno==442); + case 443: /* star_func ::= LAST */ yytestcase(yyruleno==443); + case 444: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==444); +{ yylhsminor.yy113 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy113 = yylhsminor.yy113; + break; + case 57: /* force_opt ::= */ + case 77: /* not_exists_opt ::= */ yytestcase(yyruleno==77); + case 79: /* exists_opt ::= */ yytestcase(yyruleno==79); + case 309: /* analyze_opt ::= */ yytestcase(yyruleno==309); + case 316: /* agg_func_opt ::= */ yytestcase(yyruleno==316); + case 322: /* or_replace_opt ::= */ yytestcase(yyruleno==322); + case 505: /* set_quantifier_opt ::= */ yytestcase(yyruleno==505); +{ yymsp[1].minor.yy369 = false; } + break; + case 58: /* force_opt ::= FORCE */ + case 310: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==310); + case 317: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==317); + case 506: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==506); +{ yymsp[0].minor.yy369 = true; } + break; + case 59: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 57: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + case 60: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 58: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + case 61: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 59: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + case 62: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 60: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + case 63: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 61: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + case 64: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 62: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + case 65: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 63: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + case 66: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 64: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + case 67: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 65: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + case 68: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy669, &yymsp[-1].minor.yy63, yymsp[0].minor.yy828); } + case 69: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy369, &yymsp[-1].minor.yy113, yymsp[0].minor.yy448); } + break; + case 70: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } + break; + case 71: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } + break; + case 72: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy448); } + break; + case 73: /* cmd ::= FLUSH DATABASE db_name */ +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } + break; + case 74: /* cmd ::= TRIM DATABASE db_name speed_opt */ +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy788); } break; - case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy669, &yymsp[0].minor.yy63); } + case 75: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy113, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy63); } + case 76: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy369 = true; } break; - case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy63, yymsp[0].minor.yy828); } + case 78: /* exists_opt ::= IF EXISTS */ + case 323: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==323); +{ yymsp[-1].minor.yy369 = true; } break; - case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy63); } + case 80: /* db_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultDatabaseOptions(pCxt); } break; - case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy63, yymsp[0].minor.yy332); } + case 81: /* db_options ::= db_options BUFFER NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 72: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ -{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy63, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 82: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 73: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy669 = true; } + case 83: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 75: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy669 = true; } + case 84: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 77: /* db_options ::= */ -{ yymsp[1].minor.yy828 = createDefaultDatabaseOptions(pCxt); } + case 85: /* db_options ::= db_options DURATION NK_INTEGER */ + case 86: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==86); +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 78: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 87: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 79: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 88: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 80: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 89: /* db_options ::= db_options KEEP integer_list */ + case 90: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==90); +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_KEEP, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 81: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 91: /* db_options ::= db_options PAGES NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 82: /* db_options ::= db_options DURATION NK_INTEGER */ - case 83: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==83); -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 92: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 84: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 93: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 85: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 94: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 86: /* db_options ::= db_options KEEP integer_list */ - case 87: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==87); -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_KEEP, yymsp[0].minor.yy222); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 95: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 88: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 96: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 89: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 97: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 90: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 98: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_RETENTIONS, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 91: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 99: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 92: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 100: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 93: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 101: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 94: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 102: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 95: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_RETENTIONS, yymsp[0].minor.yy222); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 103: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ +{ + SToken t = yymsp[-1].minor.yy0; + t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; + yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-3].minor.yy448, DB_OPTION_WAL_RETENTION_PERIOD, &t); + } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 104: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 105: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ +{ + SToken t = yymsp[-1].minor.yy0; + t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; + yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-3].minor.yy448, DB_OPTION_WAL_RETENTION_SIZE, &t); + } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 106: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 107: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 108: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 109: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 96: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 110: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 97: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 111: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy448 = createAlterDatabaseOptions(pCxt); yylhsminor.yy448 = setAlterDatabaseOption(pCxt, yylhsminor.yy448, &yymsp[0].minor.yy53); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 98: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 112: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy448 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy448, &yymsp[0].minor.yy53); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 113: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 100: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 114: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 115: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 116: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 117: /* alter_db_option ::= KEEP integer_list */ + case 118: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==118); +{ yymsp[-1].minor.yy53.type = DB_OPTION_KEEP; yymsp[-1].minor.yy53.pList = yymsp[0].minor.yy432; } + break; + case 119: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_PAGES; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 120: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 121: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 122: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 123: /* alter_db_option ::= MINROWS NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 124: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } + break; + case 125: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-3].minor.yy828, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yymsp[-2].minor.yy53.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy53.val = t; } - yymsp[-3].minor.yy828 = yylhsminor.yy828; break; - case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 126: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 102: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 127: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-3].minor.yy828, DB_OPTION_WAL_RETENTION_SIZE, &t); + yymsp[-2].minor.yy53.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy53.val = t; } - yymsp[-3].minor.yy828 = yylhsminor.yy828; - break; - case 103: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 104: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 105: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 106: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 107: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy828 = setDatabaseOption(pCxt, yymsp[-2].minor.yy828, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 108: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy828 = createAlterDatabaseOptions(pCxt); yylhsminor.yy828 = setAlterDatabaseOption(pCxt, yylhsminor.yy828, &yymsp[0].minor.yy233); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 109: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy828 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy828, &yymsp[0].minor.yy233); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; - break; - case 110: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 111: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 112: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 113: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 114: /* alter_db_option ::= KEEP integer_list */ - case 115: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==115); -{ yymsp[-1].minor.yy233.type = DB_OPTION_KEEP; yymsp[-1].minor.yy233.pList = yymsp[0].minor.yy222; } - break; - case 116: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_PAGES; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 117: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 118: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_WAL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 119: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } - break; - case 120: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy222 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy222 = yylhsminor.yy222; - break; - case 121: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 341: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==341); -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-2].minor.yy222, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy222 = yylhsminor.yy222; - break; - case 122: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy222 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy222 = yylhsminor.yy222; - break; - case 123: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-2].minor.yy222, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy222 = yylhsminor.yy222; - break; - case 124: /* retention_list ::= retention */ - case 154: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==154); - case 157: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==157); - case 164: /* column_def_list ::= column_def */ yytestcase(yyruleno==164); - case 208: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==208); - case 213: /* col_name_list ::= col_name */ yytestcase(yyruleno==213); - case 264: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==264); - case 278: /* func_list ::= func */ yytestcase(yyruleno==278); - case 370: /* literal_list ::= signed_literal */ yytestcase(yyruleno==370); - case 437: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==437); - case 443: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==443); - case 498: /* select_list ::= select_item */ yytestcase(yyruleno==498); - case 509: /* partition_list ::= partition_item */ yytestcase(yyruleno==509); - case 564: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==564); -{ yylhsminor.yy222 = createNodeList(pCxt, yymsp[0].minor.yy828); } - yymsp[0].minor.yy222 = yylhsminor.yy222; - break; - case 125: /* retention_list ::= retention_list NK_COMMA retention */ - case 158: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==158); - case 165: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==165); - case 209: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==209); - case 214: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==214); - case 265: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==265); - case 279: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==279); - case 371: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==371); - case 438: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==438); - case 499: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==499); - case 510: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==510); - case 565: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==565); -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-2].minor.yy222, yymsp[0].minor.yy828); } - yymsp[-2].minor.yy222 = yylhsminor.yy222; - break; - case 126: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy828 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 127: /* speed_opt ::= */ - case 311: /* bufsize_opt ::= */ yytestcase(yyruleno==311); -{ yymsp[1].minor.yy332 = 0; } - break; - case 128: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 312: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==312); -{ yymsp[-1].minor.yy332 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } - break; - case 129: /* start_opt ::= */ - case 133: /* end_opt ::= */ yytestcase(yyruleno==133); - case 258: /* like_pattern_opt ::= */ yytestcase(yyruleno==258); - case 331: /* subtable_opt ::= */ yytestcase(yyruleno==331); - case 446: /* case_when_else_opt ::= */ yytestcase(yyruleno==446); - case 476: /* from_clause_opt ::= */ yytestcase(yyruleno==476); - case 505: /* where_clause_opt ::= */ yytestcase(yyruleno==505); - case 514: /* twindow_clause_opt ::= */ yytestcase(yyruleno==514); - case 520: /* sliding_opt ::= */ yytestcase(yyruleno==520); - case 522: /* fill_opt ::= */ yytestcase(yyruleno==522); - case 536: /* having_clause_opt ::= */ yytestcase(yyruleno==536); - case 538: /* range_opt ::= */ yytestcase(yyruleno==538); - case 540: /* every_opt ::= */ yytestcase(yyruleno==540); - case 553: /* slimit_clause_opt ::= */ yytestcase(yyruleno==553); - case 557: /* limit_clause_opt ::= */ yytestcase(yyruleno==557); -{ yymsp[1].minor.yy828 = NULL; } - break; - case 130: /* start_opt ::= START WITH NK_INTEGER */ - case 134: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==134); -{ yymsp[-2].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 131: /* start_opt ::= START WITH NK_STRING */ - case 135: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==135); -{ yymsp[-2].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 132: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ - case 136: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==136); -{ yymsp[-3].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 137: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 139: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==139); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy669, yymsp[-5].minor.yy828, yymsp[-3].minor.yy222, yymsp[-1].minor.yy222, yymsp[0].minor.yy828); } - break; - case 138: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy222); } - break; - case 140: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy222); } - break; - case 141: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy669, yymsp[0].minor.yy828); } - break; - case 142: /* cmd ::= ALTER TABLE alter_table_clause */ - case 343: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==343); - case 344: /* cmd ::= insert_query */ yytestcase(yyruleno==344); -{ pCxt->pRootNode = yymsp[0].minor.yy828; } - break; - case 143: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy828); } - break; - case 144: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy828 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; - break; - case 145: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy828 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy63, yymsp[0].minor.yy686); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 146: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy828 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy828, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy63); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; - break; - case 147: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy828 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy63, yymsp[0].minor.yy686); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 148: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy828 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy63, &yymsp[0].minor.yy63); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 149: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy828 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy63, yymsp[0].minor.yy686); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 150: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy828 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy828, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy63); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; - break; - case 151: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy828 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy63, yymsp[0].minor.yy686); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 152: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy828 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy828, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy63, &yymsp[0].minor.yy63); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; break; - case 153: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy828 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy828, &yymsp[-2].minor.yy63, yymsp[0].minor.yy828); } - yymsp[-5].minor.yy828 = yylhsminor.yy828; + case 128: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy432 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 129: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 351: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==351); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 130: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy432 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 131: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 132: /* retention_list ::= retention */ + case 162: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==162); + case 165: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==165); + case 172: /* column_def_list ::= column_def */ yytestcase(yyruleno==172); + case 215: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==215); + case 220: /* col_name_list ::= col_name */ yytestcase(yyruleno==220); + case 271: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==271); + case 285: /* func_list ::= func */ yytestcase(yyruleno==285); + case 380: /* literal_list ::= signed_literal */ yytestcase(yyruleno==380); + case 447: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==447); + case 453: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==453); + case 508: /* select_list ::= select_item */ yytestcase(yyruleno==508); + case 519: /* partition_list ::= partition_item */ yytestcase(yyruleno==519); + case 574: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==574); +{ yylhsminor.yy432 = createNodeList(pCxt, yymsp[0].minor.yy448); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 133: /* retention_list ::= retention_list NK_COMMA retention */ + case 166: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==166); + case 173: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==173); + case 216: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==216); + case 221: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==221); + case 272: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==272); + case 286: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==286); + case 381: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==381); + case 448: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==448); + case 509: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==509); + case 520: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==520); + case 575: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==575); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 134: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy448 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 135: /* speed_opt ::= */ + case 318: /* bufsize_opt ::= */ yytestcase(yyruleno==318); +{ yymsp[1].minor.yy788 = 0; } + break; + case 136: /* speed_opt ::= MAX_SPEED NK_INTEGER */ + case 319: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==319); +{ yymsp[-1].minor.yy788 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + break; + case 138: /* start_opt ::= START WITH NK_INTEGER */ + case 142: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==142); +{ yymsp[-2].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 139: /* start_opt ::= START WITH NK_STRING */ + case 143: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==143); +{ yymsp[-2].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 140: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ + case 144: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==144); +{ yymsp[-3].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 145: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 147: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==147); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy369, yymsp[-5].minor.yy448, yymsp[-3].minor.yy432, yymsp[-1].minor.yy432, yymsp[0].minor.yy448); } + break; + case 146: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy432); } + break; + case 148: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy432); } + break; + case 149: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } + break; + case 150: /* cmd ::= ALTER TABLE alter_table_clause */ + case 353: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==353); + case 354: /* cmd ::= insert_query */ yytestcase(yyruleno==354); +{ pCxt->pRootNode = yymsp[0].minor.yy448; } + break; + case 151: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy448); } + break; + case 152: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy448 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 153: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 154: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy448 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy448, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 155: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 156: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy448 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 157: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 158: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy448 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy448, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 159: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 160: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy448 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 155: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 444: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==444); -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-1].minor.yy222, yymsp[0].minor.yy828); } - yymsp[-1].minor.yy222 = yylhsminor.yy222; + case 161: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy448 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy448, &yymsp[-2].minor.yy113, yymsp[0].minor.yy448); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 156: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy828 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy669, yymsp[-8].minor.yy828, yymsp[-6].minor.yy828, yymsp[-5].minor.yy222, yymsp[-2].minor.yy222, yymsp[0].minor.yy828); } - yymsp[-9].minor.yy828 = yylhsminor.yy828; + case 163: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 454: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==454); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-1].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy432 = yylhsminor.yy432; break; - case 159: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy828 = createDropTableClause(pCxt, yymsp[-1].minor.yy669, yymsp[0].minor.yy828); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 164: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy448 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy369, yymsp[-8].minor.yy448, yymsp[-6].minor.yy448, yymsp[-5].minor.yy432, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-9].minor.yy448 = yylhsminor.yy448; break; - case 160: /* specific_cols_opt ::= */ - case 191: /* tags_def_opt ::= */ yytestcase(yyruleno==191); - case 263: /* tag_list_opt ::= */ yytestcase(yyruleno==263); - case 317: /* col_list_opt ::= */ yytestcase(yyruleno==317); - case 319: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==319); - case 507: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==507); - case 532: /* group_by_clause_opt ::= */ yytestcase(yyruleno==532); - case 551: /* order_by_clause_opt ::= */ yytestcase(yyruleno==551); -{ yymsp[1].minor.yy222 = NULL; } + case 167: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy448 = createDropTableClause(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 161: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ - case 318: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==318); -{ yymsp[-2].minor.yy222 = yymsp[-1].minor.yy222; } + case 168: /* specific_cols_opt ::= */ + case 198: /* tags_def_opt ::= */ yytestcase(yyruleno==198); + case 270: /* tag_list_opt ::= */ yytestcase(yyruleno==270); + case 326: /* col_list_opt ::= */ yytestcase(yyruleno==326); + case 328: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==328); + case 517: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==517); + case 542: /* group_by_clause_opt ::= */ yytestcase(yyruleno==542); + case 561: /* order_by_clause_opt ::= */ yytestcase(yyruleno==561); +{ yymsp[1].minor.yy432 = NULL; } break; - case 162: /* full_table_name ::= table_name */ -{ yylhsminor.yy828 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy63, NULL); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 169: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ + case 327: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==327); +{ yymsp[-2].minor.yy432 = yymsp[-1].minor.yy432; } break; - case 163: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy828 = createRealTableNode(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63, NULL); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 170: /* full_table_name ::= table_name */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy113, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 166: /* column_def ::= column_name type_name */ -{ yylhsminor.yy828 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy63, yymsp[0].minor.yy686, NULL); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 171: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 167: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy828 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy63, yymsp[-2].minor.yy686, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 174: /* column_def ::= column_name type_name */ +{ yylhsminor.yy448 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728, NULL); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 168: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 175: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 169: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 176: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 170: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 177: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 171: /* type_name ::= INT */ - case 172: /* type_name ::= INTEGER */ yytestcase(yyruleno==172); -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_INT); } + case 178: /* type_name ::= INT */ + case 179: /* type_name ::= INTEGER */ yytestcase(yyruleno==179); +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 173: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 180: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 174: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 181: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 175: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 182: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 176: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy686 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 183: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 177: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 184: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 178: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy686 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 185: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 179: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy686 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 186: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 180: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy686 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 187: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 181: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy686 = createDataType(TSDB_DATA_TYPE_UINT); } + case 188: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 182: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy686 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 189: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 183: /* type_name ::= JSON */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_JSON); } + case 190: /* type_name ::= JSON */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 184: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy686 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 191: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 185: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 192: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 186: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 193: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 187: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy686 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 194: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 188: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy686 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 195: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 189: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy686 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 196: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 190: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy686 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 197: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 192: /* tags_def_opt ::= tags_def */ - case 320: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==320); - case 436: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==436); -{ yylhsminor.yy222 = yymsp[0].minor.yy222; } - yymsp[0].minor.yy222 = yylhsminor.yy222; + case 199: /* tags_def_opt ::= tags_def */ + case 329: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==329); + case 446: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==446); +{ yylhsminor.yy432 = yymsp[0].minor.yy432; } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 193: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ - case 321: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==321); -{ yymsp[-3].minor.yy222 = yymsp[-1].minor.yy222; } + case 200: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ + case 330: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==330); +{ yymsp[-3].minor.yy432 = yymsp[-1].minor.yy432; } break; - case 194: /* table_options ::= */ -{ yymsp[1].minor.yy828 = createDefaultTableOptions(pCxt); } + case 201: /* table_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultTableOptions(pCxt); } break; - case 195: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-2].minor.yy828, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 202: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 196: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-2].minor.yy828, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy222); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 203: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 197: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-2].minor.yy828, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy222); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 204: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 198: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-4].minor.yy828, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy222); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; + case 205: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-4].minor.yy448, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy432); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 199: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-2].minor.yy828, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 206: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 200: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-4].minor.yy828, TABLE_OPTION_SMA, yymsp[-1].minor.yy222); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; + case 207: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-4].minor.yy448, TABLE_OPTION_SMA, yymsp[-1].minor.yy432); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 201: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-2].minor.yy828, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy222); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 208: /* table_options ::= table_options DELETE_MARK duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 202: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy828 = createAlterTableOptions(pCxt); yylhsminor.yy828 = setTableOption(pCxt, yylhsminor.yy828, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 209: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy448 = createAlterTableOptions(pCxt); yylhsminor.yy448 = setTableOption(pCxt, yylhsminor.yy448, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 203: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy828 = setTableOption(pCxt, yymsp[-1].minor.yy828, yymsp[0].minor.yy233.type, &yymsp[0].minor.yy233.val); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 210: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 204: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy233.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } + case 211: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy53.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 205: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy233.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy233.val = yymsp[0].minor.yy0; } + case 212: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 206: /* duration_list ::= duration_literal */ - case 400: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==400); -{ yylhsminor.yy222 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - yymsp[0].minor.yy222 = yylhsminor.yy222; + case 213: /* duration_list ::= duration_literal */ + case 410: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==410); +{ yylhsminor.yy432 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 207: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 401: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==401); -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-2].minor.yy222, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - yymsp[-2].minor.yy222 = yylhsminor.yy222; + case 214: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 411: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==411); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; break; - case 210: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy828 = createFunctionNode(pCxt, &yymsp[0].minor.yy63, NULL); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 217: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[0].minor.yy113, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 211: /* rollup_func_name ::= FIRST */ - case 212: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==212); - case 267: /* tag_item ::= QTAGS */ yytestcase(yyruleno==267); -{ yylhsminor.yy828 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 218: /* rollup_func_name ::= FIRST */ + case 219: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==219); + case 274: /* tag_item ::= QTAGS */ yytestcase(yyruleno==274); +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 215: /* col_name ::= column_name */ - case 268: /* tag_item ::= column_name */ yytestcase(yyruleno==268); -{ yylhsminor.yy828 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy63); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 222: /* col_name ::= column_name */ + case 275: /* tag_item ::= column_name */ yytestcase(yyruleno==275); +{ yylhsminor.yy448 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 216: /* cmd ::= SHOW DNODES */ + case 223: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 217: /* cmd ::= SHOW USERS */ + case 224: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 218: /* cmd ::= SHOW USER PRIVILEGES */ + case 225: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } break; - case 219: /* cmd ::= SHOW DATABASES */ + case 226: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 220: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy828, yymsp[0].minor.yy828, OP_TYPE_LIKE); } + case 227: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, OP_TYPE_LIKE); } break; - case 221: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy828, yymsp[0].minor.yy828, OP_TYPE_LIKE); } + case 228: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, OP_TYPE_LIKE); } break; - case 222: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy828, NULL, OP_TYPE_LIKE); } + case 229: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy448, NULL, OP_TYPE_LIKE); } break; - case 223: /* cmd ::= SHOW MNODES */ + case 230: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 224: /* cmd ::= SHOW QNODES */ + case 231: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 225: /* cmd ::= SHOW FUNCTIONS */ + case 232: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 226: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy828, yymsp[-1].minor.yy828, OP_TYPE_EQUAL); } + case 233: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy448, yymsp[-1].minor.yy448, OP_TYPE_EQUAL); } break; - case 227: /* cmd ::= SHOW STREAMS */ + case 234: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 228: /* cmd ::= SHOW ACCOUNTS */ + case 235: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 229: /* cmd ::= SHOW APPS */ + case 236: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 230: /* cmd ::= SHOW CONNECTIONS */ + case 237: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 231: /* cmd ::= SHOW LICENCES */ - case 232: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==232); + case 238: /* cmd ::= SHOW LICENCES */ + case 239: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==239); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 233: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy63); } + case 240: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } break; - case 234: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy828); } + case 241: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy448); } break; - case 235: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy828); } + case 242: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy448); } break; - case 236: /* cmd ::= SHOW QUERIES */ + case 243: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 237: /* cmd ::= SHOW SCORES */ + case 244: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 238: /* cmd ::= SHOW TOPICS */ + case 245: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 239: /* cmd ::= SHOW VARIABLES */ - case 240: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==240); + case 246: /* cmd ::= SHOW VARIABLES */ + case 247: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==247); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 241: /* cmd ::= SHOW LOCAL VARIABLES */ + case 248: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 242: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy828); } + case 249: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy448); } break; - case 243: /* cmd ::= SHOW BNODES */ + case 250: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 244: /* cmd ::= SHOW SNODES */ + case 251: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 245: /* cmd ::= SHOW CLUSTER */ + case 252: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 246: /* cmd ::= SHOW TRANSACTIONS */ + case 253: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 247: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy828); } + case 254: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy448); } break; - case 248: /* cmd ::= SHOW CONSUMERS */ + case 255: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 249: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 256: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 250: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy828, yymsp[-1].minor.yy828, OP_TYPE_EQUAL); } + case 257: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy448, yymsp[-1].minor.yy448, OP_TYPE_EQUAL); } break; - case 251: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy828, yymsp[0].minor.yy828, yymsp[-3].minor.yy222); } + case 258: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448, yymsp[-3].minor.yy432); } break; - case 252: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 259: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 253: /* cmd ::= SHOW VNODES NK_STRING */ + case 260: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 254: /* cmd ::= SHOW db_name_cond_opt ALIVE */ -{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy828, QUERY_NODE_SHOW_DB_ALIVE_STMT); } + case 261: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy448, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 255: /* cmd ::= SHOW CLUSTER ALIVE */ + case 262: /* cmd ::= SHOW CLUSTER ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 256: /* db_name_cond_opt ::= */ - case 261: /* from_db_opt ::= */ yytestcase(yyruleno==261); -{ yymsp[1].minor.yy828 = createDefaultDatabaseCondValue(pCxt); } + case 263: /* db_name_cond_opt ::= */ + case 268: /* from_db_opt ::= */ yytestcase(yyruleno==268); +{ yymsp[1].minor.yy448 = createDefaultDatabaseCondValue(pCxt); } break; - case 257: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy828 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy63); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 264: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy448 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 259: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 266: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 260: /* table_name_cond ::= table_name */ -{ yylhsminor.yy828 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy63); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 267: /* table_name_cond ::= table_name */ +{ yylhsminor.yy448 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 262: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy828 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy63); } + case 269: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy448 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy113); } break; - case 266: /* tag_item ::= TBNAME */ -{ yylhsminor.yy828 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 273: /* tag_item ::= TBNAME */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 269: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy828 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy63), &yymsp[0].minor.yy63); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 276: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy113), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 270: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy828 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy63), &yymsp[0].minor.yy63); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 277: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy113), &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 271: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy669, yymsp[-3].minor.yy828, yymsp[-1].minor.yy828, NULL, yymsp[0].minor.yy828); } + case 278: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy369, yymsp[-3].minor.yy448, yymsp[-1].minor.yy448, NULL, yymsp[0].minor.yy448); } break; - case 272: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy669, yymsp[-5].minor.yy828, yymsp[-3].minor.yy828, yymsp[-1].minor.yy222, NULL); } + case 279: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy369, yymsp[-5].minor.yy448, yymsp[-3].minor.yy448, yymsp[-1].minor.yy432, NULL); } break; - case 273: /* cmd ::= DROP INDEX exists_opt full_index_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy669, yymsp[0].minor.yy828); } + case 280: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } break; - case 274: /* full_index_name ::= index_name */ -{ yylhsminor.yy828 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy63); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 281: /* full_index_name ::= index_name */ +{ yylhsminor.yy448 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 275: /* full_index_name ::= db_name NK_DOT index_name */ -{ yylhsminor.yy828 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 282: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy448 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 276: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy828 = createIndexOption(pCxt, yymsp[-7].minor.yy222, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), NULL, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 283: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy448 = createIndexOption(pCxt, yymsp[-7].minor.yy432, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 277: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy828 = createIndexOption(pCxt, yymsp[-9].minor.yy222, releaseRawExprNode(pCxt, yymsp[-5].minor.yy828), releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 284: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy448 = createIndexOption(pCxt, yymsp[-9].minor.yy432, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 280: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy828 = createFunctionNode(pCxt, &yymsp[-3].minor.yy63, yymsp[-1].minor.yy222); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 287: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy432); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 286: /* sma_stream_opt ::= */ - case 322: /* stream_options ::= */ yytestcase(yyruleno==322); -{ yymsp[1].minor.yy828 = createStreamOptions(pCxt); } + case 288: /* sma_func_name ::= function_name */ + case 497: /* alias_opt ::= table_alias */ yytestcase(yyruleno==497); +{ yylhsminor.yy113 = yymsp[0].minor.yy113; } + yymsp[0].minor.yy113 = yylhsminor.yy113; break; - case 287: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy828)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy828); yylhsminor.yy828 = yymsp[-2].minor.yy828; } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 293: /* sma_stream_opt ::= */ + case 331: /* stream_options ::= */ yytestcase(yyruleno==331); +{ yymsp[1].minor.yy448 = createStreamOptions(pCxt); } break; - case 288: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy828)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy828); yylhsminor.yy828 = yymsp[-2].minor.yy828; } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 294: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 289: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy828)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy828); yylhsminor.yy828 = yymsp[-2].minor.yy828; } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 295: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 290: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy669, &yymsp[-2].minor.yy63, yymsp[0].minor.yy828); } + case 296: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 291: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy669, &yymsp[-3].minor.yy63, &yymsp[0].minor.yy63, false); } + case 297: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy369, &yymsp[-2].minor.yy113, yymsp[0].minor.yy448); } break; - case 292: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy669, &yymsp[-5].minor.yy63, &yymsp[0].minor.yy63, true); } + case 298: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, &yymsp[0].minor.yy113, false); } break; - case 293: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy669, &yymsp[-3].minor.yy63, yymsp[0].minor.yy828, false); } + case 299: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, &yymsp[0].minor.yy113, true); } break; - case 294: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy669, &yymsp[-5].minor.yy63, yymsp[0].minor.yy828, true); } + case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, yymsp[0].minor.yy448, false); } break; - case 295: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy669, &yymsp[0].minor.yy63); } + case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, yymsp[0].minor.yy448, true); } break; - case 296: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy669, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63); } + case 302: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 297: /* cmd ::= DESC full_table_name */ - case 298: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==298); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy828); } + case 303: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy369, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } break; - case 299: /* cmd ::= RESET QUERY CACHE */ + case 304: /* cmd ::= DESC full_table_name */ + case 305: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==305); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy448); } + break; + case 306: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 300: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - case 301: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==301); -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy669, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 307: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 308: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==308); +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy369, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 304: /* explain_options ::= */ -{ yymsp[1].minor.yy828 = createDefaultExplainOptions(pCxt); } + case 311: /* explain_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultExplainOptions(pCxt); } break; - case 305: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy828 = setExplainVerbose(pCxt, yymsp[-2].minor.yy828, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 312: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy448 = setExplainVerbose(pCxt, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 306: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy828 = setExplainRatio(pCxt, yymsp[-2].minor.yy828, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 313: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy448 = setExplainRatio(pCxt, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 307: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy669, yymsp[-9].minor.yy669, &yymsp[-6].minor.yy63, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy686, yymsp[-1].minor.yy332, &yymsp[0].minor.yy63); } + case 314: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy369, yymsp[-9].minor.yy369, &yymsp[-6].minor.yy113, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy728, yymsp[-1].minor.yy788, &yymsp[0].minor.yy113, yymsp[-10].minor.yy369); } break; - case 308: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy669, &yymsp[0].minor.yy63); } + case 315: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 313: /* language_opt ::= */ -{ yymsp[1].minor.yy63 = nil_token; } + case 320: /* language_opt ::= */ +{ yymsp[1].minor.yy113 = nil_token; } break; - case 314: /* language_opt ::= LANGUAGE NK_STRING */ -{ yymsp[-1].minor.yy63 = yymsp[0].minor.yy0; } + case 321: /* language_opt ::= LANGUAGE NK_STRING */ +{ yymsp[-1].minor.yy113 = yymsp[0].minor.yy0; } break; - case 315: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy669, &yymsp[-8].minor.yy63, yymsp[-5].minor.yy828, yymsp[-7].minor.yy828, yymsp[-3].minor.yy222, yymsp[-2].minor.yy828, yymsp[0].minor.yy828, yymsp[-4].minor.yy222); } + case 324: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy369, &yymsp[-8].minor.yy113, yymsp[-5].minor.yy448, yymsp[-7].minor.yy448, yymsp[-3].minor.yy432, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, yymsp[-4].minor.yy432); } break; - case 316: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy669, &yymsp[0].minor.yy63); } + case 325: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 323: /* stream_options ::= stream_options TRIGGER AT_ONCE */ - case 324: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==324); -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-2].minor.yy828, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 332: /* stream_options ::= stream_options TRIGGER AT_ONCE */ + case 333: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==333); +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 325: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-3].minor.yy828, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 334: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 326: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-2].minor.yy828, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 335: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 327: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-3].minor.yy828, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 336: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 328: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-2].minor.yy828, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 337: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 329: /* stream_options ::= stream_options DELETE_MARK duration_literal */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-2].minor.yy828, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 338: /* stream_options ::= stream_options DELETE_MARK duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 330: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ -{ yylhsminor.yy828 = setStreamOptions(pCxt, yymsp[-3].minor.yy828, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 339: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 332: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 521: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==521); - case 541: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==541); -{ yymsp[-3].minor.yy828 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy828); } + case 341: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 531: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==531); + case 551: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==551); +{ yymsp[-3].minor.yy448 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy448); } break; - case 333: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 342: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 334: /* cmd ::= KILL QUERY NK_STRING */ + case 343: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 335: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 344: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 336: /* cmd ::= BALANCE VGROUP */ + case 345: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 337: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 346: /* cmd ::= BALANCE VGROUP LEADER */ +{ pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); } + break; + case 347: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 338: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy222); } + case 348: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy432); } break; - case 339: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 349: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 340: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy222 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 342: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } - break; - case 345: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ yymsp[-6].minor.yy828 = createInsertStmt(pCxt, yymsp[-4].minor.yy828, yymsp[-2].minor.yy222, yymsp[0].minor.yy828); } - break; - case 346: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ -{ yymsp[-3].minor.yy828 = createInsertStmt(pCxt, yymsp[-1].minor.yy828, NULL, yymsp[0].minor.yy828); } - break; - case 347: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 348: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 349: /* literal ::= NK_STRING */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 350: /* literal ::= NK_BOOL */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 351: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; - break; - case 352: /* literal ::= duration_literal */ - case 362: /* signed_literal ::= signed */ yytestcase(yyruleno==362); - case 383: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==383); - case 384: /* expression ::= literal */ yytestcase(yyruleno==384); - case 385: /* expression ::= pseudo_column */ yytestcase(yyruleno==385); - case 386: /* expression ::= column_reference */ yytestcase(yyruleno==386); - case 387: /* expression ::= function_expression */ yytestcase(yyruleno==387); - case 388: /* expression ::= case_when_expression */ yytestcase(yyruleno==388); - case 419: /* function_expression ::= literal_func */ yytestcase(yyruleno==419); - case 468: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==468); - case 472: /* boolean_primary ::= predicate */ yytestcase(yyruleno==472); - case 474: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==474); - case 475: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==475); - case 478: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==478); - case 480: /* table_reference ::= table_primary */ yytestcase(yyruleno==480); - case 481: /* table_reference ::= joined_table */ yytestcase(yyruleno==481); - case 485: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==485); - case 543: /* query_simple ::= query_specification */ yytestcase(yyruleno==543); - case 544: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==544); - case 547: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==547); - case 549: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==549); -{ yylhsminor.yy828 = yymsp[0].minor.yy828; } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 353: /* literal ::= NULL */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 354: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 355: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 356: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 357: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 358: /* signed ::= NK_MINUS NK_INTEGER */ + case 350: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy432 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 352: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } + break; + case 355: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ yymsp[-6].minor.yy448 = createInsertStmt(pCxt, yymsp[-4].minor.yy448, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + break; + case 356: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +{ yymsp[-3].minor.yy448 = createInsertStmt(pCxt, yymsp[-1].minor.yy448, NULL, yymsp[0].minor.yy448); } + break; + case 357: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 358: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 359: /* literal ::= NK_STRING */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 360: /* literal ::= NK_BOOL */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 361: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 362: /* literal ::= duration_literal */ + case 372: /* signed_literal ::= signed */ yytestcase(yyruleno==372); + case 393: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==393); + case 394: /* expression ::= literal */ yytestcase(yyruleno==394); + case 395: /* expression ::= pseudo_column */ yytestcase(yyruleno==395); + case 396: /* expression ::= column_reference */ yytestcase(yyruleno==396); + case 397: /* expression ::= function_expression */ yytestcase(yyruleno==397); + case 398: /* expression ::= case_when_expression */ yytestcase(yyruleno==398); + case 429: /* function_expression ::= literal_func */ yytestcase(yyruleno==429); + case 478: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==478); + case 482: /* boolean_primary ::= predicate */ yytestcase(yyruleno==482); + case 484: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==484); + case 485: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==485); + case 488: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==488); + case 490: /* table_reference ::= table_primary */ yytestcase(yyruleno==490); + case 491: /* table_reference ::= joined_table */ yytestcase(yyruleno==491); + case 495: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==495); + case 553: /* query_simple ::= query_specification */ yytestcase(yyruleno==553); + case 554: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==554); + case 557: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==557); + case 559: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==559); +{ yylhsminor.yy448 = yymsp[0].minor.yy448; } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 363: /* literal ::= NULL */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 364: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 365: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 366: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 367: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 368: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 359: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 369: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 360: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 370: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 361: /* signed ::= NK_MINUS NK_FLOAT */ + case 371: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; - break; - case 363: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 364: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 365: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 366: /* signed_literal ::= duration_literal */ - case 368: /* signed_literal ::= literal_func */ yytestcase(yyruleno==368); - case 439: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==439); - case 501: /* select_item ::= common_expression */ yytestcase(yyruleno==501); - case 511: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==511); - case 548: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==548); - case 550: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==550); - case 563: /* search_condition ::= common_expression */ yytestcase(yyruleno==563); -{ yylhsminor.yy828 = releaseRawExprNode(pCxt, yymsp[0].minor.yy828); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 367: /* signed_literal ::= NULL */ -{ yylhsminor.yy828 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 369: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy828 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 389: /* expression ::= NK_LP expression NK_RP */ - case 473: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==473); - case 562: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==562); -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy828)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 390: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 373: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 374: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 375: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 376: /* signed_literal ::= duration_literal */ + case 378: /* signed_literal ::= literal_func */ yytestcase(yyruleno==378); + case 449: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==449); + case 511: /* select_item ::= common_expression */ yytestcase(yyruleno==511); + case 521: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==521); + case 558: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==558); + case 560: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==560); + case 573: /* search_condition ::= common_expression */ yytestcase(yyruleno==573); +{ yylhsminor.yy448 = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 377: /* signed_literal ::= NULL */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 379: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy448 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 399: /* expression ::= NK_LP expression NK_RP */ + case 483: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==483); + case 572: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==572); +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 400: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 391: /* expression ::= NK_MINUS expr_or_subquery */ + case 401: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy828), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy448), NULL)); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 392: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 402: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 393: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 403: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 394: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 404: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 395: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 405: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 396: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 406: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 397: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 407: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 398: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 408: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 399: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 409: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 402: /* column_reference ::= column_name */ -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy63, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy63)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 403: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63, createColumnNode(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy63)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 404: /* pseudo_column ::= ROWTS */ - case 405: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==405); - case 407: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==407); - case 408: /* pseudo_column ::= QEND */ yytestcase(yyruleno==408); - case 409: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==409); - case 410: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==410); - case 411: /* pseudo_column ::= WEND */ yytestcase(yyruleno==411); - case 412: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==412); - case 413: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==413); - case 414: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==414); - case 415: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==415); - case 421: /* literal_func ::= NOW */ yytestcase(yyruleno==421); -{ yylhsminor.yy828 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy828 = yylhsminor.yy828; - break; - case 406: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy63)))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 416: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 417: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==417); -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy63, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy63, yymsp[-1].minor.yy222)); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; - break; - case 418: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), yymsp[-1].minor.yy686)); } - yymsp[-5].minor.yy828 = yylhsminor.yy828; - break; - case 420: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy63, NULL)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 435: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy222 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy222 = yylhsminor.yy222; - break; - case 440: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 504: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==504); -{ yylhsminor.yy828 = createColumnNode(pCxt, &yymsp[-2].minor.yy63, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 441: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy222, yymsp[-1].minor.yy828)); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; - break; - case 442: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), yymsp[-2].minor.yy222, yymsp[-1].minor.yy828)); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; - break; - case 445: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy828 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828)); } - break; - case 447: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy828 = releaseRawExprNode(pCxt, yymsp[0].minor.yy828); } - break; - case 448: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 453: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==453); + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 412: /* column_reference ::= column_name */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy113, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 413: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 414: /* pseudo_column ::= ROWTS */ + case 415: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==415); + case 417: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==417); + case 418: /* pseudo_column ::= QEND */ yytestcase(yyruleno==418); + case 419: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==419); + case 420: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==420); + case 421: /* pseudo_column ::= WEND */ yytestcase(yyruleno==421); + case 422: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==422); + case 423: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==423); + case 424: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==424); + case 425: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==425); + case 431: /* literal_func ::= NOW */ yytestcase(yyruleno==431); +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 416: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy113)))); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 426: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 427: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==427); +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy432)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 428: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy728)); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; + break; + case 430: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy113, NULL)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 445: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy432 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 450: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 514: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==514); +{ yylhsminor.yy448 = createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 451: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy432, yymsp[-1].minor.yy448)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 452: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-2].minor.yy432, yymsp[-1].minor.yy448)); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 455: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy448 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + break; + case 457: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy448 = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); } + break; + case 458: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 463: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==463); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy62, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy156, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 449: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 459: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy828), releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy448), releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-4].minor.yy828 = yylhsminor.yy828; + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 450: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 460: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy828), releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-5].minor.yy828 = yylhsminor.yy828; + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 451: /* predicate ::= expr_or_subquery IS NULL */ + case 461: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), NULL)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 452: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 462: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL)); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 454: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy62 = OP_TYPE_LOWER_THAN; } + case 464: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy156 = OP_TYPE_LOWER_THAN; } break; - case 455: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy62 = OP_TYPE_GREATER_THAN; } + case 465: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy156 = OP_TYPE_GREATER_THAN; } break; - case 456: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy62 = OP_TYPE_LOWER_EQUAL; } + case 466: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy156 = OP_TYPE_LOWER_EQUAL; } break; - case 457: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy62 = OP_TYPE_GREATER_EQUAL; } + case 467: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy156 = OP_TYPE_GREATER_EQUAL; } break; - case 458: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy62 = OP_TYPE_NOT_EQUAL; } + case 468: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy156 = OP_TYPE_NOT_EQUAL; } break; - case 459: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy62 = OP_TYPE_EQUAL; } + case 469: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy156 = OP_TYPE_EQUAL; } break; - case 460: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy62 = OP_TYPE_LIKE; } + case 470: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy156 = OP_TYPE_LIKE; } break; - case 461: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy62 = OP_TYPE_NOT_LIKE; } + case 471: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy156 = OP_TYPE_NOT_LIKE; } break; - case 462: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy62 = OP_TYPE_MATCH; } + case 472: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy156 = OP_TYPE_MATCH; } break; - case 463: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy62 = OP_TYPE_NMATCH; } + case 473: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy156 = OP_TYPE_NMATCH; } break; - case 464: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy62 = OP_TYPE_JSON_CONTAINS; } + case 474: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy156 = OP_TYPE_JSON_CONTAINS; } break; - case 465: /* in_op ::= IN */ -{ yymsp[0].minor.yy62 = OP_TYPE_IN; } + case 475: /* in_op ::= IN */ +{ yymsp[0].minor.yy156 = OP_TYPE_IN; } break; - case 466: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy62 = OP_TYPE_NOT_IN; } + case 476: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy156 = OP_TYPE_NOT_IN; } break; - case 467: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy222)); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 477: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 469: /* boolean_value_expression ::= NOT boolean_primary */ + case 479: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy828), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy448), NULL)); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 470: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 480: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 471: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 481: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy828); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy828); - yylhsminor.yy828 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; - break; - case 477: /* from_clause_opt ::= FROM table_reference_list */ - case 506: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==506); - case 537: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==537); -{ yymsp[-1].minor.yy828 = yymsp[0].minor.yy828; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 479: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy828 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy828, yymsp[0].minor.yy828, NULL); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 489: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy448 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 482: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy828 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy63, &yymsp[0].minor.yy63); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 492: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 483: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy828 = createRealTableNode(pCxt, &yymsp[-3].minor.yy63, &yymsp[-1].minor.yy63, &yymsp[0].minor.yy63); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 493: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 484: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy828 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy828), &yymsp[0].minor.yy63); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 494: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy448 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 486: /* alias_opt ::= */ -{ yymsp[1].minor.yy63 = nil_token; } + case 496: /* alias_opt ::= */ +{ yymsp[1].minor.yy113 = nil_token; } break; - case 488: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy63 = yymsp[0].minor.yy63; } + case 498: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy113 = yymsp[0].minor.yy113; } break; - case 489: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 490: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==490); -{ yymsp[-2].minor.yy828 = yymsp[-1].minor.yy828; } + case 499: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 500: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==500); +{ yymsp[-2].minor.yy448 = yymsp[-1].minor.yy448; } break; - case 491: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy828 = createJoinTableNode(pCxt, yymsp[-4].minor.yy574, yymsp[-5].minor.yy828, yymsp[-2].minor.yy828, yymsp[0].minor.yy828); } - yymsp[-5].minor.yy828 = yylhsminor.yy828; + case 501: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy448 = createJoinTableNode(pCxt, yymsp[-4].minor.yy596, yymsp[-5].minor.yy448, yymsp[-2].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 492: /* join_type ::= */ -{ yymsp[1].minor.yy574 = JOIN_TYPE_INNER; } + case 502: /* join_type ::= */ +{ yymsp[1].minor.yy596 = JOIN_TYPE_INNER; } break; - case 493: /* join_type ::= INNER */ -{ yymsp[0].minor.yy574 = JOIN_TYPE_INNER; } + case 503: /* join_type ::= INNER */ +{ yymsp[0].minor.yy596 = JOIN_TYPE_INNER; } break; - case 494: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 504: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy828 = createSelectStmt(pCxt, yymsp[-10].minor.yy669, yymsp[-9].minor.yy222, yymsp[-8].minor.yy828); - yymsp[-11].minor.yy828 = addWhereClause(pCxt, yymsp[-11].minor.yy828, yymsp[-7].minor.yy828); - yymsp[-11].minor.yy828 = addPartitionByClause(pCxt, yymsp[-11].minor.yy828, yymsp[-6].minor.yy222); - yymsp[-11].minor.yy828 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy828, yymsp[-2].minor.yy828); - yymsp[-11].minor.yy828 = addGroupByClause(pCxt, yymsp[-11].minor.yy828, yymsp[-1].minor.yy222); - yymsp[-11].minor.yy828 = addHavingClause(pCxt, yymsp[-11].minor.yy828, yymsp[0].minor.yy828); - yymsp[-11].minor.yy828 = addRangeClause(pCxt, yymsp[-11].minor.yy828, yymsp[-5].minor.yy828); - yymsp[-11].minor.yy828 = addEveryClause(pCxt, yymsp[-11].minor.yy828, yymsp[-4].minor.yy828); - yymsp[-11].minor.yy828 = addFillClause(pCxt, yymsp[-11].minor.yy828, yymsp[-3].minor.yy828); + yymsp[-11].minor.yy448 = createSelectStmt(pCxt, yymsp[-10].minor.yy369, yymsp[-9].minor.yy432, yymsp[-8].minor.yy448); + yymsp[-11].minor.yy448 = addWhereClause(pCxt, yymsp[-11].minor.yy448, yymsp[-7].minor.yy448); + yymsp[-11].minor.yy448 = addPartitionByClause(pCxt, yymsp[-11].minor.yy448, yymsp[-6].minor.yy432); + yymsp[-11].minor.yy448 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy448, yymsp[-2].minor.yy448); + yymsp[-11].minor.yy448 = addGroupByClause(pCxt, yymsp[-11].minor.yy448, yymsp[-1].minor.yy432); + yymsp[-11].minor.yy448 = addHavingClause(pCxt, yymsp[-11].minor.yy448, yymsp[0].minor.yy448); + yymsp[-11].minor.yy448 = addRangeClause(pCxt, yymsp[-11].minor.yy448, yymsp[-5].minor.yy448); + yymsp[-11].minor.yy448 = addEveryClause(pCxt, yymsp[-11].minor.yy448, yymsp[-4].minor.yy448); + yymsp[-11].minor.yy448 = addFillClause(pCxt, yymsp[-11].minor.yy448, yymsp[-3].minor.yy448); } break; - case 497: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy669 = false; } + case 507: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy369 = false; } break; - case 500: /* select_item ::= NK_STAR */ -{ yylhsminor.yy828 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy828 = yylhsminor.yy828; + case 510: /* select_item ::= NK_STAR */ +{ yylhsminor.yy448 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 502: /* select_item ::= common_expression column_alias */ - case 512: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==512); -{ yylhsminor.yy828 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy828), &yymsp[0].minor.yy63); } - yymsp[-1].minor.yy828 = yylhsminor.yy828; + case 512: /* select_item ::= common_expression column_alias */ + case 522: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==522); +{ yylhsminor.yy448 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 503: /* select_item ::= common_expression AS column_alias */ - case 513: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==513); -{ yylhsminor.yy828 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), &yymsp[0].minor.yy63); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 513: /* select_item ::= common_expression AS column_alias */ + case 523: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==523); +{ yylhsminor.yy448 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 508: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 533: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==533); - case 552: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==552); -{ yymsp[-2].minor.yy222 = yymsp[0].minor.yy222; } + case 518: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 543: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==543); + case 562: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==562); +{ yymsp[-2].minor.yy432 = yymsp[0].minor.yy432; } break; - case 515: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy828 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), releaseRawExprNode(pCxt, yymsp[-1].minor.yy828)); } + case 525: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy448 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 516: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy828 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy828)); } + case 526: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy448 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 517: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy828 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), NULL, yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 527: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy448 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 518: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy828 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy828), releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), yymsp[-1].minor.yy828, yymsp[0].minor.yy828); } + case 528: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy448 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 519: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -{ yymsp[-6].minor.yy828 = createEventWindowNode(pCxt, yymsp[-3].minor.yy828, yymsp[0].minor.yy828); } + case 529: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy448 = createEventWindowNode(pCxt, yymsp[-3].minor.yy448, yymsp[0].minor.yy448); } break; - case 523: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy828 = createFillNode(pCxt, yymsp[-1].minor.yy822, NULL); } + case 533: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy448 = createFillNode(pCxt, yymsp[-1].minor.yy46, NULL); } break; - case 524: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy828 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy222)); } + case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 525: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy828 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy222)); } + case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 526: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy822 = FILL_MODE_NONE; } + case 536: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy46 = FILL_MODE_NONE; } break; - case 527: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy822 = FILL_MODE_PREV; } + case 537: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy46 = FILL_MODE_PREV; } break; - case 528: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy822 = FILL_MODE_NULL; } + case 538: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy46 = FILL_MODE_NULL; } break; - case 529: /* fill_mode ::= NULL_F */ -{ yymsp[0].minor.yy822 = FILL_MODE_NULL_F; } + case 539: /* fill_mode ::= NULL_F */ +{ yymsp[0].minor.yy46 = FILL_MODE_NULL_F; } break; - case 530: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy822 = FILL_MODE_LINEAR; } + case 540: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy46 = FILL_MODE_LINEAR; } break; - case 531: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy822 = FILL_MODE_NEXT; } + case 541: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy46 = FILL_MODE_NEXT; } break; - case 534: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy222 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); } - yymsp[0].minor.yy222 = yylhsminor.yy222; + case 544: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy432 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 535: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy222 = addNodeToList(pCxt, yymsp[-2].minor.yy222, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy828))); } - yymsp[-2].minor.yy222 = yylhsminor.yy222; + case 545: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; break; - case 539: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy828 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy828), releaseRawExprNode(pCxt, yymsp[-1].minor.yy828)); } + case 549: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy448 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 542: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 552: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy828 = addOrderByClause(pCxt, yymsp[-3].minor.yy828, yymsp[-2].minor.yy222); - yylhsminor.yy828 = addSlimitClause(pCxt, yylhsminor.yy828, yymsp[-1].minor.yy828); - yylhsminor.yy828 = addLimitClause(pCxt, yylhsminor.yy828, yymsp[0].minor.yy828); + yylhsminor.yy448 = addOrderByClause(pCxt, yymsp[-3].minor.yy448, yymsp[-2].minor.yy432); + yylhsminor.yy448 = addSlimitClause(pCxt, yylhsminor.yy448, yymsp[-1].minor.yy448); + yylhsminor.yy448 = addLimitClause(pCxt, yylhsminor.yy448, yymsp[0].minor.yy448); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 545: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy828 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy828, yymsp[0].minor.yy828); } - yymsp[-3].minor.yy828 = yylhsminor.yy828; + case 555: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy448 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 546: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy828 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy828, yymsp[0].minor.yy828); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 556: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy448 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 554: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 558: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==558); -{ yymsp[-1].minor.yy828 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 564: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 568: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==568); +{ yymsp[-1].minor.yy448 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 555: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 559: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==559); -{ yymsp[-3].minor.yy828 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 565: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 569: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==569); +{ yymsp[-3].minor.yy448 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 556: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 560: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==560); -{ yymsp[-3].minor.yy828 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 566: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 570: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==570); +{ yymsp[-3].minor.yy448 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 561: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy828 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy828); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 571: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy448); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 566: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy828 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy828), yymsp[-1].minor.yy158, yymsp[0].minor.yy675); } - yymsp[-2].minor.yy828 = yylhsminor.yy828; + case 576: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy448 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), yymsp[-1].minor.yy666, yymsp[0].minor.yy585); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 567: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy158 = ORDER_ASC; } + case 577: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy666 = ORDER_ASC; } break; - case 568: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy158 = ORDER_ASC; } + case 578: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy666 = ORDER_ASC; } break; - case 569: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy158 = ORDER_DESC; } + case 579: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy666 = ORDER_DESC; } break; - case 570: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy675 = NULL_ORDER_DEFAULT; } + case 580: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy585 = NULL_ORDER_DEFAULT; } break; - case 571: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy675 = NULL_ORDER_FIRST; } + case 581: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy585 = NULL_ORDER_FIRST; } break; - case 572: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy675 = NULL_ORDER_LAST; } + case 582: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy585 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 1af214bfb4033502ece07a0f03455f60bd3bed18..be4f3cd80b08d86d52da4160daf99d0657341d18 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -279,15 +279,13 @@ int32_t __catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* d return g_mockCatalogService->catalogGetDBCfg(dbFName, pDbCfg); } -int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { - *pass = true; +int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) { + pRes->pass = true; return 0; } -int32_t __catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, - bool* exists) { - *pass = true; +int32_t __catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) { + pRes->pass = true; *exists = true; return 0; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4d1ef597d0b5907e05dba6cc9c565d3b5ec7d75c..f6a8b407d76071ceb0324ee492655d2959565f7c 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -589,8 +589,8 @@ class MockCatalogServiceImpl { *pUserAuthData = taosArrayInit(num, sizeof(SMetaRes)); for (int32_t i = 0; i < num; ++i) { SMetaRes res = {0}; - res.pRes = taosMemoryCalloc(1, sizeof(bool)); - *(bool*)(res.pRes) = true; + res.pRes = taosMemoryCalloc(1, sizeof(SUserAuthRes)); + ((SUserAuthRes*)res.pRes)->pass = true; taosArrayPush(*pUserAuthData, &res); } } diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index bfcf6ec27e7eec142a4a586bc31c61a8690b51a3..db74e66d2bb06b1de08a187583ee9fadfe72e8b4 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -108,7 +108,9 @@ TEST_F(ParserInitialATest, alterDnode) { * | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica * | WAL_LEVEL int_value -- enum 1, 2, default 1 * | STT_TRIGGER int_value -- rang [1, 16], default 8 - * } + * | MINROWS int_value -- rang [10, 1000], default 100 + * | WAL_RETENTION_PERIOD int_value -- rang [-1, INT32_MAX], default 0 + * | WAL_RETENTION_SIZE int_value -- rang [-1, INT32_MAX], default 0 */ TEST_F(ParserInitialATest, alterDatabase) { useDb("root", "test"); @@ -133,6 +135,9 @@ TEST_F(ParserInitialATest, alterDatabase) { expect.cacheLastSize = -1; expect.replications = -1; expect.sstTrigger = -1; + expect.minRows = -1; + expect.walRetentionPeriod = -2; + expect.walRetentionSize = -2; }; auto setAlterDbBuffer = [&](int32_t buffer) { expect.buffer = buffer; }; auto setAlterDbPageSize = [&](int32_t pageSize) { expect.pageSize = pageSize; }; @@ -150,6 +155,11 @@ TEST_F(ParserInitialATest, alterDatabase) { auto setAlterDbCacheModel = [&](int8_t cacheModel) { expect.cacheLast = cacheModel; }; auto setAlterDbReplica = [&](int8_t replications) { expect.replications = replications; }; auto setAlterDbSttTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; + auto setAlterDbMinRows = [&](int32_t minRows) { expect.minRows = minRows; }; + auto setAlterDbWalRetentionPeriod = [&](int32_t walRetentionPeriod) { + expect.walRetentionPeriod = walRetentionPeriod; + }; + auto setAlterDbWalRetentionSize = [&](int32_t walRetentionSize) { expect.walRetentionSize = walRetentionSize; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DATABASE_STMT); @@ -170,6 +180,9 @@ TEST_F(ParserInitialATest, alterDatabase) { ASSERT_EQ(req.cacheLast, expect.cacheLast); ASSERT_EQ(req.replications, expect.replications); ASSERT_EQ(req.sstTrigger, expect.sstTrigger); + ASSERT_EQ(req.minRows, expect.minRows); + ASSERT_EQ(req.walRetentionPeriod, expect.walRetentionPeriod); + ASSERT_EQ(req.walRetentionSize, expect.walRetentionSize); }); const int32_t MINUTE_PER_DAY = MILLISECOND_PER_DAY / MILLISECOND_PER_MINUTE; @@ -185,8 +198,10 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbBuffer(16); setAlterDbPages(128); setAlterDbReplica(3); + setAlterDbWalRetentionPeriod(10); + setAlterDbWalRetentionSize(20); run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " - "REPLICA 3 WAL_LEVEL 1 STT_TRIGGER 16"); + "REPLICA 3 WAL_LEVEL 1 STT_TRIGGER 16 WAL_RETENTION_PERIOD 10 WAL_RETENTION_SIZE 20"); clearAlterDbReq(); initAlterDb("test"); @@ -277,6 +292,29 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbSttTrigger(16); run("ALTER DATABASE test STT_TRIGGER 16"); clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbMinRows(10); + run("ALTER DATABASE test MINROWS 10"); + setAlterDbMinRows(50); + run("ALTER DATABASE test MINROWS 50"); + setAlterDbMinRows(1000); + run("ALTER DATABASE test MINROWS 1000"); + clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbWalRetentionPeriod(-1); + run("ALTER DATABASE test WAL_RETENTION_PERIOD -1"); + setAlterDbWalRetentionPeriod(50); + run("ALTER DATABASE test WAL_RETENTION_PERIOD 50"); + clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbWalRetentionSize(-1); + run("ALTER DATABASE test WAL_RETENTION_SIZE -1"); + setAlterDbWalRetentionSize(50); + run("ALTER DATABASE test WAL_RETENTION_SIZE 50"); + clearAlterDbReq(); } TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { @@ -599,7 +637,9 @@ TEST_F(ParserInitialATest, alterTable) { } ASSERT_EQ(req.isNull, expect.isNull); ASSERT_EQ(req.nTagVal, expect.nTagVal); - ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); + if (nullptr != req.pTagVal) { + ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); + } ASSERT_EQ(req.updateTTL, expect.updateTTL); ASSERT_EQ(req.newTTL, expect.newTTL); if (nullptr != expect.newComment) { @@ -818,4 +858,21 @@ TEST_F(ParserInitialATest, balanceVgroup) { run("BALANCE VGROUP"); } +/* + * BALANCE VGROUP LEADER + */ +TEST_F(ParserInitialATest, balanceVgroupLeader) { + useDb("root", "test"); + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_BALANCE_VGROUP_LEADER_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_BALANCE_VGROUP_LEADER); + SBalanceVgroupLeaderReq req = {0}; + ASSERT_EQ(tDeserializeSBalanceVgroupLeaderReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), + TSDB_CODE_SUCCESS); + }); + + run("BALANCE VGROUP LEADER"); +} + } // namespace ParserTest \ No newline at end of file diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 2dffdae934453bb8f5284c0d3f8972887e898584..b7ca944ebb22770b848951f833b00b0e37618244 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -398,7 +398,7 @@ TEST_F(ParserInitialCTest, createDnode) { } /* - * CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name + * CREATE [OR REPLACE] [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name * AS library_path OUTPUTTYPE type_name [BUFSIZE value] [LANGUAGE value] */ TEST_F(ParserInitialCTest, createFunction) { @@ -408,9 +408,10 @@ TEST_F(ParserInitialCTest, createFunction) { auto setCreateFuncReq = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0, int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0, - int8_t language = TSDB_FUNC_SCRIPT_BIN_LIB) { + int8_t language = TSDB_FUNC_SCRIPT_BIN_LIB, int8_t orReplace = 0) { memset(&expect, 0, sizeof(SCreateFuncReq)); strcpy(expect.name, pUdfName); + expect.orReplace = orReplace; expect.igExists = igExists; expect.funcType = funcType; expect.scriptType = language; @@ -425,6 +426,7 @@ TEST_F(ParserInitialCTest, createFunction) { ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateFuncReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.orReplace, expect.orReplace); ASSERT_EQ(req.igExists, expect.igExists); ASSERT_EQ(req.funcType, expect.funcType); ASSERT_EQ(req.scriptType, expect.scriptType); @@ -448,8 +450,8 @@ TEST_F(ParserInitialCTest, createFunction) { setCreateFuncReq("udf1", TSDB_DATA_TYPE_INT); run("CREATE FUNCTION udf1 AS 'udf' OUTPUTTYPE INT"); - setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8, TSDB_FUNC_SCRIPT_PYTHON); - run("CREATE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS 'udf' OUTPUTTYPE DOUBLE BUFSIZE 8 LANGUAGE 'python'"); + setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8, TSDB_FUNC_SCRIPT_PYTHON, 1); + run("CREATE OR REPLACE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS 'udf' OUTPUTTYPE DOUBLE BUFSIZE 8 LANGUAGE 'python'"); } /* @@ -758,10 +760,10 @@ TEST_F(ParserInitialCTest, createStable) { addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " - "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " + "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " "DELETE_MARK 1000s,200m"); @@ -1021,16 +1023,16 @@ TEST_F(ParserInitialCTest, createTable) { run("CREATE TABLE IF NOT EXISTS test.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); run("CREATE TABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " - "a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " + "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " "a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index fcd8dd1f2635040f7f4b7e59f07302246f0a7630..ec6c69ea8ded78443ba9803c7099faafaaf8765a 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -286,7 +286,7 @@ TEST_F(ParserSelectTest, interval) { TEST_F(ParserSelectTest, intervalSemanticCheck) { useDb("root", "test"); - run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN); + run("SELECT c1 FROM t1 INTERVAL(10s)", TSDB_CODE_PAR_INVALID_OPTR_USAGE); run("SELECT DISTINCT c1, c2 FROM t1 WHERE c1 > 3 INTERVAL(1d) FILL(NEXT)", TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)", diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c9ee83a64757b6fe951d93fd9c9d917ba357f58b..6544898be9ff389cf441d8824ef06b235fd7cc35 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -740,6 +740,13 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + pWindow->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pWindow->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + pSelect->hasAggFuncs = false; if (TSDB_CODE_SUCCESS == code) { @@ -1132,6 +1139,14 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving && !pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && + NULL == pSelect->pWindow) { + pPartition->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pPartition->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pPartition; } else { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 16a5ef7bae1e7aeaa4c0521022bd96913b9c222a..07ea110d7e68382a4872e4560a21eca90ac10332 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1079,11 +1079,23 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { return false; } + SNode* pChild; + FOREACH(pChild, pSort->node.pChildren) { + SLogicNode* pSortDescendent = optFindPossibleNode((SLogicNode*)pChild, sortPriKeyOptMayBeOptimized); + if (pSortDescendent != NULL) { + return false; + } + } return true; } static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize, SNodeList** pSequencingNodes) { + if (NULL != pNode->pLimit || NULL != pNode->pSlimit) { + *pNotOptimize = false; + return TSDB_CODE_SUCCESS; + } + switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -2127,7 +2139,7 @@ typedef struct SLastRowScanOptLastParaCkCxt { bool hasCol; } SLastRowScanOptLastParaCkCxt; -static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { +static EDealRes lastRowScanOptLastParaIsTagImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SLastRowScanOptLastParaCkCxt* pCxt = pContext; if (COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType || COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) { @@ -2140,10 +2152,10 @@ static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool lastRowScanOptLastParaCheck(SNode* pExpr) { +static bool lastRowScanOptLastParaIsTag(SNode* pExpr) { SLastRowScanOptLastParaCkCxt cxt = {.hasTag = false, .hasCol = false}; - nodesWalkExpr(pExpr, lastRowScanOptLastParaCheckImpl, &cxt); - return !cxt.hasTag && cxt.hasCol; + nodesWalkExpr(pExpr, lastRowScanOptLastParaIsTagImpl, &cxt); + return cxt.hasTag && !cxt.hasCol; } static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast) { @@ -2183,15 +2195,19 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { - if (hasSelectFunc || !lastRowScanOptLastParaCheck(nodesListGetNode(pAggFunc->pParameterList, 0))) { + if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } hasLastFunc = true; - } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) { if (hasLastFunc) { return false; } hasSelectFunc = true; + } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) { + return false; + } } else if (FUNCTION_TYPE_LAST_ROW != pAggFunc->funcType) { return false; } @@ -2225,7 +2241,7 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, bool erase) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -2237,7 +2253,7 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo break; } } - if (!found) { + if (!found && erase) { ERASE_NODE(pTargets); continue; } @@ -2278,9 +2294,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic pScan->igLastNull = pAgg->hasLast ? true : false; if (NULL != cxt.pLastCols) { cxt.doAgg = false; - lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); - lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); + lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); + lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); nodesClearList(cxt.pLastCols); } pAgg->hasLastRow = false; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e9a2efaac79fe237c8e4e93a91cf79f397d27f33..e2c2e4c65563e3b4a49add42d73c004f1ea39bb7 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1089,9 +1089,15 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic pExchange->srcStartGroupId = pExchangeLogicNode->srcStartGroupId; pExchange->srcEndGroupId = pExchangeLogicNode->srcEndGroupId; pExchange->seqRecvData = pExchangeLogicNode->seqRecvData; - *pPhyNode = (SPhysiNode*)pExchange; - return TSDB_CODE_SUCCESS; + int32_t code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pExchange); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pExchange; + } else { + nodesDestroyNode((SNode*)pExchange); + } + + return code; } static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, @@ -1119,6 +1125,9 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc if (TSDB_CODE_SUCCESS == code) { code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pScan); + } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pScan; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 361cf33d58bffb63523156d6fab555b8ccb5d5d4..fd77261818702defef8aa928e10a73cb66e28b20 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1365,7 +1365,8 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou pExchange->srcEndGroupId = pCxt->groupId - 1; pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); - if (NULL == pExchange->node.pTargets) { + pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions); + if (NULL == pExchange->node.pTargets || (NULL != pProject->node.pConditions && NULL == pExchange->node.pConditions)) { return TSDB_CODE_OUT_OF_MEMORY; } TSWAP(pExchange->node.pLimit, pProject->node.pLimit); diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c68a08682c57886b407dcb9f6ab7c6245ddf61a2..9d8c17000345a7aa73761c1e3f9af5ae4a4a3363 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -194,6 +194,8 @@ char* jobTaskStatusStr(int32_t status) { return "EXECUTING"; case JOB_TASK_STATUS_PART_SUCC: return "PARTIAL_SUCCEED"; + case JOB_TASK_STATUS_FETCH: + return "FETCHING"; case JOB_TASK_STATUS_SUCC: return "SUCCEED"; case JOB_TASK_STATUS_FAIL: diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index d2934e1ff86d964cc80e391e53052c25900b0d4a..b62a3e49326181ba6d32b07f307e1e18f5fbc76f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -587,7 +587,8 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { memcpy(output, funcInfo, sizeof(*funcInfo)); taosArrayDestroy(out.pFuncInfos); - + taosArrayDestroy(out.pFuncExtraInfos); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 0ab501ddd5abe806f62ef611900df1455b716e9f..59e63e9eae2636ae1676ac612083e3857e3a7510 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -259,19 +259,30 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { static int32_t ignoreTime = 0; if (++ignoreTime > 10 && 0 == taosRand() % 9) { + if (ctx->msgType == TDMT_SCH_FETCH) { + qwBuildAndSendErrorRsp(TDMT_SCH_LINK_BROKEN, &ctx->ctrlConnInfo, TSDB_CODE_RPC_BROKEN_LINK); + qwBuildAndSendErrorRsp(ctx->msgType + 1, &ctx->dataConnInfo, TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); + *rsped = true; + + taosSsleep(3); + return; + } + +#if 0 SRpcHandleInfo *pConn = ((ctx->msgType == TDMT_SCH_FETCH || ctx->msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo - : &ctx->ctrlConnInfo); + : &ctx->ctrlConnInfo); qwBuildAndSendErrorRsp(ctx->msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); - + qwBuildAndSendDropMsg(QW_FPARAMS(), pConn); *rsped = true; - + return; +#endif } } -int32_t qwDbgEnableDebug(char *option) { +int32_t qWorkerDbgEnableDebug(char *option) { if (0 == strcasecmp(option, "lock")) { gQWDebug.lockEnable = true; qError("qw lock debug enabled"); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ec4049a3eb45c2c274a58837e9f06b1af940e631..92c7852dbca5ba9bd2bf0542e31e7b63b15aa632 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -199,7 +199,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_ERR_JRET(code); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + QW_TASK_DLOG("data put into sink, rows:%" PRId64 ", continueExecTask:%d", pRes->info.rows, qcontinue); } if (numOfResBlock == 0 || (hasMore == false)) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 25938a7678967a32d2f6f9193f14710a42de61cd..0521076d23d9662219574388e86960b5dddbea6f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -59,6 +59,8 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara pParam->columnData = pColumnData; pParam->colAlloced = true; + pParam->numOfRows = numOfRows; + return TSDB_CODE_SUCCESS; } @@ -740,6 +742,10 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } + if (rowNum == 0) { + goto _return; + } + code = (*ffpSet.process)(params, paramNum, output); if (code) { sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); @@ -1032,23 +1038,40 @@ bool sclContainsAggFuncNode(SNode *pNode) { return aggFunc; } +static uint8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) { + uint8_t lPrec = ((SExprNode *)pLeft)->resType.precision; + uint8_t rPrec = ((SExprNode *)pRight)->resType.precision; + + uint8_t lType = ((SExprNode *)pLeft)->resType.type; + uint8_t rType = ((SExprNode *)pRight)->resType.type; + + if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP == rType) { + return TMAX(lPrec, rPrec); + } else if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP != rType) { + return lPrec; + } else if (TSDB_DATA_TYPE_TIMESTAMP == rType && TSDB_DATA_TYPE_TIMESTAMP != lType) { + return rPrec; + } + + return 0; +} int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) { int32_t code = 0; - + if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, (SValueNode*)node->pLeft)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft)); } } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)node->pRight)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight)); } else if (QUERY_NODE_NODE_LIST == node->pRight->type) { SNode* pNode; FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) { if (SCL_IS_VAR_VALUE_NODE(pNode)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)pNode)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode)); } } } @@ -1071,7 +1094,7 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { if (NULL == node->pCase) { return TSDB_CODE_SUCCESS; } - + if (SCL_IS_VAR_VALUE_NODE(node->pCase)) { SNode* pNode; FOREACH(pNode, node->pWhenThenList) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index f4c36695742a7f266cee4bee6494b83fd3a96c9a..835264eabe165f9eaf2fe3869ebb03e13cf2dd05 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1055,9 +1055,9 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { timeVal = timeVal / 1000; } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / (1000 * 1000); + timeVal = timeVal / ((int64_t)(1000 * 1000)); } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / (1000 * 1000 * 1000); + timeVal = timeVal / ((int64_t)(1000 * 1000 * 1000)); } else { colDataSetNULL(pOutput->columnData, i); continue; @@ -1067,9 +1067,15 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * } struct tm tmInfo; - taosLocalTime((const time_t *)&timeVal, &tmInfo); + int32_t len = 0; + + if (taosLocalTime((const time_t *)&timeVal, &tmInfo, buf) == NULL) { + len = (int32_t)strlen(buf); + goto _end; + } + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tmInfo); - int32_t len = (int32_t)strlen(buf); + len = (int32_t)strlen(buf); // add timezone string if (tzLen > 0) { @@ -1103,6 +1109,7 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * len += fracLen; } +_end: memmove(buf + VARSTR_HEADER_SIZE, buf, len); varDataSetLen(buf, len); @@ -1117,7 +1124,8 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + int32_t idx = (inputNum == 2) ? 1 : 2; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -1317,19 +1325,19 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara case 86400000: { /* 1d */ if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (86400L * 1000); + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (((int64_t)86400) * 1000); } else { timeVal = timeVal / 1000 / 86400 * 86400 * 1000; } } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (86400L * 1000000); + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (((int64_t)86400) * 1000000); } else { timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; } } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { if (ignoreTz) { - timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (86400L * 1000000000); + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (((int64_t)86400) * 1000000000); } else { timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; } @@ -1755,7 +1763,11 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { char* p = colDataGetVarData(pInput->columnData, 0); - colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows); + int32_t code = colDataSetNItems(pOutput->columnData, pOutput->numOfRows, p, pInput->numOfRows, true); + if (code) { + return code; + } + pOutput->numOfRows += pInput->numOfRows; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 85b952937fbc356879adc1b53a059767b9164e5f..7840fe2017a6f384e70f1553105c7293a4abb2d7 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -54,7 +54,6 @@ typedef enum { #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 @@ -134,7 +133,7 @@ typedef struct SSchStatusFps { typedef struct SSchedulerCfg { uint32_t maxJobNum; - int32_t maxNodeTableNum; + int64_t maxNodeTableNum; SCH_POLICY schPolicy; bool enableReSchedule; } SSchedulerCfg; @@ -175,7 +174,7 @@ typedef struct SSchHbCallbackParam { typedef struct SSchFlowControl { SRWLatch lock; bool sorted; - int32_t tableNumSum; + int64_t tableNumSum; uint32_t execTaskNum; SArray *taskList; // Element is SSchTask* } SSchFlowControl; @@ -193,7 +192,7 @@ typedef struct SSchLevel { int32_t taskSucceed; int32_t taskNum; int32_t taskLaunchedNum; - int32_t taskDoneNum; + int32_t taskExecDoneNum; SArray *subTasks; // Element is SSchTask } SSchLevel; @@ -299,6 +298,7 @@ typedef struct SSchJob { SExecResult execRes; void *fetchRes; // TODO free it or not bool fetched; + bool noMoreRetry; int64_t resNumOfRows; // from int32_t to int64_t SSchResInfo userRes; char *sql; @@ -333,13 +333,16 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) -#define SCH_UPDATE_REDICT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) -#define SCH_GET_REDICT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) +#define SCH_UPDATE_REDIRECT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_GET_REDIRECT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) +#define SCH_TASK_ALREADY_LAUNCHED(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_EXEC) +#define SCH_TASK_EXEC_DONE(task) (SCH_GET_TASK_STATUS(task) >= JOB_TASK_STATUS_PART_SUCC) + #define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL) #define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle)) @@ -361,6 +364,7 @@ extern SSchedulerMgmt schMgmt; (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY) +#define SCH_MULTI_LEVEL_LAUNCHED(_job) ((_job)->levelIdx != ((_job)->levelNum - 1)) #define SCH_SET_JOB_TYPE(_job, type) \ do { \ @@ -377,16 +381,24 @@ extern SSchedulerMgmt schMgmt; #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job)) #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) -#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ - (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect)) #define SCH_REDIRECT_MSGTYPE(_msgType) \ ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \ (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) -#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) \ - (SCH_REDIRECT_MSGTYPE(_msgType) && \ - (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) -#define SCH_NEED_RETRY(_msgType, _code) \ - ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define SCH_LOW_LEVEL_NETWORK_ERR(_job, _task, _code) \ + (SCH_NETWORK_ERR(_code) && ((_task)->level->level == (_job)->levelIdx)) +#define SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code) \ + (SCH_NETWORK_ERR(_code) && ((_task)->level->level > (_job)->levelIdx)) +#define SCH_TASK_RETRY_NETWORK_ERR(_task, _code) \ + (SCH_NETWORK_ERR(_code) && (_task)->redirectCtx.inRedirect) + +#define SCH_JOB_NEED_RETRY(_job, _task, _msgType, _code) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && SCH_TOP_LEVEL_NETWORK_ERR(_job, _task, _code)) +#define SCH_TASKSET_NEED_RETRY(_job, _task, _msgType, _code) \ + (SCH_REDIRECT_MSGTYPE(_msgType) && \ + (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_LOW_LEVEL_NETWORK_ERR((_job), (_task), (_code)) || SCH_TASK_RETRY_NETWORK_ERR((_task), (_code)))) +#define SCH_TASK_NEED_RETRY(_msgType, _code) \ + ((SCH_REDIRECT_MSGTYPE(_msgType) && SCH_NETWORK_ERR(_code)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) + #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum) #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse]) @@ -510,6 +522,11 @@ extern SSchedulerMgmt schMgmt; } \ } while (0) +#define SCH_RESET_JOB_LEVEL_IDX(_job) do { \ + (_job)->levelIdx = (_job)->levelNum - 1; \ + SCH_JOB_DLOG("set job levelIdx to %d", (_job)->levelIdx); \ +} while (0) + void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); void schCleanClusterHb(void *pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); @@ -562,7 +579,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); int32_t schExecJob(SSchJob *pJob, SSchedulerReq *pReq); int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes); int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet); -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); +int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode); void schProcessOnOpEnd(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq, int32_t errCode); int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq); void schProcessOnCbEnd(SSchJob *pJob, SSchTask *pTask, int32_t errCode); @@ -591,6 +608,10 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rspCode); int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp); +int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode); +int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode); +void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask); +int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 5e4fe4b8a1f2795cfbd4dff838304fe0e36343ff..8c2b65e1259685fcb1fb8502cbf3710faa4e426c 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -46,7 +46,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { return TSDB_CODE_SUCCESS; } - int32_t sum = 0; + int64_t sum = 0; int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); for (int32_t i = 0; i < taskNum; ++i) { SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); @@ -55,7 +55,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { } if (schMgmt.cfg.maxNodeTableNum <= 0 || sum < schMgmt.cfg.maxNodeTableNum) { - SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%d", sum); + SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%" PRId64, sum); return TSDB_CODE_SUCCESS; } @@ -68,7 +68,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { SCH_SET_JOB_NEED_FLOW_CTRL(pJob); - SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%d", sum); + SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%" PRId64, sum); return TSDB_CODE_SUCCESS; } @@ -94,7 +94,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { --ctrl->execTaskNum; ctrl->tableNumSum -= pTask->plan->execNodeStat.tableNum; - SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); _return: @@ -125,7 +125,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); *enough = true; @@ -142,7 +142,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { break; } - int32_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; + int64_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; if (sum <= schMgmt.cfg.maxNodeTableNum) { ctrl->tableNumSum = sum; @@ -173,7 +173,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { _return: - SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", + SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ((*enough) ? "" : "NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); @@ -203,7 +203,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { return TSDB_CODE_SUCCESS; } - int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; + int64_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; int32_t taskNum = taosArrayGetSize(ctrl->taskList); int32_t code = 0; SSchTask *pTask = NULL; @@ -217,7 +217,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { - SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); continue; @@ -228,14 +228,14 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { taosArrayRemove(ctrl->taskList, i); - SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum); break; @@ -244,7 +244,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); if (remainNum < pLastTask->plan->execNodeStat.tableNum) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); break; @@ -282,7 +282,6 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask) { } int32_t code = schLaunchTasksInFlowCtrlListImpl(pJob, ctrl); - ; SCH_ERR_RET(code); return code; // to avoid compiler error diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 980a8ac6a193e32c38e64f0b323c5311ddacef55..e7bfe95795f44f61ee4d1cc02873a21b80438369 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -83,6 +83,10 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { oriStatus = SCH_GET_JOB_STATUS(pJob); if (oriStatus == newStatus) { + if (JOB_TASK_STATUS_FETCH == newStatus) { + return code; + } + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -108,10 +112,19 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && - newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && + newStatus != JOB_TASK_STATUS_FETCH) { SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } + break; + case JOB_TASK_STATUS_FETCH: + if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && + newStatus != JOB_TASK_STATUS_FETCH) { + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); + } + break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: @@ -288,7 +301,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } pJob->levelNum = levelNum; - pJob->levelIdx = levelNum - 1; + SCH_RESET_JOB_LEVEL_IDX(pJob); SSchLevel level = {0}; SNodeListNode *plans = NULL; @@ -550,9 +563,9 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { } SSchLevel *pLevel = pTask->level; - int32_t doneNum = atomic_add_fetch_32(&pLevel->taskDoneNum, 1); + int32_t doneNum = atomic_add_fetch_32(&pLevel->taskExecDoneNum, 1); if (doneNum == pLevel->taskNum) { - pJob->levelIdx--; + atomic_sub_fetch_32(&pJob->levelIdx, 1); pLevel = taosArrayGet(pJob->levels, pJob->levelIdx); for (int32_t i = 0; i < pLevel->taskNum; ++i) { @@ -562,6 +575,10 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { continue; } + if (SCH_TASK_ALREADY_LAUNCHED(pTask)) { + continue; + } + SCH_ERR_RET(schLaunchTask(pJob, pTask)); } } @@ -811,6 +828,75 @@ void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode) { } } +int32_t schChkResetJobRetry(SSchJob *pJob, int32_t rspCode) { + if (pJob->status >= JOB_TASK_STATUS_PART_SUCC) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->fetched) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + pJob->noMoreRetry = true; + SCH_JOB_ELOG("already fetched while got error %s", tstrerror(rspCode)); + SCH_ERR_RET(rspCode); + } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schResetJobForRetry(SSchJob *pJob, int32_t rspCode) { + SCH_ERR_RET(schChkResetJobRetry(pJob, rspCode)); + + int32_t numOfLevels = taosArrayGetSize(pJob->levels); + for (int32_t i = 0; i < numOfLevels; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + pLevel->taskExecDoneNum = 0; + pLevel->taskLaunchedNum = 0; + + int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); + for (int32_t j = 0; j < numOfTasks; ++j) { + SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); + SCH_LOCK_TASK(pTask); + SCH_ERR_RET(schChkUpdateRedirectCtx(pJob, pTask, NULL, rspCode)); + qClearSubplanExecutionNode(pTask->plan); + schResetTaskForRetry(pJob, pTask); + SCH_UNLOCK_TASK(pTask); + } + } + + SCH_RESET_JOB_LEVEL_IDX(pJob); + + return TSDB_CODE_SUCCESS; +} + + +int32_t schHandleJobRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + + taosMemoryFreeClear(pMsg->pData); + taosMemoryFreeClear(pMsg->pEpSet); + + SCH_UNLOCK_TASK(pTask); + + SCH_TASK_DLOG("start to redirect all job tasks cause of error: %s", tstrerror(rspCode)); + + SCH_ERR_JRET(schResetJobForRetry(pJob, rspCode)); + + SCH_ERR_JRET(schLaunchJob(pJob)); + + SCH_LOCK_TASK(pTask); + + SCH_RET(code); + +_return: + + SCH_LOCK_TASK(pTask); + + SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); +} + bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { bool r = false; SCH_LOCK(SCH_READ, &pJob->opStatus.lock); @@ -907,7 +993,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq SCH_ERR_RET(TSDB_CODE_APP_ERROR); } - if (status != JOB_TASK_STATUS_PART_SUCC) { + if (status != JOB_TASK_STATUS_PART_SUCC && status != JOB_TASK_STATUS_FETCH) { SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 6f4130fd9ffe2814f0dd30655a58aefc92801a13..80fdc7594cc63b8e893de9e4e7cf193592deb885 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -36,7 +36,7 @@ int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR); } - if (taskStatus != JOB_TASK_STATUS_PART_SUCC) { + if (taskStatus != JOB_TASK_STATUS_FETCH) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%s, rspType:%s", jobTaskStatusStr(taskStatus), TMSG_INFO(msgType)); SCH_ERR_RET(TSDB_CODE_QW_MSG_ERROR); @@ -137,25 +137,12 @@ int32_t schProcessExplainRsp(SSchJob *pJob, SSchTask *pTask, SExplainRsp *rsp) { return TSDB_CODE_SUCCESS; } -// Note: no more task error processing, handled in function internal -int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { +int32_t schProcessResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; char *msg = pMsg->pData; int32_t msgSize = pMsg->len; int32_t msgType = pMsg->msgType; - bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); - if (SCH_IS_QUERY_JOB(pJob)) { - SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); - } - - SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType)); - - int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); - if (SCH_TASK_NEED_REDIRECT(pTask, reqType, rspCode, pMsg->len)) { - SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode)); - } - pTask->redirectCtx.inRedirect = false; switch (msgType) { @@ -423,6 +410,38 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } + + +// Note: no more task error processing, handled in function internal +int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + int32_t msgType = pMsg->msgType; + char *msg = pMsg->pData; + + bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || SCH_NETWORK_ERR(rspCode)); + if (SCH_IS_QUERY_JOB(pJob)) { + SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, execId)); + } + + SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType)); + + int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1); + if (SCH_JOB_NEED_RETRY(pJob, pTask, reqType, rspCode)) { + SCH_RET(schHandleJobRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); + } else if (SCH_TASKSET_NEED_RETRY(pJob, pTask, reqType, rspCode)) { + SCH_RET(schHandleTaskSetRetry(pJob, pTask, (SDataBuf *)pMsg, rspCode)); + } + + pTask->redirectCtx.inRedirect = false; + + SCH_RET(schProcessResponseMsg(pJob, pTask, execId, pMsg, rspCode)); + +_return: + + taosMemoryFreeClear(msg); + + SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); +} int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 4c16a81a05003166978dd9ee0f96b2c876f6fefd..9d0ad30e2a47de92136ec1f25bffa6f761ea7725 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -34,6 +34,9 @@ int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { case JOB_TASK_STATUS_PART_SUCC: SCH_ERR_JRET(schProcessOnJobPartialSuccess(pJob)); break; + case JOB_TASK_STATUS_FETCH: + SCH_ERR_JRET(schJobFetchRows(pJob)); + break; case JOB_TASK_STATUS_SUCC: break; case JOB_TASK_STATUS_FAIL: diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index bdab7393277617afa8dbd183700d54efa2caf4a7..207753ae2571fe292ca009cb6af51030bf0667f0 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -378,7 +378,8 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, if (lastTime > tsMaxRetryWaitTime) { SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); - SCH_ERR_RET(SCH_GET_REDICT_CODE(pJob, rspCode)); + pJob->noMoreRetry = true; + SCH_ERR_RET(SCH_GET_REDIRECT_CODE(pJob, rspCode)); } pCtx->periodMs *= tsRedirectFactor; @@ -404,32 +405,35 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { - int32_t code = 0; - - SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); - - if (NULL == pData) { - pTask->retryTimes = 0; - } - - if (!NO_RET_REDIRECT_ERROR(rspCode)) { - SCH_UPDATE_REDICT_CODE(pJob, rspCode); - } - - SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); - +void schResetTaskForRetry(SSchJob *pJob, SSchTask *pTask) { pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); + if (pTask->delayTimer) { + taosTmrStopA(&pTask->delayTimer); + } taosHashClear(pTask->execNodes); schRemoveTaskFromExecList(pJob, pTask); schDeregisterTaskHb(pJob, pTask); - atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); taosMemoryFreeClear(pTask->msg); pTask->msgLen = 0; pTask->lastMsgType = 0; + pTask->childReady = 0; memset(&pTask->succeedAddr, 0, sizeof(pTask->succeedAddr)); +} + +int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { + int32_t code = 0; + + SCH_TASK_DLOG("task will be redirected now, status:%s, code:%s", SCH_GET_TASK_STATUS_STR(pTask), tstrerror(rspCode)); + + if (!NO_RET_REDIRECT_ERROR(rspCode)) { + SCH_UPDATE_REDIRECT_CODE(pJob, rspCode); + } + + SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); + + schResetTaskForRetry(pJob, pTask); if (SCH_IS_DATA_BIND_TASK(pTask)) { if (pData && pData->pEpSet) { @@ -445,12 +449,6 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 SCH_TASK_DLOG("switch task target node %d epset to %d/%d", addr->nodeId, addr->epSet.inUse, addr->epSet.numOfEps); } - if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { - if (JOB_TASK_STATUS_EXEC == SCH_GET_TASK_STATUS(pTask)) { - SCH_ERR_JRET(schLaunchTasksInFlowCtrlList(pJob, pTask)); - } - } - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); SCH_ERR_JRET(schDelayLaunchTask(pJob, pTask)); @@ -486,20 +484,10 @@ _return: SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } -int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { +int32_t schHandleTaskSetRetry(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; - if (JOB_TASK_STATUS_PART_SUCC == pJob->status) { - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->fetched) { - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); - SCH_ERR_JRET(rspCode); - } - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); - - schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); - } + SCH_ERR_JRET(schChkResetJobRetry(pJob, rspCode)); if (SYNC_OTHER_LEADER_REDIRECT_ERROR(rspCode)) { if (NULL == pData->pEpSet) { @@ -509,7 +497,19 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 } } + SCH_TASK_DLOG("start to redirect current task set cause of error: %s", tstrerror(rspCode)); + + for (int32_t i = 0; i < pJob->levelNum; ++i) { + SSchLevel *pLevel = taosArrayGet(pJob->levels, i); + + pLevel->taskExecDoneNum = 0; + pLevel->taskLaunchedNum = 0; + } + + SCH_RESET_JOB_LEVEL_IDX(pJob); + code = schDoTaskRedirect(pJob, pTask, pData, rspCode); + taosMemoryFreeClear(pData->pData); taosMemoryFreeClear(pData->pEpSet); @@ -621,6 +621,13 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { */ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { + if (pJob->noMoreRetry) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since job no more retry, retryTimes:%d/%d", pTask->retryTimes, + pTask->maxRetryTimes); + return TSDB_CODE_SUCCESS; + } + if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { pTask->maxExecTimes++; pTask->maxRetryTimes++; @@ -645,7 +652,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } - if (!SCH_NEED_RETRY(pTask->lastMsgType, errCode)) { + if (!SCH_TASK_NEED_RETRY(pTask->lastMsgType, errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry cause of errCode, errCode:%x - %s", errCode, tstrerror(errCode)); return TSDB_CODE_SUCCESS; @@ -1067,7 +1074,6 @@ int32_t schLaunchTaskImpl(void *param) { SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } - // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXEC) { SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXEC); @@ -1272,6 +1278,8 @@ int32_t schLaunchFetchTask(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } + SCH_SET_TASK_STATUS(pJob->fetchTask, JOB_TASK_STATUS_FETCH); + if (SCH_IS_LOCAL_EXEC_TASK(pJob, pJob->fetchTask)) { SCH_ERR_JRET(schExecLocalFetch(pJob, pJob->fetchTask)); } else { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 7cd5e957b6b8d47871274f80632a65ec11485753..e7561ccb7ecf68c5030e12a19c04f3d1a9661cb5 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -18,6 +18,7 @@ #include "schInt.h" #include "tmsg.h" #include "tref.h" +#include "tglobal.h" SSchedulerMgmt schMgmt = { .jobRef = -1, @@ -30,11 +31,12 @@ int32_t schedulerInit() { } schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; - schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; + schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables; schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; schMgmt.cfg.enableReSchedule = true; - qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy); + qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", + schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { @@ -91,7 +93,7 @@ int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq *pReq) { SCH_ERR_JRET(schHandleOpBeginEvent(jobId, &pJob, SCH_OP_FETCH, pReq)); - SCH_ERR_JRET(schJobFetchRows(pJob)); + SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_FETCH, pReq)); _return: diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index ceddf4f2153735c4d8423dde806272419dc35fc5..790547bb6154e6a53847f609d918fe4fac784c29 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( stream PUBLIC tdb - PRIVATE os util transport qcom executor + PRIVATE os util transport qcom executor wal ) if(${BUILD_TEST}) diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 66496f11f8b2da8ad46d91637efdeb1444f26319..876b80697a9679125350f8b9b2a1e26e4fdb0572 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); #ifdef __cplusplus } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 60729c4d0e5a292990eae57bccbdd3cc4c783ab9..0f000f1f5017e6bf1cd737ee424a4ed72654ac5f 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,6 +16,8 @@ #include "streamInc.h" #include "ttimer.h" +#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 100000 + int32_t streamInit() { int8_t old; while (1) { @@ -50,7 +52,7 @@ void streamCleanUp() { void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(NULL, pTask); return; } @@ -64,15 +66,16 @@ void streamSchedByTimer(void* param, void* tmrId) { taosFreeQitem(trigger); return; } - trigger->pBlock->info.type = STREAM_GET_ALL; + trigger->pBlock->info.type = STREAM_GET_ALL; atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) { taosFreeQitem(trigger); taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); return; } + streamSchedExec(pTask); } @@ -91,31 +94,33 @@ int32_t streamSetupTrigger(SStreamTask* pTask) { int32_t streamSchedExec(SStreamTask* pTask) { int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); return -1; } + pRunReq->head.vgId = pTask->nodeId; - pRunReq->streamId = pTask->streamId; - pRunReq->taskId = pTask->taskId; - SRpcMsg msg = { - .msgType = TDMT_STREAM_TASK_RUN, - .pCont = pRunReq, - .contLen = sizeof(SStreamTaskRunReq), - }; + pRunReq->streamId = pTask->id.streamId; + pRunReq->taskId = pTask->id.taskId; + + SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) }; tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); + qDebug("trigger to run s-task:%s", pTask->id.idStr); } + return 0; } -int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { +int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); int8_t status; - // enqueue + // enqueue data block if (pData != NULL) { pData->type = STREAM_INPUT__DATA_BLOCK; pData->srcVgId = pReq->dataSrcVgId; @@ -123,10 +128,10 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamDispatchReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; - } else { - status = TASK_INPUT_STATUS__FAILED; + } else { // input queue is full, upstream is blocked now + status = TASK_INPUT_STATUS__BLOCKED; } } else { streamTaskInputFail(pTask); @@ -142,10 +147,12 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR pCont->upstreamNodeId = htonl(pReq->upstreamNodeId); pCont->upstreamTaskId = htonl(pReq->upstreamTaskId); pCont->downstreamNodeId = htonl(pTask->nodeId); - pCont->downstreamTaskId = htonl(pTask->taskId); + pCont->downstreamTaskId = htonl(pTask->id.taskId); pRsp->pCont = buf; + pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } @@ -155,7 +162,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pReq->srcTaskId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; @@ -164,7 +171,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; @@ -205,10 +212,10 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { } int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { - qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, + qDebug("vgId:%d s-task:%s receive dispatch req from taskId:%d", pReq->upstreamNodeId, pTask->id.idStr, pReq->upstreamTaskId); - streamTaskEnqueue(pTask, pReq, pRsp); + streamTaskEnqueueBlocks(pTask, pReq, pRsp); tDeleteStreamDispatchReq(pReq); if (exec) { @@ -228,13 +235,14 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - - qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code); + qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp); - if (leftRsp > 0) return 0; + qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + if (leftRsp > 0) { + return 0; + } } int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); @@ -261,7 +269,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); + qDebug("task %d receive retrieve req from node %d task %d", pTask->id.taskId, pReq->srcNodeId, pReq->srcTaskId); streamTaskEnqueueRetrieve(pTask, pReq, pRsp); @@ -275,7 +283,77 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { - // +bool tInputQueueIsFull(const SStreamTask* pTask) { + return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY; +} + +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { + int8_t type = pItem->type; + + if (type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); + if (pSubmitBlock == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + return -1; + } + + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + qDebug("s-task:%s submit enqueue %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr, + pItem, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, + pSubmitBlock->submit.ver, total); + + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + streamDataSubmitDestroy(pSubmitBlock); + return -1; + } + + taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); + } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || + type == STREAM_INPUT__REF_DATA_BLOCK) { + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + return -1; + } + + qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, total); + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__CHECKPOINT) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__GET_RES) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } + + if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + } + +#if 0 + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); +#endif + return 0; } + +void* streamQueueNextItem(SStreamQueue* queue) { + int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); + if (dequeueFlag == STREAM_QUEUE__FAILED) { + ASSERT(queue->qItem != NULL); + return streamQueueCurItem(queue); + } else { + queue->qItem = NULL; + taosGetQitem(queue->qall, &queue->qItem); + if (queue->qItem == NULL) { + taosReadAllQitems(queue->queue, queue->qall); + taosGetQitem(queue->qall, &queue->qItem); + } + return streamQueueCurItem(queue); + } +} + +void streamTaskInputFail(SStreamTask* pTask) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); +} \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8baebaee4224e6b9af5d2eacf62910412171bc63..ae616260f3b38038f103b661b4a05aabace9ac31 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -48,10 +48,12 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock if (pArray == NULL) { return -1; } + taosArrayPush(pArray, &(SSDataBlock){0}); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); blockDecode(pDataBlock, pRetrieve->data); + // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -65,35 +67,53 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) { SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) { + return NULL; + } - if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); - if (pDataSubmit->dataRef == NULL) goto FAIL; + if (pDataSubmit->dataRef == NULL) { + taosFreeQitem(pDataSubmit); + return NULL; + } + pDataSubmit->submit = submit; - *pDataSubmit->dataRef = 1; - pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; + *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 + pDataSubmit->type = type; + return pDataSubmit; -FAIL: - taosFreeQitem(pDataSubmit); - return NULL; +} + +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit) { + int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); + ASSERT(ref >= 0 && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + if (ref == 0) { + taosMemoryFree(pDataSubmit->submit.msgStr); + taosMemoryFree(pDataSubmit->dataRef); + } } SStreamMergedSubmit2* streamMergedSubmitNew() { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) { + return NULL; + } - if (pMerged == NULL) return NULL; pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; + + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) { + taosArrayDestroy(pMerged->submits); + taosArrayDestroy(pMerged->dataRefs); + taosFreeQitem(pMerged); + return NULL; + } + pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; -FAIL: - if (pMerged->submits) taosArrayDestroy(pMerged->submits); - if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); - taosFreeQitem(pMerged); - return NULL; } int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { @@ -107,48 +127,38 @@ static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) { SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); - if (pSubmitClone == NULL) { return NULL; } + streamDataSubmitRefInc(pSubmit); memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { - int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); - ASSERT(ref >= 0); - if (ref == 0) { - taosMemoryFree(pDataSubmit->submit.msgStr); - taosMemoryFree(pDataSubmit->dataRef); - } -} - -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { - ASSERT(elem); - if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; - SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)elem; + SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; - SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)pElem; streamMergeSubmit(pMerged, pBlockSrc); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)pElem); taosFreeQitem(dst); - taosFreeQitem(elem); + taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { return NULL; @@ -164,7 +174,7 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit2*)data); + streamDataSubmitDestroy((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7e7c23f98addfae5689093bedb1a86196cd380c9..549374ed9402b3abb775e804d472845a47c44f53 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -121,9 +121,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .srcNodeId = pTask->nodeId, - .srcTaskId = pTask->taskId, + .srcTaskId = pTask->id.taskId, .pRetrieve = pRetrieve, .retrieveLen = dataStrLen, }; @@ -168,7 +168,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) } buf = NULL; - qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId, + qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr, pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; @@ -230,22 +230,21 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { - goto FAIL; + rpcFreeCont(buf); + return code; } + tEncoderClear(&encoder); msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId); + qDebug("dispatch from s-task:%s to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, + pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); - return 0; -FAIL: - if (buf) rpcFreeCont(buf); - return code; } int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, @@ -282,7 +281,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov tmsgSendReq(pEpSet, &msg); - qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId); + qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->id.taskId, pReq->taskId, vgId); return 0; FAIL: @@ -319,8 +318,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId); - + qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); code = 0; @@ -382,9 +380,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { SStreamDispatchReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .dataSrcVgId = pData->srcVgId, - .upstreamTaskId = pTask->taskId, + .upstreamTaskId = pTask->id.taskId, .upstreamChildId = pTask->selfChildId, .upstreamNodeId = pTask->nodeId, .blockNum = blockNum, @@ -402,14 +400,15 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat goto FAIL_FIXED_DISPATCH; } } + int32_t vgId = pTask->fixedEpDispatcher.nodeId; SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; req.taskId = downstreamTaskId; - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); + qDebug("s-task:%s (child taskId:%d) dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, + pTask->selfChildId, blockNum, downstreamTaskId, vgId); if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { goto FAIL_FIXED_DISPATCH; @@ -432,9 +431,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } for (int32_t i = 0; i < vgSz; i++) { - pReqs[i].streamId = pTask->streamId; + pReqs[i].streamId = pTask->id.streamId; pReqs[i].dataSrcVgId = pData->srcVgId; - pReqs[i].upstreamTaskId = pTask->taskId; + pReqs[i].upstreamTaskId = pTask->id.taskId; pReqs[i].upstreamChildId = pTask->selfChildId; pReqs[i].upstreamNodeId = pTask->nodeId; pReqs[i].blockNum = 0; @@ -494,6 +493,8 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); + qDebug("s-task:%s try to dispatch intermediate result block to downstream, numofBlocks in outputQ:%d", pTask->id.idStr, + taosQueueItemSize(pTask->outputQueue->queue)); int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); @@ -503,13 +504,12 @@ int32_t streamDispatch(SStreamTask* pTask) { SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue); if (pBlock == NULL) { - qDebug("stream stop dispatching since no output: task %d", pTask->taskId); + qDebug("s-task:%s stream stop dispatching since no output in output queue", pTask->id.idStr); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return 0; } - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream dispatching: task %d", pTask->taskId); + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { @@ -518,6 +518,7 @@ int32_t streamDispatch(SStreamTask* pTask) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); goto FREE; } + FREE: taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cb9774b584937bc0846a0e761cceda37036b83d4..9a6ff302efc3b470f9e71caf07e2d0ae9d70a13f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,70 +18,82 @@ #define STREAM_EXEC_MAX_BATCH_NUM 100 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - int32_t code; - void* exec = pTask->exec.executor; - while(atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { - qError("stream task wait for the end of fill history"); - taosMsleep(2); - continue; + int32_t code = TSDB_CODE_SUCCESS; + void* pExecutor = pTask->exec.pExecutor; + + while (pTask->taskLevel == TASK_LEVEL__SOURCE) { + int8_t status = atomic_load_8(&pTask->status.taskStatus); + if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) { + qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, + atomic_load_8(&pTask->status.taskStatus)); + taosMsleep(2); + } else { + break; + } } // set input const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; - qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); - qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; - SArray* blocks = pBlock->blocks; - qDebug("task %d %p set ssdata input", pTask->taskId, pTask); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); + + SArray* pBlockList = pBlock->blocks; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%"PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; - SArray* blocks = pMerged->submits; - qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); + + SArray* pBlockList = pMerged->submits; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("st-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; - qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else { ASSERT(0); } - // exec + // pExecutor while (1) { - if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { return 0; } SSDataBlock* output = NULL; uint64_t ts = 0; - if ((code = qExecTask(exec, &output, &ts)) < 0) { + if ((code = qExecTask(pExecutor, &output, &ts)) < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { - resetTaskInfo(exec); + resetTaskInfo(pExecutor); } - /*ASSERT(false);*/ - qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, - terrstr()); + + qError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, terrstr()); continue; } + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; + SSDataBlock block = {0}; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pRetrieveBlock->reqId); } break; @@ -94,20 +106,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("task %d(child %d) executed and get block", pTask->taskId, pTask->selfChildId); + qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); SSDataBlock block = {0}; assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + return 0; } int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); bool finished = false; @@ -121,7 +134,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { taosArrayDestroy(pRes); return 0; } @@ -147,13 +160,19 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { batchCnt++; - qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz); + qDebug("task %d scan exec block num %d, block limit %d", pTask->id.taskId, batchCnt, batchSz); if (batchCnt >= batchSz) break; } if (taosArrayGetSize(pRes) == 0) { - taosArrayDestroy(pRes); - break; + if (finished) { + taosArrayDestroy(pRes); + qDebug("task %d finish recover exec task ", pTask->id.taskId); + break; + } else { + qDebug("task %d continue recover exec task ", pTask->id.taskId); + continue; + } } SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { @@ -167,10 +186,13 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { streamTaskOutput(pTask, qRes); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt); + qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt); streamDispatch(pTask); } - if (finished) break; + + if (finished) { + break; + } } return 0; } @@ -180,7 +202,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { // fetch all queue item, merge according to batchLimit int32_t numOfItems = taosReadAllQitems(pTask->inputQueue1, pTask->inputQall); if (numOfItems == 0) { - qDebug("task: %d, stream task exec over, queue empty", pTask->taskId); + qDebug("task: %d, stream task exec over, queue empty", pTask->id.taskId); return 0; } SStreamQueueItem* pMerged = NULL; @@ -215,105 +237,141 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t batchCnt = 1; - void* input = NULL; + int32_t batchSize = 1; + void* pInput = NULL; + + // merge multiple input data if possible in the input queue. while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { - qDebug("stream task exec over, queue empty, task: %d", pTask->taskId); +// qDebug("s-task:%s extract data from input queue, queue is empty, abort", pTask->id.idStr); break; } - if (input == NULL) { - input = qItem; + + if (pInput == NULL) { + pInput = qItem; streamQueueProcessSuccess(pTask->inputQueue); if (pTask->taskLevel == TASK_LEVEL__SINK) { break; } } else { - void* newRet; - if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) { + void* newRet = NULL; + if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - batchCnt++; - input = newRet; + batchSize++; + pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); - if (batchCnt > STREAM_EXEC_MAX_BATCH_NUM) { + if (batchSize > STREAM_EXEC_MAX_BATCH_NUM) { break; } } } } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (input) streamFreeQitem(input); + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (pInput) { + streamFreeQitem(pInput); + } return 0; } - if (input == NULL) { + if (pInput == NULL) { break; } if (pTask->taskLevel == TASK_LEVEL__SINK) { - ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, input); + ASSERT(((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_BLOCK); + qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); + streamTaskOutput(pTask, pInput); continue; } SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + qDebug("s-task:%s exec begin, numOfBlocks:%d", pTask->id.idStr, batchSize); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); - streamTaskExecImpl(pTask, input, pRes); - qDebug("stream task %d exec end", pTask->taskId); + streamTaskExecImpl(pTask, pInput, pRes); + + int64_t ckId = 0; + int64_t dataVer = 0; + qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); + if (dataVer > pTask->chkInfo.version) { // save it since the checkpoint is updated + qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); + + pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; + + taosWLockLatch(&pTask->pMeta->lock); + streamMetaSaveTask(pTask->pMeta, pTask); + if (streamMetaCommit(pTask->pMeta) < 0) { + taosWUnLockLatch(&pTask->pMeta->lock); + qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr()); + return -1; + } else { + taosWUnLockLatch(&pTask->pMeta->lock); + qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr); + } + } else { + qDebug("s-task:%s exec end", pTask->id.idStr); + } if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); return -1; } + qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; + if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; - } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; + } else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); taosFreeQitem(qRes); return -1; } } else { taosArrayDestroy(pRes); } - streamFreeQitem(input); + streamFreeQitem(pInput); } return 0; } int32_t streamTryExec(SStreamTask* pTask) { + // this function may be executed by multi-threads, so status check is required. int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); if (code < 0) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + + // todo the task should be commit here + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed", pTask->id.idStr); if (!taosQueueEmpty(pTask->inputQueue->queue)) { streamSchedExec(pTask); } } + return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 66d98e90bfaa7f81e5c985de205d245c9a3b0120..860514bb44fc97c4c0191fc16c2ff070230d4085 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -18,11 +18,13 @@ #include "ttimer.h" SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) { + int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + int32_t len = strlen(path) + 20; char* streamPath = taosMemoryCalloc(1, len); sprintf(streamPath, "%s/%s", path, "stream"); @@ -33,7 +35,12 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } sprintf(streamPath, "%s/%s", pMeta->path, "checkpoints"); - taosMulModeMkDir(streamPath, 0755); + code = taosMulModeMkDir(streamPath, 0755); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + taosMemoryFree(streamPath); + goto _err; + } taosMemoryFree(streamPath); if (tdbTbOpen("task.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { @@ -44,7 +51,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + pMeta->pTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK); if (pMeta->pTasks == NULL) { goto _err; } @@ -53,9 +61,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; - + taosInitRWLatch(&pMeta->lock); return pMeta; _err: @@ -75,19 +84,28 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbClose(pMeta->db); void* pIter = NULL; +// while(pMeta->walScan) { +// qDebug("wait stream daemon quit"); +// taosMsleep(100); +// } + while (1) { pIter = taosHashIterate(pMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->timer) { taosTmrStop(pTask->timer); pTask->timer = NULL; } - tFreeSStreamTask(pTask); + + tFreeStreamTask(pTask); /*streamMetaReleaseTask(pMeta, pTask);*/ } + taosHashCleanup(pMeta->pTasks); - taosHashCleanup(pMeta->pRecoverStatus); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); } @@ -100,7 +118,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, } SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSStreamTask(&decoder, pTask) < 0) { + if (tDecodeStreamTask(&decoder, pTask) < 0) { tDecoderClear(&decoder); goto FAIL; } @@ -111,12 +129,12 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, goto FAIL; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { goto FAIL; } - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { - taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { + taosHashRemove(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t)); ASSERT(0); goto FAIL; } @@ -124,7 +142,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, return 0; FAIL: - if (pTask) tFreeSStreamTask(pTask); + if (pTask) tFreeStreamTask(pTask); return -1; } #endif @@ -133,7 +151,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; int32_t len; int32_t code; - tEncodeSize(tEncodeSStreamTask, pTask, len, code); + tEncodeSize(tEncodeStreamTask, pTask, len, code); if (code < 0) { return -1; } @@ -144,10 +162,10 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { return -1; } @@ -155,8 +173,8 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -#if 1 -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { +// add to the ready tasks hash map, not the restored tasks hash map +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { return -1; } @@ -165,39 +183,24 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { return -1; } - taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); - + taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES); return 0; } -#endif -#if 0 -SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - ASSERT((*ppTask)->taskId == taskId); - return *ppTask; - } else { - return NULL; - } +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { + return (int32_t) taosHashGetSize(pMeta->pTasks); } -#endif SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__DROPPING) { - atomic_add_fetch_32(&pTask->refCnt, 1); - taosRUnLockLatch(&pMeta->lock); - return pTask; - } else { - taosRUnLockLatch(&pMeta->lock); - return NULL; - } + if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { + atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + taosRUnLockLatch(&pMeta->lock); + return *ppTask; } + taosRUnLockLatch(&pMeta->lock); return NULL; } @@ -206,8 +209,8 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); ASSERT(left >= 0); if (left == 0) { - ASSERT(atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING); - tFreeSStreamTask(pTask); + ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); + tFreeStreamTask(pTask); } } @@ -221,7 +224,7 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { * taosTmrStop(pTask->timer);*/ /*pTask->timer = NULL;*/ /*}*/ - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); @@ -239,9 +242,12 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { int32_t streamMetaCommit(SStreamMeta* pMeta) { if (tdbCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } + if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } @@ -287,25 +293,26 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSStreamTask(&decoder, pTask); + tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - /*pTask->taskStatus = TASK_STATUS__NORMAL;*/ + + /*pTask->status.taskStatus = TASK_STATUS__NORMAL;*/ if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; streamTaskCheckDownstream(pTask, ver); } } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 87058bf490d43f2cbbd1b18eb8411c034b40a9cd..55c745e4173ffadc6869d52627d43a8abafa84e9 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -16,9 +16,10 @@ #include "streamInc.h" int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { - qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId); + qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId); + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); @@ -33,34 +34,31 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { memcpy(serializedReq, &req, len); - SRpcMsg rpcMsg = { - .contLen = len, - .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, - }; - + SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE }; if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { /*ASSERT(0);*/ } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); streamSetParamForRecover(pTask); streamAggRecoverPrepare(pTask); } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); } + return 0; } // checkstatus int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { SStreamTaskCheckReq req = { - .streamId = pTask->streamId, - .upstreamTaskId = pTask->taskId, + .streamId = pTask->id.streamId, + .upstreamTaskId = pTask->id.taskId, .upstreamNodeId = pTask->nodeId, .childId = pTask->selfChildId, }; + // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { req.reqId = tGenIdPI64(); @@ -68,7 +66,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId, + qDebug("task %d at node %d check downstream task %d at node %d", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -83,12 +81,12 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId, + qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId); + qDebug("task %d at node %d direct launch recover since no downstream", pTask->id.taskId, pTask->nodeId); streamTaskLaunchRecover(pTask, version); } return 0; @@ -104,8 +102,10 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .downstreamNodeId = pRsp->downstreamNodeId, .childId = pRsp->childId, }; - qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId, + + qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -118,16 +118,18 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } } } + return 0; } int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) { - return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL; + return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL; } int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); + if (pRsp->status == 1) { if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; @@ -138,7 +140,11 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* break; } } - if (!found) return -1; + + if (!found) { + return -1; + } + int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); ASSERT(left >= 0); if (left == 0) { @@ -147,41 +153,46 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - if (pRsp->reqId != pTask->checkReqId) return -1; + if (pRsp->reqId != pTask->checkReqId) { + return -1; + } + streamTaskLaunchRecover(pTask, version); } else { ASSERT(0); } - } else { + } else { // not ready, it should wait for at least 100ms and then retry streamRecheckOneDownstream(pTask, pRsp); } + return 0; } // common int32_t streamSetParamForRecover(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSetParamForRecover(exec); } int32_t streamRestoreParam(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamRestoreParam(exec); } + int32_t streamSetStatusNormal(SStreamTask* pTask) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); return 0; } // source int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSourceRecoverStep1(exec, ver); } int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } @@ -192,13 +203,13 @@ int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamSourceRecoverStep2(exec, ver) < 0) { } return streamScanExec(pTask, 100); @@ -206,7 +217,7 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { SStreamRecoverFinishReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .childId = pTask->selfChildId, }; // serialize @@ -227,13 +238,13 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { - void* exec = pTask->exec.executor; pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); + qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream); return 0; } int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamRestoreParam(exec) < 0) { return -1; } @@ -247,6 +258,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { if (pTask->taskLevel == TASK_LEVEL__AGG) { int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); + qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left); ASSERT(left >= 0); if (left == 0) { streamAggChildrenRecoverFinish(pTask); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index a2b3e20dbfbab336ef50ef786daff4ccba95858a..7bea989e3a63be5110deb9e8dbb5b00ac6eed643 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -121,7 +121,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->taskId); + sprintf(statePath, "%s/%d", path, pTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); @@ -130,21 +130,25 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int char cfgPath[1030]; sprintf(cfgPath, "%s/cfg", statePath); + szPage = szPage < 0 ? 4096 : szPage; + pages = pages < 0 ? 256 : pages; char cfg[1024]; memset(cfg, 0, 1024); TdFilePtr pCfgFile = taosOpenFile(cfgPath, TD_FILE_READ); if (pCfgFile != NULL) { - int64_t size; + int64_t size = 0; taosFStatFile(pCfgFile, &size, NULL); - taosReadFile(pCfgFile, cfg, size); - sscanf(cfg, "%d\n%d\n", &szPage, &pages); + if (size > 0) { + taosReadFile(pCfgFile, cfg, size); + sscanf(cfg, "%d\n%d\n", &szPage, &pages); + } } else { - taosMulModeMkDir(statePath, 0755); - pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); - szPage = szPage < 0 ? 4096 : szPage; - pages = pages < 0 ? 256 : pages; - sprintf(cfg, "%d\n%d\n", szPage, pages); - taosWriteFile(pCfgFile, cfg, strlen(cfg)); + int32_t code = taosMulModeMkDir(statePath, 0755); + if (code == 0) { + pCfgFile = taosOpenFile(cfgPath, TD_FILE_WRITE | TD_FILE_CREATE); + sprintf(cfg, "%d\n%d\n", szPage, pages); + taosWriteFile(pCfgFile, cfg, strlen(cfg)); + } } taosCloseFile(&pCfgFile); @@ -189,6 +193,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int } pState->pTdbState->pOwner = pTask; + pState->checkPointId = 0; return pState; @@ -239,6 +244,7 @@ int32_t streamStateCommit(SStreamState* pState) { TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } + pState->checkPointId++; return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index e9aba0bc3941d501ba25a5ea28aaf02e11d42cbd..67c60008fdb98b5f341be5271668a17bd3563c92 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -15,15 +15,22 @@ #include "executor.h" #include "tstream.h" +#include "wal.h" -SStreamTask* tNewSStreamTask(int64_t streamId) { +SStreamTask* tNewStreamTask(int64_t streamId) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return NULL; } - pTask->taskId = tGenIdPI32(); - pTask->streamId = streamId; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + + pTask->id.taskId = tGenIdPI32(); + pTask->id.streamId = streamId; + + char buf[128] = {0}; + sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId); + + pTask->id.idStr = taosStrdup(buf); + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; @@ -48,24 +55,24 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { return 0; } -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->schedStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1; if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->recoverSnapVer) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->startVer) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1; if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); @@ -101,24 +108,24 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { return pEncoder->pos; } -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->schedStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->recoverSnapVer) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->startVer) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1; int32_t epSz; @@ -162,24 +169,47 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { return 0; } -void tFreeSStreamTask(SStreamTask* pTask) { - qDebug("free stream task %d", pTask->taskId); - if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); - if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); - if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); - if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); +void tFreeStreamTask(SStreamTask* pTask) { + qDebug("free s-task:%s", pTask->id.idStr); + + if (pTask->inputQueue) { + streamQueueClose(pTask->inputQueue); + } + if (pTask->outputQueue) { + streamQueueClose(pTask->outputQueue); + } + if (pTask->exec.qmsg) { + taosMemoryFree(pTask->exec.qmsg); + } + + if (pTask->exec.pExecutor) { + qDestroyTask(pTask->exec.pExecutor); + pTask->exec.pExecutor = NULL; + } + + if (pTask->exec.pWalReader != NULL) { + walCloseReader(pTask->exec.pWalReader); + } + taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); if (pTask->outputType == TASK_OUTPUT__TABLE) { tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); } + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; } - if (pTask->pState) streamStateClose(pTask->pState); + if (pTask->pState) { + streamStateClose(pTask->pState); + } + + if (pTask->id.idStr != NULL) { + taosMemoryFree((void*)pTask->id.idStr); + } taosMemoryFree(pTask); } diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 4e6ab284f8cf9b10a06f5880abd8f14e7c13de81..a4e83dc4956160648362b55900b5131ae80998cf 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -24,12 +24,13 @@ extern "C" { // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { - SRaftId (*replicas)[TSDB_MAX_REPLICA]; - SyncIndex index[TSDB_MAX_REPLICA]; - SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function - int64_t startTimeArr[TSDB_MAX_REPLICA]; - int64_t recvTimeArr[TSDB_MAX_REPLICA]; + SRaftId (*replicas)[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex index[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // for advanced function + int64_t startTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int64_t recvTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; + int32_t totalReplicaNum; SSyncNode *pNode; } SSyncIndexMgr; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6f2c1a1ad0e76d233d516e36ee419cae43fcc409..7d336c83135a20449f09431c2d9427a6dd6cc0c5 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -54,13 +54,13 @@ typedef struct SSyncLogReplMgr SSyncLogReplMgr; #define MAX_CONFIG_INDEX_COUNT 256 typedef struct SRaftCfg { - SSyncCfg cfg; - int32_t batchSize; - int8_t isStandBy; - int8_t snapshotStrategy; - SyncIndex lastConfigIndex; - int32_t configIndexCount; - SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; + SSyncCfg cfg; + int32_t batchSize; + int8_t isStandBy; + int8_t snapshotStrategy; + SyncIndex lastConfigIndex; + int32_t configIndexCount; + SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; } SRaftCfg; typedef struct SRaftId { @@ -127,12 +127,13 @@ typedef struct SSyncNode { SRaftId myRaftId; int32_t peersNum; - SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; - SEpSet peersEpset[TSDB_MAX_REPLICA]; - SRaftId peersId[TSDB_MAX_REPLICA]; + SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SEpSet peersEpset[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SRaftId peersId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; - SRaftId replicasId[TSDB_MAX_REPLICA]; + int32_t totalReplicaNum; + SRaftId replicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // raft algorithm SSyncFSM* pFsm; @@ -188,7 +189,7 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // peer heartbeat timer - SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA]; + SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // tools SSyncRespMgr* pSyncRespMgr; @@ -196,13 +197,13 @@ typedef struct SSyncNode { // restore state bool restoreFinish; // SSnapshot* pSnapshot; - SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; + SSyncSnapshotSender* senders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; // log replication mgr - SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA]; + SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; - SPeerState peerStates[TSDB_MAX_REPLICA]; + SPeerState peerStates[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // is config changing bool changing; @@ -275,6 +276,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term); void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm); void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr); +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr); void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr); void syncNodeCandidate2Leader(SSyncNode* pSyncNode); void syncNodeFollower2Candidate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index c3566c7c820b97c4484f15ce59239d423db9d480..f8c96d8be296f136292996daed85f78d2b4aa698 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -237,6 +237,7 @@ typedef struct SyncLeaderTransfer { typedef enum { SYNC_LOCAL_CMD_STEP_DOWN = 100, SYNC_LOCAL_CMD_FOLLOWER_CMT, + SYNC_LOCAL_CMD_LEARNER_CMT, } ESyncLocalCmd; typedef struct SyncLocalCmd { diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index a1de2ee71ac12fba4a43676c589c9acb323492ac..02790732a23e02e21a5708169780e222711754c2 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -56,39 +56,42 @@ typedef struct SSyncLogBuffer { int64_t size; TdThreadMutex mutex; TdThreadMutexAttr attr; + int64_t totalIndex; + bool isCatchup; } SSyncLogBuffer; // SSyncLogRepMgr -SSyncLogReplMgr* syncLogReplMgrCreate(); -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +SSyncLogReplMgr* syncLogReplCreate(); +void syncLogReplDestroy(SSyncLogReplMgr* pMgr); +void syncLogReplReset(SSyncLogReplMgr* pMgr); -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); -void syncNodeLogReplMgrDestroy(SSyncNode* pNode); +int32_t syncNodeLogReplInit(SSyncNode* pNode); +void syncNodeLogReplDestroy(SSyncNode* pNode); // access -static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int64_t syncLogReplGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { return ((int64_t)1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; } -static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF); } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier); -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); + +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); // SSyncLogBuffer SSyncLogBuffer* syncLogBufferCreate(); @@ -100,6 +103,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode); int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf); SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf); bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf); + int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry); int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm); int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm); @@ -111,8 +115,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode); +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index a39e043c52a0016a6e7242572667a9d02d62be5e..f9447e016882f9c219a74ef96394d00535bc411f 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -45,7 +45,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) void syncEntryDestroy(SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) { +static FORCE_INLINE bool syncLogReplBarrier(SSyncRaftEntry* pEntry) { return pEntry->originalRpcType == TDMT_SYNC_NOOP; } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index bd1dae54d956a9a722d70eac30c596989d117cf3..b488455dba4cfe5e1c103cbf2319bd730e59e7b3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -105,6 +105,7 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SRpcMsg rpcRsp = {0}; bool accepted = false; SSyncRaftEntry* pEntry = NULL; + bool resetElect = false; // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { @@ -136,8 +137,10 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pReply->term = pMsg->term; } - syncNodeStepDown(ths, pMsg->term); - syncNodeResetElectTimer(ths); + if(ths->raftCfg.cfg.nodeInfo[ths->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + syncNodeStepDown(ths, pMsg->term); + resetElect = true; + } if (pMsg->dataLen < sizeof(SSyncRaftEntry)) { sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d", @@ -184,10 +187,9 @@ _SEND_RESPONSE: // commit index, i.e. leader notice me if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { sError("vgId:%d, failed to commit raft fsm log since %s.", ths->vgId, terrstr()); - goto _out; } -_out: + if (resetElect) syncNodeResetElectTimer(ths); return 0; _IGNORE: diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index a60f43cd5ed053fc00eba4b97c32e908698ea499..7c343c0e5d343e79d9f74e63b9548830e9b78a5f 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -85,7 +85,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr); return -1; } - (void)syncLogReplMgrProcessReply(pMgr, ths, pMsg); + (void)syncLogReplProcessReply(pMgr, ths, pMsg); } return 0; } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index e53b8ade1ca71281f4ec2536195456def7882781..86e28db90c93bf69a5ad40d0a6149bb7a7cad111 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -41,6 +41,8 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { int32_t ret = 0; for (int i = 0; i < pNode->peersNum; ++i) { + if(pNode->peersNodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; + SRpcMsg rpcMsg = {0}; ret = syncBuildRequestVote(&rpcMsg, pNode->vgId); if (ret < 0) { @@ -115,6 +117,5 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { ASSERT(ret == 0); syncNodeResetElectTimer(pSyncNode); - return ret; } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 7ecb9d77825b496ea290d4bbca6f3c873edc716a..e3c3f63a4f89b4b0021325f54a8b595e28f56970 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -26,6 +26,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); @@ -35,6 +36,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); } @@ -50,14 +52,14 @@ void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr) { memset(pIndexMgr->privateTerm, 0, sizeof(pIndexMgr->privateTerm)); int64_t timeNow = taosGetTimestampMs(); - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { pIndexMgr->startTimeArr[i] = 0; pIndexMgr->recvTimeArr[i] = timeNow; } } void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->index)[i] = index; return; @@ -69,7 +71,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, Sync } SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], pRaftId)) { return pNode->logReplMgrs[i]; } @@ -80,7 +82,7 @@ SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncIndex idx = (pIndexMgr->index)[i]; return idx; @@ -93,7 +95,7 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) } void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->startTimeArr)[i] = startTime; return; @@ -105,7 +107,7 @@ void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, } int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t startTime = (pIndexMgr->startTimeArr)[i]; return startTime; @@ -118,7 +120,7 @@ int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftI } void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->recvTimeArr)[i] = recvTime; return; @@ -130,7 +132,7 @@ void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, i } int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t recvTime = (pIndexMgr->recvTimeArr)[i]; return recvTime; @@ -143,7 +145,7 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId } void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->privateTerm)[i] = term; return; @@ -155,7 +157,7 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncT } SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncTerm term = (pIndexMgr->privateTerm)[i]; return term; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 04e4859c5e3578453431dffa635367117c372f06..cb1919d16ea1db667d841b58302363b17c6c05a0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -37,6 +37,7 @@ #include "syncVoteMgr.h" #include "tglobal.h" #include "tref.h" +#include "syncUtil.h" static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); @@ -140,6 +141,13 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return -1; + if(pSyncNode->raftCfg.lastConfigIndex >= pNewCfg->lastIndex){ + syncNodeRelease(pSyncNode); + sInfo("vgId:%d, no need Reconfig, current index:%" PRId64 ", new index:%" PRId64, pSyncNode->vgId, + pSyncNode->raftCfg.lastConfigIndex, pNewCfg->lastIndex); + return 0; + } + if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -148,12 +156,12 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); - syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + syncNodeDoConfigChange(pSyncNode, pNewCfg, pNewCfg->lastIndex); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeStopHeartbeatTimer(pSyncNode); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &pSyncNode->peerHeartbeatTimerArr[i], pSyncNode->replicasId[i]); } @@ -182,6 +190,9 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { case TDMT_SYNC_TIMEOUT: code = syncNodeOnTimeout(pSyncNode, pMsg); break; + case TDMT_SYNC_TIMEOUT_ELECTION: + code = syncNodeOnTimeout(pSyncNode, pMsg); + break; case TDMT_SYNC_CLIENT_REQUEST: code = syncNodeOnClientRequest(pSyncNode, pMsg, NULL); break; @@ -206,6 +217,9 @@ int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { case TDMT_SYNC_LOCAL_CMD: code = syncNodeOnLocalCmd(pSyncNode, pMsg); break; + case TDMT_SYNC_FORCE_FOLLOWER: + code = syncForceBecomeFollower(pSyncNode, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = -1; @@ -228,6 +242,20 @@ int32_t syncLeaderTransfer(int64_t rid) { return ret; } +int32_t syncForceBecomeFollower(SSyncNode* ths, const SRpcMsg* pRpcMsg) { + syncNodeBecomeFollower(ths, "force election"); + + SRpcMsg rsp = { + .code = 0, + .pCont = pRpcMsg->info.rsp, + .contLen = pRpcMsg->info.rspLen, + .info = pRpcMsg->info, + }; + tmsgSendRsp(&rsp); + + return 0; +} + int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) { SSyncNode* pNode = syncNodeAcquire(rid); if (pNode == NULL) return -1; @@ -294,8 +322,9 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { } } - if (pSyncNode->replicaNum > 1) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->totalReplicaNum > 1) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER + && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { sNTrace(pSyncNode, "new-snapshot-index:%" PRId64 " candidate or unknown state, do not delete wal", lastApplyIndex); syncNodeRelease(pSyncNode); @@ -516,7 +545,8 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { + if(pSyncNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; SEp* pEp = &pEpSet->eps[i]; tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort; @@ -543,6 +573,46 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { return ret; } +int32_t syncIsCatchUp(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + int32_t isCatchUp = 0; + if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || + pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || + pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ + sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 0; + } + else{ + sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 1; + } + + syncNodeRelease(pSyncNode); + return isCatchUp; +} + +ESyncRole syncGetRole(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + ESyncRole role = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole; + + syncNodeRelease(pSyncNode); + return role; +} + int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; @@ -596,7 +666,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + sWarn("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); } @@ -634,6 +704,8 @@ static int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { pData->logicClock = pSyncTimer->logicClock; pData->execTime = tsNow + pSyncTimer->timerMS; + sTrace("vgId:%d, start hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, pData->rid, pData->destId.addr); + taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS / HEARTBEAT_TICK_NUM, (void*)(pData->rid), syncEnv()->pTimerManager, &pSyncTimer->pTimer); } else { @@ -698,7 +770,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; - pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID; + pSyncNode->raftCfg.lastConfigIndex = pSyncInfo->syncCfg.lastIndex; pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize; pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; pSyncNode->raftCfg.configIndexCount = 1; @@ -715,7 +787,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { goto _error; } - if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { + if (pSyncInfo->syncCfg.totalReplicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId); pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; if (syncWriteCfgFile(pSyncNode) != 0) { @@ -732,8 +804,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->vgId = pSyncInfo->vgId; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; bool updated = false; - sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + sInfo("vgId:%d, start to open sync node, totalReplicaNum:%d replicaNum:%d selfIndex:%d", + pSyncNode->vgId, pCfg->totalReplicaNum, pCfg->replicaNum, pCfg->myIndex); + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -771,9 +844,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -789,7 +862,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; @@ -913,7 +987,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->heartbeatTimerCounter = 0; // init peer heartbeat timer - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &(pSyncNode->peerHeartbeatTimerArr[i]), (pSyncNode->replicasId)[i]); } @@ -928,7 +1002,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->restoreFinish = false; // snapshot senders - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); if (pSender == NULL) return NULL; @@ -946,7 +1020,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->changing = false; // replication mgr - if (syncNodeLogReplMgrInit(pSyncNode) < 0) { + if (syncNodeLogReplInit(pSyncNode) < 0) { sError("vgId:%d, failed to init repl mgr since %s.", pSyncNode->vgId, terrstr()); goto _error; } @@ -1038,14 +1112,19 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft - if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode); - syncNodeBecomeLeader(pSyncNode, "one replica start"); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER){ + syncNodeBecomeLearner(pSyncNode, "first start"); + } + else{ + if (pSyncNode->replicaNum == 1) { + raftStoreNextTerm(pSyncNode); + syncNodeBecomeLeader(pSyncNode, "one replica start"); - // Raft 3.6.2 Committing entries from previous terms - syncNodeAppendNoop(pSyncNode); - } else { - syncNodeBecomeFollower(pSyncNode, "first start"); + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + } else { + syncNodeBecomeFollower(pSyncNode, "first start"); + } } int32_t ret = 0; @@ -1082,21 +1161,15 @@ void syncNodePreClose(SSyncNode* pSyncNode) { ASSERT(pSyncNode->pFsm != NULL); ASSERT(pSyncNode->pFsm->FpApplyQueueItems != NULL); - while (1) { - int32_t aqItems = pSyncNode->pFsm->FpApplyQueueItems(pSyncNode->pFsm); - sTrace("vgId:%d, pre close, %d items in apply queue", pSyncNode->vgId, aqItems); - if (aqItems == 0 || aqItems == -1) { - break; - } - taosMsleep(20); - } - // stop elect timer syncNodeStopElectTimer(pSyncNode); // stop heartbeat timer syncNodeStopHeartbeatTimer(pSyncNode); + // stop ping timer + syncNodeStopPingTimer(pSyncNode); + // clean rsp syncRespCleanRsp(pSyncNode->pSyncRespMgr); } @@ -1120,11 +1193,12 @@ void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); + syncRespCleanRsp(pSyncNode->pSyncRespMgr); + syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - - syncNodeLogReplMgrDestroy(pSyncNode); + syncNodeLogReplDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -1141,7 +1215,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncLogBufferDestroy(pSyncNode->pLogBuf); pSyncNode->pLogBuf = NULL; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] != NULL) { sDebug("vgId:%d, snapshot sender destroy while close, data:%p", pSyncNode->vgId, pSyncNode->senders[i]); @@ -1334,7 +1408,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { bool b1 = false; bool b2 = false; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 && pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) { b1 = true; @@ -1342,7 +1416,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } } - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pCfg->nodeInfo[i]), .vgId = pNode->vgId, @@ -1359,13 +1433,14 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg) { - if (pOldCfg->replicaNum != pNewCfg->replicaNum) return true; + if (pOldCfg->totalReplicaNum != pNewCfg->totalReplicaNum) return true; if (pOldCfg->myIndex != pNewCfg->myIndex) return true; - for (int32_t i = 0; i < pOldCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pOldCfg->totalReplicaNum; ++i) { const SNodeInfo* pOldInfo = &pOldCfg->nodeInfo[i]; const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; if (pOldInfo->nodePort != pNewInfo->nodePort) return true; + if(pOldInfo->nodeRole != pNewInfo->nodeRole) return true; } return false; @@ -1402,8 +1477,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - sNInfo(pSyncNode, "begin do config change, from %d to %d, replicas:%d", pSyncNode->vgId, oldConfig.replicaNum, - pNewConfig->replicaNum); + sNInfo(pSyncNode, "begin do config change, from %d to %d, from %" PRId64 " to %" PRId64 ", replicas:%d", + pSyncNode->vgId, + oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum, + oldConfig.lastIndex, pNewConfig->lastIndex); if (IamInNew) { pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal @@ -1420,11 +1497,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde int32_t ret = 0; // save snapshot senders - int32_t oldReplicaNum = pSyncNode->replicaNum; - SRaftId oldReplicasId[TSDB_MAX_REPLICA]; + SRaftId oldReplicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); - SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { oldSenders[i] = pSyncNode->senders[i]; sSTrace(oldSenders[i], "snapshot sender save old"); } @@ -1434,9 +1510,9 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -1449,7 +1525,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } @@ -1464,15 +1541,15 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // reset snapshot senders // clear new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->senders[i] = NULL; } // reset new - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { // reset sender bool reset = false; - for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + for (int32_t j = 0; j < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) { sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p", (pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]); @@ -1494,7 +1571,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // create new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] == NULL) { pSyncNode->senders[i] = snapshotSenderCreate(pSyncNode, i); if (pSyncNode->senders[i] == NULL) { @@ -1509,7 +1586,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // free old - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (oldSenders[i] != NULL) { sSDebug(oldSenders[i], "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); @@ -1534,12 +1611,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } else { // persist cfg syncWriteCfgFile(pSyncNode); - sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } // raft state change -------------- @@ -1598,8 +1675,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; syncNodeStopHeartbeatTimer(pSyncNode); - // reset elect timer - syncNodeResetElectTimer(pSyncNode); + // trace log + sNTrace(pSyncNode, "become follower %s", debugStr); // send rsp to client syncNodeLeaderChangeRsp(pSyncNode); @@ -1615,8 +1692,29 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset log buffer syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode); + // reset elect timer + syncNodeResetElectTimer(pSyncNode); +} + +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { + pSyncNode->hbSlowNum = 0; + + // state change + pSyncNode->state = TAOS_SYNC_STATE_LEARNER; + // trace log - sNTrace(pSyncNode, "become follower %s", debugStr); + sNTrace(pSyncNode, "become learner %s", debugStr); + + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLearnerCb != NULL) { + pSyncNode->pFsm->FpBecomeLearnerCb(pSyncNode->pFsm); + } + + // min match index + pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; + + // reset log buffer + syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode); } // TLA+ Spec @@ -1736,7 +1834,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { bool syncNodeIsMnode(SSyncNode* pSyncNode) { return (pSyncNode->vgId == 1); } int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->peerStates[i].lastSendIndex = SYNC_INDEX_INVALID; pSyncNode->peerStates[i].lastSendTime = 0; } @@ -2023,7 +2121,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; SSyncNode* pNode = param; - if (pNode->replicaNum > 1) { + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pNode->heartbeatTimerLogicClock), @@ -2085,9 +2183,9 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); + sTrace("vgId:%d, eq peer hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, hbDataRid, pData->destId.addr); - if (pSyncNode->replicaNum > 1) { + if (pSyncNode->totalReplicaNum > 1) { int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); int64_t msgLogicClock = atomic_load_64(&pData->logicClock); @@ -2114,6 +2212,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { pSyncTimer->timeStamp = tsNow; // send msg + sTrace("vgId:%d, send heartbeat to dnode:%d", pSyncNode->vgId, DID(&(pSyncMsg->destId))); syncLogSendHeartbeat(pSyncNode, pSyncMsg, false, timerElapsed, pData->execTime); syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); } else { @@ -2165,8 +2264,8 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { // append to log buffer if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); - terrno = TSDB_CODE_SYN_BUFFER_FULL; - (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, TSDB_CODE_SYN_BUFFER_FULL); + ASSERT(terrno != 0); + (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); syncEntryDestroy(pEntry); return -1; } @@ -2196,7 +2295,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { } bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { - if (pSyncNode->replicaNum == 1) { + if (pSyncNode->totalReplicaNum == 1) { return false; } @@ -2221,7 +2320,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { bool syncNodeSnapshotSending(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return false; bool b = false; - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { if (pSyncNode->senders[i] != NULL && pSyncNode->senders[i]->start) { b = true; break; @@ -2282,6 +2381,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) { int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncHeartbeat* pMsg = pRpcMsg->pCont; + bool resetElect = false; const STraceId* trace = &pRpcMsg->info.traceId; char tbuf[40] = {0}; @@ -2291,6 +2391,14 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int64_t timeDiff = tsMs - pMsg->timeStamp; syncLogRecvHeartbeat(ths, pMsg, timeDiff, tbuf); + if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { + sWarn( + "vgId:%d, drop heartbeat msg from dnode:%d, because it come from another cluster:%d, differ from current " + "cluster:%d", + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), CID(&(ths->myRaftId))); + return 0; + } + SRpcMsg rpcMsg = {0}; (void)syncBuildHeartbeatReply(&rpcMsg, ths->vgId); SyncTerm currentTerm = raftStoreGetTerm(ths); @@ -2303,37 +2411,40 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; + sTrace( + "vgId:%d, heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); + resetElect = true; - syncNodeResetElectTimer(ths); ths->minMatchIndex = pMsg->minMatchIndex; - if (ths->state == TAOS_SYNC_STATE_FOLLOWER) { - // syncNodeFollowerCommit(ths, pMsg->commitIndex); + if (ths->state == TAOS_SYNC_STATE_FOLLOWER || ths->state == TAOS_SYNC_STATE_LEARNER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); SyncLocalCmd* pSyncMsg = rpcMsgLocalCmd.pCont; - pSyncMsg->cmd = SYNC_LOCAL_CMD_FOLLOWER_CMT; + pSyncMsg->cmd = + (ths->state == TAOS_SYNC_STATE_LEARNER) ? SYNC_LOCAL_CMD_LEARNER_CMT : SYNC_LOCAL_CMD_FOLLOWER_CMT; pSyncMsg->commitIndex = pMsg->commitIndex; pSyncMsg->currentTerm = pMsg->term; - SyncIndex fcIndex = pSyncMsg->commitIndex; if (ths->syncEqMsg != NULL && ths->msgcb != NULL) { int32_t code = ths->syncEqMsg(ths->msgcb, &rpcMsgLocalCmd); if (code != 0) { - sError("vgId:%d, sync enqueue fc-commit msg error, code:%d", ths->vgId, code); + sError("vgId:%d, failed to enqueue commit msg from heartbeat since %s, code:%d", ths->vgId, terrstr(), code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index:%" PRId64, ths->vgId, fcIndex); + sTrace("vgId:%d, enqueue commit msg from heartbeat, commit-index:%" PRId64 ", term:%" PRId64, ths->vgId, + pMsg->commitIndex, pMsg->term); } } } } - if (pMsg->term >= currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { - // syncNodeStepDown(ths, pMsg->term); + if (pMsg->term >= currentTerm && ths->state == TAOS_SYNC_STATE_LEADER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2348,20 +2459,15 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue step-down msg, new-term:%" PRId64, ths->vgId, pSyncMsg->currentTerm); + sTrace("vgId:%d, sync enqueue step-down msg, new-term:%" PRId64, ths->vgId, pMsg->term); } } } - /* - // htonl - SMsgHead* pHead = rpcMsg.pCont; - pHead->contLen = htonl(pHead->contLen); - pHead->vgId = htonl(pHead->vgId); - */ - // reply syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); + + if (resetElect) syncNodeResetElectTimer(ths); return 0; } @@ -2382,7 +2488,7 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); - return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg); + return syncLogReplProcessHeartbeatReply(pMgr, ths, pMsg); } int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { @@ -2408,7 +2514,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) { syncNodeStepDown(ths, pMsg->currentTerm); - } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) { + } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT || pMsg->cmd == SYNC_LOCAL_CMD_LEARNER_CMT) { if (syncLogBufferIsEmpty(ths->pLogBuf)) { sError("vgId:%d, sync log buffer is empty.", ths->vgId); return 0; @@ -2484,13 +2590,15 @@ const char* syncStr(ESyncState state) { return "error"; case TAOS_SYNC_STATE_OFFLINE: return "offline"; + case TAOS_SYNC_STATE_LEARNER: + return "learner"; default: return "unknown"; } } int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) { - for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pNewCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]), .vgId = ths->vgId, @@ -2510,7 +2618,7 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { } bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) { return true; } @@ -2520,7 +2628,7 @@ bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) { SSyncSnapshotSender* pSender = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pSender = (ths->senders)[i]; } @@ -2530,7 +2638,7 @@ SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SSyncTimer* pTimer = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pTimer = &((ths->peerHeartbeatTimerArr)[i]); } @@ -2540,7 +2648,7 @@ SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SPeerState* syncNodeGetPeerState(SSyncNode* ths, const SRaftId* pDestId) { SPeerState* pState = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pState = &((ths->peerStates)[i]); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 2a44588eefc2287bce3cfc310907c081f32338fb..72c8887803c778e0cd8a82b4b11273ad2b1df5f2 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -22,7 +22,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l SSyncNode* pNode) { int32_t bytes = sizeof(SyncTimeout); pMsg->pCont = rpcMallocCont(bytes); - pMsg->msgType = TDMT_SYNC_TIMEOUT; + pMsg->msgType = (timeoutType == SYNC_TIMEOUT_ELECTION) ? TDMT_SYNC_TIMEOUT_ELECTION : TDMT_SYNC_TIMEOUT; pMsg->contLen = bytes; if (pMsg->pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -31,7 +31,7 @@ int32_t syncBuildTimeout(SRpcMsg* pMsg, ESyncTimeoutType timeoutType, uint64_t l SyncTimeout* pTimeout = pMsg->pCont; pTimeout->bytes = bytes; - pTimeout->msgType = TDMT_SYNC_TIMEOUT; + pTimeout->msgType = pMsg->msgType; pTimeout->vgId = pNode->vgId; pTimeout->timeoutType = timeoutType; pTimeout->logicClock = logicClock; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index cc1a40a430f4698b18bed863fee5253030308b86..85c18220a6d9cb362bca2b45cad418b6db7502dd 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -48,7 +48,23 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt SyncIndex index = pEntry->index; if (index - pBuf->startIndex >= pBuf->size) { - sError("vgId:%d, failed to append due to sync log buffer full. index:%" PRId64 "", pNode->vgId, index); + terrno = TSDB_CODE_SYN_BUFFER_FULL; + sError("vgId:%d, failed to append since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); + goto _err; + } + + if (pNode->restoreFinish && index - pBuf->commitIndex >= TSDB_SYNC_NEGOTIATION_WIN) { + terrno = TSDB_CODE_SYN_NEGO_WIN_EXCEEDED; + sError("vgId:%d, failed to append since %s, index:%" PRId64 ", commit-index:%" PRId64, pNode->vgId, terrstr(), + index, pBuf->commitIndex); + goto _err; + } + + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); + if (pNode->restoreFinish && pBuf->commitIndex - appliedIndex >= TSDB_SYNC_APPLYQ_SIZE_LIMIT) { + terrno = TSDB_CODE_SYN_WRITE_STALL; + sError("vgId:%d, failed to append since %s. index:%" PRId64 ", commit-index:%" PRId64 ", applied-index:%" PRId64, + pNode->vgId, terrstr(), index, pBuf->commitIndex, appliedIndex); goto _err; } @@ -74,10 +90,11 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt _err: syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); + taosMsleep(1); return -1; } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncRaftEntry* pEntry = NULL; SyncIndex prevIndex = index - 1; @@ -233,6 +250,8 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { // update startIndex pBuf->startIndex = takeDummy ? index : index + 1; + pBuf->isCatchup = false; + sInfo("vgId:%d, init sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -307,7 +326,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm term = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index + 1); + SyncTerm term = syncLogReplGetPrevLogTerm(NULL, pNode, index + 1); ASSERT(pEntry->term >= 0); if (term == pEntry->term) { ret = 0; @@ -315,6 +334,15 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _out; } + if(pNode->raftCfg.cfg.nodeInfo[pNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER && + index > 0 && index > pBuf->totalIndex){ + pBuf->totalIndex = index; + sTrace("vgId:%d, update learner progress. index:%" PRId64 ", term:%" PRId64 ": prevterm:%" PRId64 + " != lastmatch:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, pEntry->index, pEntry->term, prevTerm, lastMatchTerm, pBuf->startIndex, pBuf->commitIndex, + pBuf->matchIndex, pBuf->endIndex); + } + if (index - pBuf->startIndex >= pBuf->size) { sWarn("vgId:%d, out of buffer range. index:%" PRId64 ", term:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -342,7 +370,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm existPrevTerm = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index); + SyncTerm existPrevTerm = syncLogReplGetPrevLogTerm(NULL, pNode, index); ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm)); ret = 0; goto _out; @@ -473,9 +501,9 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, int32_t applyCode) { - if ((pNode->replicaNum == 1) && pNode->restoreFinish && pNode->vgId != 1) { + if (pNode->replicaNum == 1 && pNode->restoreFinish && pNode->vgId != 1) { return 0; } @@ -555,7 +583,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { + if (syncFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role:%d, current term:%" PRId64, vgId, pEntry->index, pEntry->term, role, currentTerm); @@ -587,10 +615,10 @@ _out: // mark as restored if needed if (!pNode->restoreFinish && pBuf->commitIndex >= pNode->commitIndex && pEntry != NULL && currentTerm <= pEntry->term) { - pNode->pFsm->FpRestoreFinishCb(pNode->pFsm); + pNode->pFsm->FpRestoreFinishCb(pNode->pFsm, pBuf->commitIndex); pNode->restoreFinish = true; - sInfo("vgId:%d, restore finished. log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, - pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + sInfo("vgId:%d, restore finished. term:%" PRId64 ", log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, currentTerm, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } if (!inBuf) { @@ -602,7 +630,7 @@ _out: return ret; } -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplReset(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) return; ASSERT(pMgr->startIndex >= 0); @@ -616,22 +644,22 @@ void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->retryBackoff = 0; } -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->endIndex <= pMgr->startIndex) { return 0; } SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) { - syncLogReplMgrReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, + syncLogReplReset(pMgr); + sWarn("vgId:%d, reset sync log repl since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, pDestId->addr); return -1; } int32_t ret = -1; bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int64_t retryWaitMs = syncLogReplGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -648,16 +676,16 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->states[pos].acked) { if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { - syncLogReplMgrReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, - index, pDestId->addr); + syncLogReplReset(pMgr); + sWarn("vgId:%d, reset sync log repl since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index, + pDestId->addr); goto _out; } continue; } bool barrier = false; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -678,7 +706,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ret = 0; _out: if (retried) { - pMgr->retryBackoff = syncLogGetNextRetryBackoff(pMgr); + pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr); SSyncLogBuffer* pBuf = pNode->pLogBuf; sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64 ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 @@ -689,7 +717,7 @@ _out: return ret; } -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -699,7 +727,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p ASSERT(pMgr->matchIndex == 0); if (pMsg->matchIndex < 0) { pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -707,7 +735,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } } else { if (pMsg->lastSendIndex < pMgr->startIndex || pMsg->lastSendIndex >= pMgr->endIndex) { - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); return 0; } @@ -716,7 +744,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) { pMgr->matchIndex = pMsg->matchIndex; pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -741,7 +769,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p SyncIndex index = TMIN(pMsg->matchIndex, pNode->pLogBuf->matchIndex); if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { - term = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index + 1); + term = syncLogReplGetPrevLogTerm(pMgr, pNode, index + 1); if ((index + 1 < firstVer) || (term < 0) || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); @@ -764,53 +792,52 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } // attempt to replicate the raft log at index - (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbe(pMgr, pNode, index); + (void)syncLogReplReset(pMgr); + return syncLogReplProbe(pMgr, pNode, index); } -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", + sInfo("vgId:%d, reset sync log repl in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 - ", old:%" PRId64, + sInfo("vgId:%d, reset sync log repl in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64, pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); + (void)syncLogReplAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -820,12 +847,12 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) { return 0; } - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -848,7 +875,7 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy return 0; } -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; @@ -870,7 +897,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -893,7 +920,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) } } - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); SSyncLogBuffer* pBuf = pNode->pLogBuf; sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64 @@ -904,14 +931,14 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) return 0; } -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { - int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; - int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; - int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; + int64_t lastMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; + int64_t diffMs = lastMs - firstMs; + if (diffMs > 0 && diffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { pMgr->retryBackoff -= 1; } } @@ -923,10 +950,10 @@ int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNo pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttempt(pMgr, pNode); + return syncLogReplAttempt(pMgr, pNode); } -SSyncLogReplMgr* syncLogReplMgrCreate() { +SSyncLogReplMgr* syncLogReplCreate() { SSyncLogReplMgr* pMgr = taosMemoryCalloc(1, sizeof(SSyncLogReplMgr)); if (pMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -940,7 +967,7 @@ SSyncLogReplMgr* syncLogReplMgrCreate() { return pMgr; } -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { +void syncLogReplDestroy(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) { return; } @@ -948,10 +975,10 @@ void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { return; } -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { +int32_t syncNodeLogReplInit(SSyncNode* pNode) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { ASSERT(pNode->logReplMgrs[i] == NULL); - pNode->logReplMgrs[i] = syncLogReplMgrCreate(); + pNode->logReplMgrs[i] = syncLogReplCreate(); if (pNode->logReplMgrs[i] == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -961,9 +988,9 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { return 0; } -void syncNodeLogReplMgrDestroy(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - syncLogReplMgrDestroy(pNode->logReplMgrs[i]); +void syncNodeLogReplDestroy(SSyncNode* pNode) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { + syncLogReplDestroy(pNode->logReplMgrs[i]); pNode->logReplMgrs[i] = NULL; } } @@ -1092,9 +1119,9 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { pBuf->endIndex = pBuf->matchIndex + 1; // reset repl mgr - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); } syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); @@ -1118,8 +1145,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; @@ -1132,16 +1159,16 @@ int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { - sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, - pDestId->addr, terrstr(), index); - (void)syncLogReplMgrReset(pMgr); + sInfo("vgId:%d, reset sync log repl of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr, + terrstr(), index); + (void)syncLogReplReset(pMgr); } } goto _err; } - *pBarrier = syncLogIsReplicationBarrier(pEntry); + *pBarrier = syncLogReplBarrier(pEntry); - prevLogTerm = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index); + prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index); if (prevLogTerm < 0) { sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); goto _err; diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index f780e255ce77eb485ffcd1de8d078718dc1973a5..576b9d62f578e61a4f0c1292398fbfe53432eeea 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -18,21 +18,45 @@ #include "syncUtil.h" #include "tjson.h" +const char* syncRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "voter"; + case TAOS_SYNC_ROLE_LEARNER: + return "learner"; + default: + return "unknown"; + } +} + +const ESyncRole syncStrToRole(char* str) { + if(strcmp(str, "voter") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "learner") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { SSyncCfg *pCfg = (SSyncCfg *)pObj; + if (tjsonAddDoubleToObject(pJson, "totalReplicaNum", pCfg->totalReplicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1; SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); if (info == NULL) return -1; if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1; if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeRole", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; } @@ -90,7 +114,8 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { if (taosRenameFile(file, realfile) != 0) goto _OVER; code = 0; - sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len); + sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d, lastConfigIndex:%" PRId64, pNode->vgId, + realfile, len, pNode->raftCfg.lastConfigIndex); _OVER: if (pJson != NULL) tjsonDelete(pJson); @@ -108,6 +133,7 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SSyncCfg *pCfg = (SSyncCfg *)pObj; int32_t code = 0; + tjsonGetInt32ValueFromDouble(pJson, "totalReplicaNum", pCfg->totalReplicaNum, code); tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code); if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code); @@ -115,9 +141,9 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo"); if (nodeInfo == NULL) return -1; - pCfg->replicaNum = tjsonGetArraySize(nodeInfo); + pCfg->totalReplicaNum = tjsonGetArraySize(nodeInfo); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); if (info == NULL) return -1; tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code); @@ -126,6 +152,15 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); + char role[10] = {0}; + code = tjsonGetStringValue(info, "nodeRole", role); + if(code < 0) return -1; + if(strlen(role) != 0){ + pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); + } + else{ + pCfg->nodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + } } return 0; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 1d94b288d3a337d23efa5324a32df5f1ca564821..77e73402fae7f75b8b2240a58d2ed71f48b19e0e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -52,7 +52,7 @@ int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); taosThreadMutexUnlock(&pBuf->mutex); return 0; } @@ -66,15 +66,15 @@ int32_t syncNodeReplicate(SSyncNode* pNode) { } int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { - if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->replicaNum == 1) { + if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->raftCfg.cfg.totalReplicaNum == 1) { return -1; } - for (int32_t i = 0; i < pNode->replicaNum; i++) { + for (int32_t i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplMgrReplicateOnce(pMgr, pNode); + (void)syncLogReplDoOnce(pMgr, pNode); } return 0; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 2fda2a19b8dad08a49d35e8d0492f7e3259fbffe..77e5498127ecea7bcead68d83876b5334f551567 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -20,6 +20,7 @@ #include "syncRaftStore.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "syncUtil.h" // TLA+ Spec // HandleRequestVoteRequest(i, j, m) == @@ -89,6 +90,7 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { int32_t ret = 0; SyncRequestVote* pMsg = pRpcMsg->pCont; + bool resetElect = false; // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &pMsg->srcId)) { @@ -115,7 +117,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncNodeStepDown(ths, currentTerm); // forbid elect for this round - syncNodeResetElectTimer(ths); + resetElect = true; } // send msg @@ -134,5 +136,7 @@ int32_t syncNodeOnRequestVote(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncLogRecvRequestVote(ths, pMsg, pReply->voteGranted, ""); syncLogSendRequestVoteReply(ths, pReply, ""); syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + + if (resetElect) syncNodeResetElectTimer(ths); return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 413d6adf057ca5447ae31ef07398a9cf24d2a796..763d4ec5d6b0fa8015d841231fc094d96661fadc 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -795,14 +795,18 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } - if (pMsg->term > raftStoreGetTerm(pSyncNode)) { - syncNodeStepDown(pSyncNode, pMsg->term); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + if (pMsg->term > raftStoreGetTerm(pSyncNode)) { + syncNodeStepDown(pSyncNode, pMsg->term); + } + } + else{ + syncNodeUpdateTermWithoutStepDown(pSyncNode, pMsg->term); } - syncNodeResetElectTimer(pSyncNode); // state, term, seq/ack int32_t code = 0; - if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); @@ -840,6 +844,7 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { code = -1; } + syncNodeResetElectTimer(pSyncNode); return code; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 859183db9529ace6bf914fafbc381ca388880dea..5ee67da9ab8d2cef07b7d85ef275199e8cf2ef0e 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -58,7 +58,7 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { static int32_t syncNodeTimerRoutine(SSyncNode* ths) { ths->tmrRoutineNum++; - if (ths->tmrRoutineNum % 60 == 0 && ths->replicaNum > 1) { + if (ths->tmrRoutineNum % 60 == 0 && ths->totalReplicaNum > 1) { sNInfo(ths, "timer routines"); } else { sNTrace(ths, "timer routines"); @@ -120,9 +120,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) { if (atomic_load_64(&ths->pingTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->pingTimerCounter); - // syncNodePingAll(ths); - // syncNodePingPeers(ths); - syncNodeTimerRoutine(ths); } @@ -138,8 +135,6 @@ int32_t syncNodeOnTimeout(SSyncNode* ths, const SRpcMsg* pRpc) { ++(ths->heartbeatTimerCounter); sTrace("vgId:%d, sync timer, type:replicate count:%" PRIu64 ", lc-user:%" PRIu64, ths->vgId, ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser); - - // syncNodeReplicate(ths, true); } } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index a519c76cda53a5f66faeaefa17f56053c5d083e6..cf796c386297271d1138189ea812fe4b1fbe984c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -125,7 +125,7 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } -static void syncLogReplMgrStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { +static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int len = 0; len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { @@ -178,7 +178,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char replMgrStatesStr[1024] = ""; - syncLogReplMgrStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); + syncLogReplStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); char bufferStatesStr[256] = ""; syncLogBufferStates2Str(pNode, bufferStatesStr, sizeof(bufferStatesStr)); @@ -202,21 +202,22 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo // restore error code terrno = errCode; - + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); + if (pNode != NULL) { taosPrintLog(flags, level, dflag, - "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", first-ver:%" PRId64 - ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 + "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", applied-index:%" PRId64 + ", first-ver:%" PRId64 ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 ", elect-times:%d, as-leader-times:%d, cfg-ch-times:%d, hb-slow:%d, hbr-slow:%d, " "aq-items:%d, snaping:%" PRId64 ", replicas:%d, last-cfg:%" PRId64 ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s", - pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, logBeginIndex, - logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, - pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, - pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, - pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, - bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, appliedIndex, + logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + pNode->electNum, pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, + aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, + pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, + pNode->heartbeatTimerLogicClockUser, bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); } } diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 6bd9625276c898184341cec291abed6c304700fb..83b0dde8e0c77b5c199a72d4a28bda64108b0810 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -30,7 +30,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pNode) { return NULL; } - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -49,7 +49,7 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted) { } void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode) { - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -115,7 +115,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pNode) { return NULL; } - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; @@ -130,7 +130,7 @@ void votesRespondDestory(SVotesRespond *pVotesRespond) { } void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode) { - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 29b7fa740c85e9ea74def8e53d47edd69c76b743..5ea9be63dbf7bb9415668d232324ca1272b49db6 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -947,6 +947,12 @@ static int tdbPagerRestore(SPager *pPager, const char *jFileName) { return 0; } +static int32_t txnIdCompareDesc(const void *pLeft, const void *pRight) { + int64_t lhs = *(int64_t *)pLeft; + int64_t rhs = *(int64_t *)pRight; + return lhs > rhs ? -1 : 1; +} + int tdbPagerRestoreJournals(SPager *pPager) { tdbDirEntryPtr pDirEntry; tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); @@ -955,23 +961,33 @@ int tdbPagerRestoreJournals(SPager *pPager) { return -1; } + SArray *pTxnList = taosArrayInit(16, sizeof(int64_t)); + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { - char jname[TD_PATH_MAX] = {0}; - int dirLen = strlen(pPager->pEnv->dbName); - memcpy(jname, pPager->pEnv->dbName, dirLen); - jname[dirLen] = '/'; - memcpy(jname + dirLen + 1, name, strlen(name)); - if (tdbPagerRestore(pPager, jname) < 0) { - tdbCloseDir(&pDir); - - tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); - return -1; - } + int64_t txnId = -1; + sscanf(name, TDB_MAINDB_NAME "-journal.%" PRId64, &txnId); + taosArrayPush(pTxnList, &txnId); + } + } + taosArraySort(pTxnList, txnIdCompareDesc); + for (int i = 0; i < TARRAY_SIZE(pTxnList); ++i) { + int64_t *pTxnId = taosArrayGet(pTxnList, i); + char jname[TD_PATH_MAX] = {0}; + int dirLen = strlen(pPager->pEnv->dbName); + memcpy(jname, pPager->pEnv->dbName, dirLen); + jname[dirLen] = '/'; + sprintf(jname + dirLen + 1, TDB_MAINDB_NAME "-journal.%" PRId64, *pTxnId); + if (tdbPagerRestore(pPager, jname) < 0) { + tdbCloseDir(&pDir); + + tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), jname); + return -1; } } + taosArrayDestroy(pTxnList); tdbCloseDir(&pDir); return 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b6f995b11e1e485bd27d5692326ca5f58330cc51..c23d6d0a1f42fc46f3b52b7bba8c1ca21f8e4f39 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -425,21 +425,6 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - // if (TMSG_INFO(pHead->msgType - 1) != 0) { - // char buf[128] = {0}; - // sprintf(buf, "%s", TMSG_INFO(pHead->msgType - 1)); - // int* count = taosHashGet(pThrd->msgCount, TMSG_INFO(pHead->msgType - 1), strlen(TMSG_INFO(pHead->msgType - 1))); - // if (NULL == 0) { - // int localCount = 1; - // taosHashPut(pThrd->msgCount, TMSG_INFO(pHead->msgType - 1), strlen(TMSG_INFO(pHead->msgType - 1)), &localCount, - // sizeof(localCount)); - // } else { - // int localCount = *count - 1; - // taosHashPut(pThrd->msgCount, TMSG_INFO(pHead->msgType - 1), strlen(TMSG_INFO(pHead->msgType - 1)), &localCount, - // sizeof(localCount)); - // } - // } - STraceId* trace = &transMsg.info.traceId; tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, code str:%s", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, pHead->msgLen, tstrerror(transMsg.code)); @@ -477,6 +462,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); return; } @@ -536,6 +522,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { @@ -1118,19 +1105,6 @@ void cliSend(SCliConn* pConn) { msgLen = (int32_t)ntohl((uint32_t)(pHead->msgLen)); } - // if (tmsgIsValid(pHead->msgType)) { - // char buf[128] = {0}; - // sprintf(buf, "%s", TMSG_INFO(pHead->msgType)); - // int* count = taosHashGet(pThrd->msgCount, buf, sizeof(buf)); - // if (NULL == 0) { - // int localCount = 1; - // taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); - // } else { - // int localCount = *count + 1; - // taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); - // } - // } - tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, msgLen); @@ -1525,16 +1499,19 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { destroyCmsg(pMsg); return; } - if (tmsgIsValid(pMsg->msg.msgType)) { - char buf[128] = {0}; - sprintf(buf, "%s", TMSG_INFO(pMsg->msg.msgType)); - int* count = taosHashGet(pThrd->msgCount, buf, sizeof(buf)); - if (NULL == 0) { - int localCount = 1; - taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); - } else { - int localCount = *count + 1; - taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + + if (rpcDebugFlag & DEBUG_TRACE) { + if (tmsgIsValid(pMsg->msg.msgType)) { + char buf[128] = {0}; + sprintf(buf, "%s", TMSG_INFO(pMsg->msg.msgType)); + int* count = taosHashGet(pThrd->msgCount, buf, sizeof(buf)); + if (NULL == 0) { + int localCount = 1; + taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + } else { + int localCount = *count + 1; + taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + } } } @@ -1782,18 +1759,20 @@ static void cliAsyncCb(uv_async_t* handle) { QUEUE_MOVE(&item->qmsg, &wq); taosThreadMutexUnlock(&item->mtx); - void* pIter = taosHashIterate(pThrd->msgCount, NULL); - while (pIter != NULL) { - int* count = pIter; - size_t len = 0; - char* key = taosHashGetKey(pIter, &len); - if (*count != 0) { - tDebug("key: %s count: %d", key, *count); - } + if (rpcDebugFlag & DEBUG_TRACE) { + void* pIter = taosHashIterate(pThrd->msgCount, NULL); + while (pIter != NULL) { + int* count = pIter; + size_t len = 0; + char* key = taosHashGetKey(pIter, &len); + if (*count != 0) { + tDebug("key: %s count: %d", key, *count); + } - pIter = taosHashIterate(pThrd->msgCount, pIter); + pIter = taosHashIterate(pThrd->msgCount, pIter); + } + tDebug("all conn count: %d", pThrd->newConnCount); } - tDebug("all conn count: %d", pThrd->newConnCount); int8_t supportBatch = pTransInst->supportBatch; if (supportBatch == 0) { @@ -2379,17 +2358,18 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn); } } - - if (tmsgIsValid(pResp->msgType - 1)) { - char buf[128] = {0}; - sprintf(buf, "%s", TMSG_INFO(pResp->msgType - 1)); - int* count = taosHashGet(pThrd->msgCount, buf, sizeof(buf)); - if (NULL == 0) { - int localCount = 0; - taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); - } else { - int localCount = *count - 1; - taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + if (rpcDebugFlag & DEBUG_TRACE) { + if (tmsgIsValid(pResp->msgType - 1)) { + char buf[128] = {0}; + sprintf(buf, "%s", TMSG_INFO(pResp->msgType - 1)); + int* count = taosHashGet(pThrd->msgCount, buf, sizeof(buf)); + if (NULL == 0) { + int localCount = 0; + taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + } else { + int localCount = *count - 1; + taosHashPut(pThrd->msgCount, buf, sizeof(buf), &localCount, sizeof(localCount)); + } } } if (pCtx->pSem != NULL) { diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 702d05f57696c55e88b94d2d335674bd8f262c49..54e6abd85a50a5367bcac9c1c42d910df15d6b06 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -179,17 +179,23 @@ _err: int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { if (pWal == NULL) return TSDB_CODE_APP_ERROR; - if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, - pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); + if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod && + pWal->cfg.retentionPeriod == pCfg->retentionPeriod && pWal->cfg.retentionSize == pCfg->retentionSize) { + wDebug("vgId:%d, walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 " not change", + pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize); return 0; } - wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->cfg.vgId, pWal->cfg.level, - pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); + wInfo("vgId:%d, change old walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64 + ", new walLevel:%d fsync:%d walRetentionPeriod:%d walRetentionSize:%" PRId64, + pWal->cfg.vgId, pWal->cfg.level, pWal->cfg.fsyncPeriod, pWal->cfg.retentionPeriod, pWal->cfg.retentionSize, + pCfg->level, pCfg->fsyncPeriod, pCfg->retentionPeriod, pCfg->retentionSize); pWal->cfg.level = pCfg->level; pWal->cfg.fsyncPeriod = pCfg->fsyncPeriod; + pWal->cfg.retentionPeriod = pCfg->retentionPeriod; + pWal->cfg.retentionSize = pCfg->retentionSize; + pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 3e1e36ccc1e7ac862aaad36ebc4d4a4c0a6a2c62..19694a6126ee2dc758e49495827069814d9620dd 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -33,7 +33,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pLogFile = NULL; pReader->curVersion = -1; pReader->curFileFirstVer = -1; - pReader->curInvalid = 1; pReader->capacity = 0; if (cond) { pReader->cond = *cond; @@ -81,7 +80,6 @@ int32_t walNextValidMsg(SWalReader *pReader) { wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64 ", end index:%" PRId64, pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); - pReader->curStopped = 0; while (fetchVer <= endVer) { if (walFetchHeadNew(pReader, fetchVer) < 0) { return -1; @@ -99,10 +97,18 @@ int32_t walNextValidMsg(SWalReader *pReader) { fetchVer = pReader->curVersion; } } - pReader->curStopped = 1; return -1; } +int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } + +void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever) { + *sver = walGetFirstVer(pReader->pWal); + int64_t lastVer = walGetLastVer(pReader->pWal); + int64_t committedVer = walGetCommittedVer(pReader->pWal); + *ever = pReader->cond.scanUncommited ? lastVer : committedVer; +} + static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; @@ -196,32 +202,26 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { return -1; } - wDebug("vgId:%d, wal version reset from %" PRId64 "(invalid:%d) to %" PRId64, pReader->pWal->cfg.vgId, - pReader->curVersion, pReader->curInvalid, ver); + wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, + pReader->curVersion, ver); pReader->curVersion = ver; - pReader->curInvalid = 0; return 0; } int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { SWal *pWal = pReader->pWal; - if (!pReader->curInvalid && ver == pReader->curVersion) { + if (ver == pReader->curVersion) { wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); return 0; } -// pReader->curInvalid = 1; -// pReader->curVersion = ver; - if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { - wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, + wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, ver, pWal->vers.firstVer, pWal->vers.lastVer); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } -// if (ver < pWal->vers.snapshotVer) { -// } if (walReadSeekVerImpl(pReader, ver) < 0) { return -1; @@ -238,10 +238,8 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer); - if (pRead->curInvalid || pRead->curVersion != fetchVer) { + if (pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { -// pRead->curVersion = fetchVer; -// pRead->curInvalid = 1; return -1; } seeked = true; @@ -260,7 +258,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } -// pRead->curInvalid = 1; return -1; } } @@ -344,7 +341,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { return -1; } - if (pRead->curInvalid || pRead->curVersion != ver) { + if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); if (code < 0) { // pRead->curVersion = ver; @@ -479,7 +476,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { taosThreadMutexLock(&pReader->mutex); - if (pReader->curInvalid || pReader->curVersion != ver) { + if (pReader->curVersion != ver) { if (walReadSeekVer(pReader, ver) < 0) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); @@ -575,7 +572,6 @@ void walReadReset(SWalReader *pReader) { taosThreadMutexLock(&pReader->mutex); taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pLogFile); - pReader->curInvalid = 1; pReader->curFileFirstVer = -1; pReader->curVersion = -1; taosThreadMutexUnlock(&pReader->mutex); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 00ad7a3d67ea6118a0ed9d2fe8bd10b988bdd350..848de4f36da4db57e881df4977b4f2399314a68f 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -290,14 +290,22 @@ int32_t walEndSnapshot(SWal *pWal) { int ts = taosGetTimestampSec(); ver = TMAX(ver - pWal->vers.logRetention, pWal->vers.firstVer - 1); + + bool hasTopic = false; + int64_t refVer = ver; void *pIter = NULL; while (1) { pIter = taosHashIterate(pWal->pRefHash, pIter); if (pIter == NULL) break; SWalRef *pRef = *(SWalRef **)pIter; if (pRef->refVer == -1) continue; - ver = TMIN(ver, pRef->refVer - 1); + refVer = TMIN(refVer, pRef->refVer - 1); wDebug("vgId:%d, wal found ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId); + hasTopic = true; + } + // compatible mode + if (pWal->cfg.retentionPeriod == 0 && hasTopic) { + ver = TMIN(ver, refVer); } int deleteCnt = 0; @@ -327,7 +335,7 @@ int32_t walEndSnapshot(SWal *pWal) { wDebug("vgId:%d, wal check remove file %" PRId64 "(file size %" PRId64 " close ts %" PRId64 "), new tot size %" PRId64, pWal->cfg.vgId, iter->firstVer, iter->fileSize, iter->closeTs, newTotSize); - if (((pWal->cfg.retentionSize == 0) || (pWal->cfg.retentionSize != -1 && newTotSize > pWal->cfg.retentionSize)) || + if ((pWal->cfg.retentionSize != -1 && pWal->cfg.retentionSize != 0 && newTotSize > pWal->cfg.retentionSize) || ((pWal->cfg.retentionPeriod == 0) || (pWal->cfg.retentionPeriod != -1 && iter->closeTs != -1 && iter->closeTs + pWal->cfg.retentionPeriod < ts))) { // delete according to file size or close time diff --git a/source/os/src/osMath.c b/source/os/src/osMath.c index 1c7c825ce8317712583b51a6a2144099e0452f2b..0cff0f78a618b5d82f05b55d45e2cc6303a186fd 100644 --- a/source/os/src/osMath.c +++ b/source/os/src/osMath.c @@ -25,7 +25,7 @@ int32_t qsortHelper(const void* p1, const void* p2, const void* param) { // todo refactor: 1) move away; 2) use merge sort instead; 3) qsort is not a stable sort actually. void taosSort(void* base, int64_t sz, int64_t width, __compar_fn_t compar) { -#ifdef _ALPINE +#if defined(WINDOWS) || defined(_ALPINE) void* param = compar; taosqsort(base, sz, width, param, qsortHelper); #else diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 4998eb45a4527fd52d9559f5f99fe7f1e5362428..83c36a422d37b0af4b20a20992e79812c1544b02 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -25,7 +25,15 @@ void taosSeedRand(uint32_t seed) { return srand(seed); } -uint32_t taosRand(void) { return rand(); } +uint32_t taosRand(void) { +#ifdef WINDOWS + unsigned int pSeed; + rand_s(&pSeed); + return pSeed; +#else + return rand(); +#endif +} uint32_t taosRandR(uint32_t* pSeed) { #ifdef WINDOWS diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 52309a7b35cbdad87baf87ab10fae2bf42c1e335..84004ed3c18d1bb4baa218d450e7c825d25cd385 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,6 +17,10 @@ #include "os.h" #include "taoserror.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define PROCESS_ITEM 12 #define UUIDLEN37 37 @@ -252,7 +256,11 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS // ASSERT(0); #elif defined(_TD_DARWIN_64) +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; +#else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; @@ -264,8 +272,12 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { taosCloseFile(&pFile); return 0; +#else +#ifdef CUS_PROMPT + const char *filepath = "/usr/local/"CUS_PROMPT"/email"; #else const char *filepath = "/usr/local/taos/email"; +#endif // CUS_PROMPT TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); if (pFile == NULL) return false; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 685693a709f4dd14efc3ffd5f277946a3293b38a..39d1de043713b83301c8f15a4030345d00536658 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -37,6 +37,9 @@ // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) // until 00:00:00 January 1, 1970 static const uint64_t TIMEEPOCH = ((uint64_t)116444736000000000ULL); +// This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) +// until 00:00:00 January 1, 1900 +static const uint64_t TIMEEPOCH1900 = ((uint64_t)116445024000000000ULL); /* * We do not implement alternate representations. However, we always @@ -407,29 +410,39 @@ time_t taosMktime(struct tm *timep) { #endif } -struct tm *taosLocalTime(const time_t *timep, struct tm *result) { +struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { + struct tm *res = NULL; + if (timep == NULL) { + return NULL; + } if (result == NULL) { - return localtime(timep); + res = localtime(timep); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } + return res; } #ifdef WINDOWS if (*timep < 0) { - SYSTEMTIME ss, s; - FILETIME ff, f; + if (*timep < -2208988800LL) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + + SYSTEMTIME s; + FILETIME f; LARGE_INTEGER offset; struct tm tm1; time_t tt = 0; - localtime_s(&tm1, &tt); - ss.wYear = tm1.tm_year + 1900; - ss.wMonth = tm1.tm_mon + 1; - ss.wDay = tm1.tm_mday; - ss.wHour = tm1.tm_hour; - ss.wMinute = tm1.tm_min; - ss.wSecond = tm1.tm_sec; - ss.wMilliseconds = 0; - SystemTimeToFileTime(&ss, &ff); - offset.QuadPart = ff.dwHighDateTime; - offset.QuadPart <<= 32; - offset.QuadPart |= ff.dwLowDateTime; + if (localtime_s(&tm1, &tt) != 0 ) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + offset.QuadPart = TIMEEPOCH1900; offset.QuadPart += *timep * 10000000; f.dwLowDateTime = offset.QuadPart & 0xffffffff; f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; @@ -444,10 +457,18 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { result->tm_yday = 0; result->tm_isdst = 0; } else { - localtime_s(result, timep); + if (localtime_s(result, timep) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } } #else - localtime_r(timep, result); + res = localtime_r(timep, result); + if (res == NULL && buf != NULL) { + sprintf(buf, "NaN"); + } #endif return result; } @@ -469,12 +490,16 @@ struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) } #ifdef WINDOWS if (*timep < 0) { + return NULL; + // TODO: bugs in following code SYSTEMTIME ss, s; FILETIME ff, f; LARGE_INTEGER offset; struct tm tm1; time_t tt = 0; - localtime_s(&tm1, &tt); + if (localtime_s(&tm1, &tt) != 0) { + return NULL; + } ss.wYear = tm1.tm_year + 1900; ss.wMonth = tm1.tm_mon + 1; ss.wDay = tm1.tm_mday; @@ -500,7 +525,9 @@ struct tm *taosLocalTimeNolock(struct tm *result, const time_t *timep, int dst) result->tm_yday = 0; result->tm_isdst = 0; } else { - localtime_s(result, timep); + if (localtime_s(result, timep) != 0) { + return NULL; + } } #elif defined(LINUX) time_t secsMin = 60, secsHour = 3600, secsDay = 3600 * 24; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index ad223bff27de80a454d09215926dcdf44e95c23c..cd6ad7cdb5b45c35bae19093a2a97b0c0d8e9bdf 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -893,7 +893,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); daylight = tm1.tm_isdst; /* @@ -921,7 +921,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); /* load time zone string from /etc/timezone */ // FILE *f = fopen("/etc/timezone", "r"); errno = 0; @@ -1008,7 +1008,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { */ time_t tx1 = taosGetTimestampSec(); struct tm tm1; - taosLocalTime(&tx1, &tm1); + taosLocalTime(&tx1, &tm1, NULL); /* * format example: diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 366c3fc720f7c1543d1d11bdd69f1d2ae085eb38..5dd15db4ab929edb0a200e28b2fe1b2d230642ca 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -46,3 +46,80 @@ TEST(osTimeTests, taosLocalTimeNolock) { EXPECT_EQ(expectedTime.tm_yday, result->tm_yday); EXPECT_EQ(expectedTime.tm_isdst, result->tm_isdst); } + + +TEST(osTimeTests, taosLocalTime) { + // Test 1: Test when both timep and result are not NULL + time_t timep = 1617531000; // 2021-04-04 18:10:00 + struct tm result; + struct tm* local_time = taosLocalTime(&timep, &result, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 121); + ASSERT_EQ(local_time->tm_mon, 3); + ASSERT_EQ(local_time->tm_mday, 4); + ASSERT_EQ(local_time->tm_hour, 18); + ASSERT_EQ(local_time->tm_min, 10); + ASSERT_EQ(local_time->tm_sec, 00); + + // Test 2: Test when timep is NULL + local_time = taosLocalTime(NULL, &result, NULL); + ASSERT_EQ(local_time, nullptr); + + // Test 3: Test when result is NULL + local_time = taosLocalTime(&timep, NULL, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 121); + ASSERT_EQ(local_time->tm_mon, 3); + ASSERT_EQ(local_time->tm_mday, 4); + + // Test 4: Test when timep is negative on Windows +#ifdef WINDOWS + time_t pos_timep = 1609459200; // 2021-01-01 08:00:00 + local_time = taosLocalTime(&pos_timep, &result, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 121); + ASSERT_EQ(local_time->tm_mon, 0); + ASSERT_EQ(local_time->tm_mday, 1); + ASSERT_EQ(local_time->tm_hour, 8); + ASSERT_EQ(local_time->tm_min, 0); + ASSERT_EQ(local_time->tm_sec, 0); + + time_t neg_timep = -1617531000; // 1918-09-29 21:50:00 + local_time = taosLocalTime(&neg_timep, &result, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 18); + ASSERT_EQ(local_time->tm_mon, 8); + ASSERT_EQ(local_time->tm_mday, 29); + ASSERT_EQ(local_time->tm_hour, 21); + ASSERT_EQ(local_time->tm_min, 50); + ASSERT_EQ(local_time->tm_sec, 0); + + time_t neg_timep2 = -315619200; // 1960-01-01 08:00:00 + local_time = taosLocalTime(&neg_timep2, &result, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 60); + ASSERT_EQ(local_time->tm_mon, 0); + ASSERT_EQ(local_time->tm_mday, 1); + ASSERT_EQ(local_time->tm_hour, 8); + ASSERT_EQ(local_time->tm_min, 0); + ASSERT_EQ(local_time->tm_sec, 0); + + time_t zero_timep = 0; // 1970-01-01 08:00:00 + local_time = taosLocalTime(&zero_timep, &result, NULL); + ASSERT_NE(local_time, nullptr); + ASSERT_EQ(local_time->tm_year, 70); + ASSERT_EQ(local_time->tm_mon, 0); + ASSERT_EQ(local_time->tm_mday, 1); + ASSERT_EQ(local_time->tm_hour, 8); + ASSERT_EQ(local_time->tm_min, 0); + ASSERT_EQ(local_time->tm_sec, 0); + + time_t over_timep = 6406301441633558; + local_time = taosLocalTime(&over_timep, &result, NULL); + ASSERT_EQ(local_time, nullptr); + + time_t neg_timep3 = -78115158887; + local_time = taosLocalTime(&neg_timep3, &result, NULL); + ASSERT_EQ(local_time, nullptr); +#endif +} \ No newline at end of file diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 7467fa29481d932632b612fc44550fd024542fc3..b15bb519e7344a9ccc6469b8f06570fba242af5a 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -140,46 +140,7 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp) taosArraySet(pArray, pos + 1, p2); memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize); - pos += 1; - } else { pos += 1; - } - } - } - - if (fp != NULL) { - for (int32_t i = pos + 1; i < pArray->size; ++i) { - void* p = taosArrayGet(pArray, i); - fp(p); - } - } - - pArray->size = pos + 1; -} - -void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) { - size_t size = pArray->size; - if (size <= 1) { - return; - } - - int32_t pos = 0; - for (int32_t i = 1; i < size; ++i) { - char* p1 = taosArrayGet(pArray, pos); - char* p2 = taosArrayGet(pArray, i); - - if (comparFn(p1, p2) == 0) { - // do nothing - } else { - if (pos + 1 != i) { - void* p = taosArrayGetP(pArray, pos + 1); - if (fp != NULL) { - fp(p); - } - - taosArraySet(pArray, pos + 1, p2); - memset(TARRAY_GET_ELEM(pArray, i), 0, pArray->elemSize); - pos += 1; } else { pos += 1; } @@ -188,7 +149,7 @@ void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp if (fp != NULL) { for (int32_t i = pos + 1; i < pArray->size; ++i) { - void* p = taosArrayGetP(pArray, i); + void* p = taosArrayGet(pArray, i); fp(p); } } @@ -392,20 +353,6 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) { pArray->size = 0; } -void taosArrayClearP(SArray* pArray, FDelete fp) { - if (pArray == NULL) return; - if (fp == NULL) { - pArray->size = 0; - return; - } - - for (int32_t i = 0; i < pArray->size; ++i) { - fp(*(void**)TARRAY_GET_ELEM(pArray, i)); - } - - pArray->size = 0; -} - void* taosArrayDestroy(SArray* pArray) { if (pArray) { taosMemoryFree(pArray->pData); @@ -495,6 +442,7 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void if (pArray->size <= 1) { return; } + for (int32_t i = 1; i <= pArray->size - 1; ++i) { for (int32_t j = i; j > 0; --j) { if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { @@ -507,7 +455,6 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } } } - return; } int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode) { @@ -556,4 +503,4 @@ void taosArraySwap(SArray* a, SArray* b) { void* data = a->pData; a->pData = b->pData; b->pData = data; -} +} \ No newline at end of file diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index c15bd96903c901ea9f0c7dd6aae317926c2b8ac9..288ea6052b580909198788f528871633048e19f1 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -187,7 +187,7 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int64_t ival = (int64_t)atoi(value); + int64_t ival = (int64_t)atoll(value); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); @@ -637,6 +637,8 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { int32_t code = 0; char **pEnv = environ; line[1023] = 0; + + if (pEnv == NULL) return 0; while (*pEnv != NULL) { name = value = value2 = value3 = NULL; olen = vlen = vlen2 = vlen3 = 0; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 33b562c8dde1e8251efe89b65579ffe65d8ea1cf..dbdb40b93d39851c422e9fe2f4f1246d183ad25b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -184,11 +184,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_ACCTS, "Too many accounts") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_NOT_EXIST, "Invalid user") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, "Invalid user format") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_NOT_AVAILABLE, "User not available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password format") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_PRIVILEDGE_EXIST, "User already have this priviledge") //mnode-stable-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") @@ -211,8 +213,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "Tag index already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "Tag index not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "index not exist") // mnode-db @@ -223,6 +225,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database in dropping status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO, "WAL retention period is zero") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") @@ -301,9 +304,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STREAMS, "Too many streams") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TARGET_TABLE, "Cannot write the same stable as other stream") // mnode-sma -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "index not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma index option") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") @@ -318,7 +321,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") @@ -411,7 +414,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limite // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_IS_LEADER, "Sync is leader") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync not leader") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync leader is unreachable") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_ONE_REPLICA, "Sync one replica") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new config") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") @@ -419,9 +422,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for re TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync is restoring") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_WRITE_STALL, "Sync write stall") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEGO_WIN_EXCEEDED, "Sync negotiation win exceeded") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq @@ -514,7 +519,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_ONLY_ONE_JSON_TAG, "Only one tag if ther TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_NUM_OF_COL, "Query block has incorrect number of result columns") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL, "Incorrect TIMESTAMP value") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_OFFSET_LESS_ZERO, "soffset/offset can not be less than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY, "slimit/soffset only available for PARTITION BY query") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_GROUP_BY, "slimit/soffset only available for PARTITION/GROUP BY query") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TOPIC_QUERY, "Invalid topic query") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DROP_STABLE, "Cannot drop super table in batch") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE, "Start(end) time of query range required or time range too large") @@ -576,13 +581,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NOT_EXIST, "udf pipe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED, "udf program language not supported") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_FUNC_EXEC_FAILURE, "udf function execution failure") //schemaless TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") @@ -626,6 +631,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is inval TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 89dd51a8924b3a24e70a665a3ae76e14541f6081..a3d3c399abcd6040cf042dec045024549e2d3179 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -21,7 +21,7 @@ #include "tjson.h" #include "tglobal.h" -#define LOG_MAX_LINE_SIZE (1024) +#define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) #define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) @@ -121,7 +121,7 @@ static FORCE_INLINE void taosUpdateDaylight() { struct timeval timeSecs; taosGetTimeOfDay(&timeSecs); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm); + ptm = taosLocalTime(&curTime, &Tm, NULL); tsDaylightActive = ptm->tm_isdst; } static FORCE_INLINE int32_t taosGetDaylight() { return tsDaylightActive; } @@ -437,7 +437,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { taosGetTimeOfDay(&timeSecs); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTime(&curTime, &Tm); + ptm = taosLocalTime(&curTime, &Tm, NULL); return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), flags); diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index 264883be4e71c99f1f3f3a1fbc872bf0c522728e..f4172fbb4427463f8a6cf8f937eac1df7eae2ce6 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -580,6 +580,16 @@ static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) { return usage; } +static int32_t taosLRUCacheShardGetElems(SLRUCacheShard *shard) { + int32_t elems = 0; + + taosThreadMutexLock(&shard->mutex); + elems = shard->table.elems; + taosThreadMutexUnlock(&shard->mutex); + + return elems; +} + static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) { size_t usage = 0; @@ -755,6 +765,16 @@ size_t taosLRUCacheGetUsage(SLRUCache *cache) { return usage; } +int32_t taosLRUCacheGetElems(SLRUCache *cache) { + int32_t elems = 0; + + for (int i = 0; i < cache->numShards; ++i) { + elems += taosLRUCacheShardGetElems(&cache->shards[i]); + } + + return elems; +} + size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache) { size_t usage = 0; diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index 70acffed5d62fb0b13fd02ff719b0aab577ae76c..ec1991923fa5caf6f05fe441c386c015b3b22972 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -28,19 +28,23 @@ #define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_n, fp) \ + do { \ + if (fp) { \ + fp((_n)->data); \ + } \ + taosMemoryFreeClear(_n); \ } while (0); struct SSHashObj { - SHNode **hashList; - size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - SArray* pHashNodeBuf;// hash node allocation buffer, 1k size of each page by default - int32_t offset; // allocation offset in current page + SHNode **hashList; + size_t capacity; // number of slots + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + _hash_free_fn_t freeFp; // free function + SArray *pHashNodeBuf; // hash node allocation buffer, 1k size of each page by default + int32_t offset; // allocation offset in current page }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -71,7 +75,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn) { pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->pHashNodeBuf = taosArrayInit(10, sizeof(void*)); + pHashObj->freeFp = NULL; pHashObj->offset = 0; pHashObj->size = 0; @@ -92,6 +96,10 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { return (int32_t) pHashObj->size; } +void tSimpleHashSetFreeFp(SSHashObj* pHashObj, _hash_free_fn_t freeFp) { + pHashObj->freeFp = freeFp; +} + static void* doInternalAlloc(SSHashObj* pHashObj, int32_t size) { #if 0 void** p = taosArrayGetLast(pHashObj->pHashNodeBuf); @@ -306,7 +314,8 @@ int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { } else { pPrev->next = pNode->next; } - FREE_HASH_NODE(pNode); + + FREE_HASH_NODE(pNode, pHashObj->freeFp); pHashObj->size -= 1; code = TSDB_CODE_SUCCESS; break; @@ -341,7 +350,7 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke *pIter = pPrev ? GET_SHASH_NODE_DATA(pPrev) : NULL; } - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode, pHashObj->freeFp); pHashObj->size -= 1; break; } @@ -370,14 +379,13 @@ void tSimpleHashClear(SSHashObj *pHashObj) { while (pNode) { pNext = pNode->next; - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pNode, pHashObj->freeFp); pNode = pNext; } pHashObj->hashList[i] = NULL; } - taosArrayClearEx(pHashObj->pHashNodeBuf, destroyItems); pHashObj->offset = 0; pHashObj->size = 0; } @@ -388,7 +396,6 @@ void tSimpleHashCleanup(SSHashObj *pHashObj) { } tSimpleHashClear(pHashObj); - taosArrayDestroy(pHashObj->pHashNodeBuf); taosMemoryFreeClear(pHashObj->hashList); taosMemoryFree(pHashObj); } diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index dc357c61a10df4294f2bc78bcebe514a11aa0879..0f84f6bafa99464f98fea3e064ac9fb94076ffc0 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -26,8 +26,9 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { int32_t vnum[4] = {0}; int32_t len = strlen(vstr); char tmp[16] = {0}; + int32_t vpos = 0; - for (int32_t spos = 0, tpos = 0, vpos = 0; spos < len && vpos < 4; ++spos) { + for (int32_t spos = 0, tpos = 0; spos < len && vpos < 4; ++spos) { if (vstr[spos] != '.') { tmp[spos - tpos] = vstr[spos]; } else { @@ -38,6 +39,10 @@ int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { } } + if ('\0' != tmp[0] && vpos < 4) { + vnum[vpos] = atoi(tmp); + } + if (vnum[0] <= 0) { terrno = TSDB_CODE_INVALID_VERSION_STRING; return -1; @@ -66,16 +71,16 @@ int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t case 4: break; case 3: - clientVer %= 100; - serverVer %= 100; + clientVer /= 100; + serverVer /= 100; break; case 2: - clientVer %= 10000; - serverVer %= 10000; + clientVer /= 10000; + serverVer /= 10000; break; case 1: - clientVer %= 1000000; - serverVer %= 1000000; + clientVer /= 1000000; + serverVer /= 1000000; break; default: terrno = TSDB_CODE_INVALID_VERSION_NUMBER; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 631bcb443ebecdc89ab3338d5dd5254964232093..6edee27c052d474355812a4b46d53331a5ab6324 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -217,7 +217,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem int32_t queueNum = taosGetQueueNumber(pool->qset); int32_t curWorkerNum = taosArrayGetSize(pool->workers); - int32_t dstWorkerNum = ceil(queueNum * pool->ratio); + int32_t dstWorkerNum = ceilf(queueNum * pool->ratio); if (dstWorkerNum < 1) dstWorkerNum = 1; // spawn a thread to process queue @@ -248,7 +248,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem } taosThreadAttrDestroy(&thAttr); - uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers)); + int32_t numOfThreads = taosArrayGetSize(pool->workers); + uInfo("worker:%s:%d is launched, total:%d, expect:%d", pool->name, worker->id, numOfThreads, dstWorkerNum); curWorkerNum++; } diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index cb307b57fce37ba4243aea83995e66612f3c4371..71998e3321e79c08e0885ac09463c0ef05b4a6fa 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -1,6 +1,7 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; +char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; char buildinfo[64] = "Built at ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py index 9da678bf818d6e6140e1e821e1a49410ae563eb7..c4426010e439fcb24774c0322e0e928f0c83a572 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py @@ -31,6 +31,8 @@ class TDTestCase: tdSql.init(conn.cursor()) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py index ba296584ba6e81e2f8db6860240c6535cfc62cbf..b353566b256c796f7c60af8a161fc767099391a4 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py @@ -33,6 +33,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py index 0fb39c210c96d9a5db7b2f4216fe023499002f01..d1a807c201217cd04a9ec348411daa2fab7266f6 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 7599c82483509807d222d73557d3164c35159403..35ed2982130bb7cb3cef31414a9225b3411aeb7c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -29,25 +29,23 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] - elif "src" in selfPath: - projPath = selfPath[: selfPath.find("src")] - elif "/tools/" in selfPath: - projPath = selfPath[: selfPath.find("/tools/")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - projPath = selfPath[: selfPath.find("tests")] + projPath = selfPath[:selfPath.find("tests")] paths = [] for root, dirs, files in os.walk(projPath): - if (tool) in files: + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: + if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break - if len(paths) == 0: + if (len(paths) == 0): tdLog.exit("taosBenchmark not found!") return else: @@ -61,9 +59,9 @@ class TDTestCase: os.system("%s" % cmd) tdSql.execute("reset query cache") tdSql.query("show db.tables") - tdSql.checkRows(10) + tdSql.checkRows(8) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 100) + tdSql.checkData(0, 0, 80) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 25a8d92f7e9b45e55011d5fe0bb241bb68962340..3f9c2160de24bfc7dde983178bbcef31c1e0c45f 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -33,6 +33,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if "community" in selfPath: diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py index 54657995e2b0abcfa80b1b08be0b40b9bf415126..cf76e362e5b8174fccff02615efc8778496e70df 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index d706eea068221602e057baa236b1dd671ddd1674..1a7f422123f826bbc4f22426de8c89732215a69e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index da22ef75e2a648e897341338bb762e4fedc6662b..c841e90b5168afc11bcf8248a0f37c9ba16d30d0 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -15,13 +15,18 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db" + "name": "db", + "drop": "yes" + }, "super_tables": [{ + "child_table_exists":"no", "name": "stb", "childtable_prefix": "stb_", "childtable_count": 10, "insert_rows": 10, + "childtable_from": 1, + "childtable_to": 9, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json index 9b99521f52794b94558daef935a8aefbe57b5c6b..078688bb7e075753bf972d4275daa21369ae10fd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb3-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json index 7b87919a6dbebd5f4abc05bb1139f309f23fed6f..e808c9d3f3901b2ccaf63b315de164bb11a729cb 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb4-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json index 9e1241397fe13eced0000c600ad76971a30227ca..e609fcfebdf3cfb30a88b4e86d59f6b38ce4535b 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_telnet_tcp.json @@ -16,7 +16,7 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db", + "name": "opentsdb_telnet", "drop": "yes" }, "super_tables": [{ diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json index baf0384e46d51f802f7dd842b1b49925efd00abc..0054d985ee6e68012d7840b1cb088d5d73147bec 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb2-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json index f683cc016b1e068c6ebf2933084977ffeb11966c..bed3598bfe8a32319968388b435ed0cd1a8a415a 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -55,7 +55,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb1-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py index afefabef66ef8b242a98874d5c15442e173f2df9..ea1bc165202c53fa7a714a1b249dfd15a786310f 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py index d95e6741f8af4ee837fe903dbc60fee894bebd7f..a1a77ba0bbf0e2fccfca222b20b30a619627e1bc 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py index c906f6167526c707f1a065b32dfd65fe5bdc7eac..f300e6520341e5c36ff216f1378f106a9699fce1 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py @@ -35,6 +35,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py index fd79f1e01b4ad28f8dfb75a13d114113ab1604ea..5d4dd794887efd55b5c81b2f6f89a7403c3390ea 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py index 200a3c0e69bc5b2e6c3b3dd6926885c5ca8d6f8f..3e40dd6a5e69548d20178d12aa7bd0a73ef6f1c2 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py index 0d10741331578f678198ad8dac597fe56a6c0709..ff82713ca3d4edf417c45d99f96393fe2cf64de5 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py @@ -29,28 +29,23 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] - elif "src" in selfPath: - projPath = selfPath[: selfPath.find("src")] - elif "/tools/" in selfPath: - projPath = selfPath[: selfPath.find("/tools/")] - elif "/tests/" in selfPath: - projPath = selfPath[: selfPath.find("/tests/")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath)) - projPath = "/usr/local/taos/bin/" + projPath = selfPath[:selfPath.find("tests")] paths = [] - for root, dummy, files in os.walk(projPath): - if (tool) in files: + for root, dirs, files in os.walk(projPath): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: + if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break - if len(paths) == 0: + if (len(paths) == 0): tdLog.exit("taosBenchmark not found!") return else: diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py index 241c6d63593d9cc5f203e88d7515d1085e446122..e8fa4f81fe234c6415eed425d4a91d90614cb635 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py @@ -30,6 +30,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py index 4d0adde1922bb62c8987133f14d0b7647c12b504..488bb6f81e7f48a7ce5924be1b0d948eddfd7da8 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py @@ -31,6 +31,8 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py b/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py index 19500c7dca946f3c0a1887eaa3d4bdcfa35390ef..91ebf1b1df1f5d60068a354f5c4fc10deff70b99 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py @@ -32,30 +32,28 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] - elif "src" in selfPath: - projPath = selfPath[: selfPath.find("src")] - elif "/tools/" in selfPath: - projPath = selfPath[: selfPath.find("/tools/")] - elif "/tests/" in selfPath: - projPath = selfPath[: selfPath.find("/tests/")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath)) - projPath = "/usr/local/taos/bin/" + projPath = selfPath[:selfPath.find("tests")] paths = [] for root, dirs, files in os.walk(projPath): - if (tool) in files: + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: + if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break - if len(paths) == 0: - return "" - return paths[0] + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] # 获取taosc接口查询的结果文件中的内容,返回每行数据,并断言数据的第一列内容。 def assertfileDataTaosc(self, filename, expectResult): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py index 23e788916f63d899d3ae3059a8062d54dcf329c6..0392b58aa2df54ebc75dc8647fed7e9bc51dd3f2 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py @@ -29,27 +29,23 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def getPath(self, tool="taosBenchmark"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] - elif "src" in selfPath: - projPath = selfPath[: selfPath.find("src")] - elif "/tools/" in selfPath: - projPath = selfPath[: selfPath.find("/tools/")] - elif "/tests/" in selfPath: - projPath = selfPath[: selfPath.find("/tests/")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - tdLog.exit("cannot found %s in path: %s, use system's" % (tool, selfPath)) + projPath = selfPath[:selfPath.find("tests")] paths = [] for root, dirs, files in os.walk(projPath): - if (tool) in files: + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: + if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break - if len(paths) == 0: + if (len(paths) == 0): tdLog.exit("taosBenchmark not found!") return else: diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py index 1ccbb1f7d659d4ce64ba83475997539d3f9d5845..197f05aac9889181dc5d988bfb27137fed53ea0d 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -30,30 +30,28 @@ class TDTestCase: self.tmpdir = "tmp" def getPath(self, tool="taosdump"): + if (platform.system().lower() == 'windows'): + tool = tool + ".exe" selfPath = os.path.dirname(os.path.realpath(__file__)) - if "community" in selfPath: - projPath = selfPath[: selfPath.find("community")] - elif "src" in selfPath: - projPath = selfPath[: selfPath.find("src")] - elif "/tools/" in selfPath: - projPath = selfPath[: selfPath.find("/tools/")] - elif "/tests/" in selfPath: - projPath = selfPath[: selfPath.find("/tests/")] + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] else: - tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath)) - projPath = "/usr/local/taos/bin" + projPath = selfPath[:selfPath.find("tests")] paths = [] - for root, dummy, files in os.walk(projPath): - if (tool) in files: + for root, dirs, files in os.walk(projPath): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) - if "packaging" not in rootRealPath: + if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break - if len(paths) == 0: - return "" - return paths[0] + if (len(paths) == 0): + tdLog.exit("taosBenchmark not found!") + return + else: + tdLog.info("taosBenchmark found in %s" % paths[0]) + return paths[0] def run(self): tdSql.prepare() diff --git a/tests/develop-test/test.py b/tests/develop-test/test.py index 600925174f358b6a4659b10b66a825cc7e12641d..6b1c63a1c0c4fe9415c921b97beb3dbe336235ce 100644 --- a/tests/develop-test/test.py +++ b/tests/develop-test/test.py @@ -316,9 +316,9 @@ if __name__ == "__main__": print(r) else: pass - if restful: - tAdapter.init(deployPath, masterIp) - tAdapter.stop(force_kill=True) + + tAdapter.init(deployPath, masterIp) + tAdapter.stop(force_kill=True) if dnodeNums == 1: tdDnodes.deploy(1, updateCfgDict) diff --git a/tests/develop-test/win-test-file b/tests/develop-test/win-test-file new file mode 100644 index 0000000000000000000000000000000000000000..e4f3bcf56ee0ea3154232d384e005855b60e3c82 --- /dev/null +++ b/tests/develop-test/win-test-file @@ -0,0 +1,13 @@ +python3 ./test.py -f 2-query/table_count_scan.py +python3 ./test.py -f 2-query/show_create_db.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R +python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R diff --git a/tests/docs-examples-test/jdbc.sh b/tests/docs-examples-test/jdbc.sh index d71085a40306956ea8d25e9b575c97ae9945df76..4fcc5404b6585ec64383cb8cd341687119698dd6 100644 --- a/tests/docs-examples-test/jdbc.sh +++ b/tests/docs-examples-test/jdbc.sh @@ -6,15 +6,10 @@ cd ../../docs/examples/java mvn clean test > jdbc-out.log 2>&1 tail -n 20 jdbc-out.log - -cases=`grep 'Tests run' jdbc-out.log | awk 'END{print $3}'` -totalJDBCCases=`echo ${cases/%,}` -failed=`grep 'Tests run' jdbc-out.log | awk 'END{print $5}'` -JDBCFailed=`echo ${failed/%,}` -error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'` -JDBCError=`echo ${error/%,}` - -totalJDBCFailed=`expr $JDBCFailed + $JDBCError` +totalJDBCCases=`grep 'Tests run' jdbc-out.log | awk -F"[:,]" 'END{ print $2 }'` +failed=`grep 'Tests run' jdbc-out.log | awk -F"[:,]" 'END{ print $4 }'` +error=`grep 'Tests run' jdbc-out.log | awk -F"[:,]" 'END{ print $6 }'` +totalJDBCFailed=`expr $failed + $error` totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed` if [ "$totalJDBCSuccess" -gt "0" ]; then diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index a7501b54ed56b64121ee13913f8abd52bba9e381..2a44ee75528ed462460cc6783a2cca62f3a022ef 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -52,7 +52,7 @@ python3 conn_rest_pandas.py taos -s "drop database if exists power" # 11 -taos -s "create database if not exists test" +taos -s "create database if not exists test wal_retention_period 3600" python3 connect_native_reference.py # 12 diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dfb8230cc7e91c8c50a9043b95faa912b3423838..dda4ec3e84c585af785447f5b2c154cf67199c48 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,414 +5,110 @@ #unit-test ,,y,unit-test,bash test.sh -#tsim test -,,y,script,./test.sh -f tsim/user/basic.sim -,,y,script,./test.sh -f tsim/user/password.sim -,,y,script,./test.sh -f tsim/user/privilege_db.sim -,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim -,,y,script,./test.sh -f tsim/user/privilege_topic.sim -,,y,script,./test.sh -f tsim/db/alter_option.sim -,,y,script,./test.sh -f tsim/db/alter_replica_13.sim -,,y,script,./test.sh -f tsim/db/alter_replica_31.sim -,,y,script,./test.sh -f tsim/db/basic1.sim -,,y,script,./test.sh -f tsim/db/basic2.sim -,,y,script,./test.sh -f tsim/db/basic3.sim -,,y,script,./test.sh -f tsim/db/basic4.sim -,,y,script,./test.sh -f tsim/db/basic5.sim -,,y,script,./test.sh -f tsim/db/basic6.sim -,,y,script,./test.sh -f tsim/db/commit.sim -,,y,script,./test.sh -f tsim/db/create_all_options.sim -,,y,script,./test.sh -f tsim/db/delete_reuse1.sim -,,y,script,./test.sh -f tsim/db/delete_reuse2.sim -,,y,script,./test.sh -f tsim/db/delete_reusevnode.sim -,,y,script,./test.sh -f tsim/db/delete_reusevnode2.sim -,,y,script,./test.sh -f tsim/db/delete_writing1.sim -,,y,script,./test.sh -f tsim/db/delete_writing2.sim -,,y,script,./test.sh -f tsim/db/error1.sim -,,y,script,./test.sh -f tsim/db/keep.sim -,,y,script,./test.sh -f tsim/db/len.sim -,,y,script,./test.sh -f tsim/db/repeat.sim -,,y,script,./test.sh -f tsim/db/show_create_db.sim -,,y,script,./test.sh -f tsim/db/show_create_table.sim -,,y,script,./test.sh -f tsim/db/tables.sim -,,y,script,./test.sh -f tsim/db/taosdlog.sim -,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim -,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim -,,y,script,./test.sh -f tsim/dnode/balance1.sim -,,y,script,./test.sh -f tsim/dnode/balance2.sim -,,y,script,./test.sh -f tsim/dnode/balance3.sim -,,y,script,./test.sh -f tsim/dnode/balancex.sim -,,y,script,./test.sh -f tsim/dnode/create_dnode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim -,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim -,,y,script,./test.sh -f tsim/dnode/offline_reason.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim -,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim -,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim -,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim -,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim -,,y,script,./test.sh -f tsim/import/basic.sim -,,y,script,./test.sh -f tsim/import/commit.sim -,,y,script,./test.sh -f tsim/import/large.sim -,,y,script,./test.sh -f tsim/import/replica1.sim -,,y,script,./test.sh -f tsim/insert/backquote.sim -,,y,script,./test.sh -f tsim/insert/basic.sim -,,y,script,./test.sh -f tsim/insert/basic0.sim -,,y,script,./test.sh -f tsim/insert/basic1.sim -,,y,script,./test.sh -f tsim/insert/basic2.sim -,,y,script,./test.sh -f tsim/insert/commit-merge0.sim -,,y,script,./test.sh -f tsim/insert/insert_drop.sim -,,y,script,./test.sh -f tsim/insert/insert_select.sim -,,y,script,./test.sh -f tsim/insert/null.sim -,,y,script,./test.sh -f tsim/insert/query_block1_file.sim -,,y,script,./test.sh -f tsim/insert/query_block1_memory.sim -,,y,script,./test.sh -f tsim/insert/query_block2_file.sim -,,y,script,./test.sh -f tsim/insert/query_block2_memory.sim -,,y,script,./test.sh -f tsim/insert/query_file_memory.sim -,,y,script,./test.sh -f tsim/insert/query_multi_file.sim -,,y,script,./test.sh -f tsim/insert/tcp.sim -,,y,script,./test.sh -f tsim/insert/update0.sim -,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim -,,y,script,./test.sh -f tsim/insert/update2.sim -,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim -,,y,script,./test.sh -f tsim/parser/alter_column.sim -,,y,script,./test.sh -f tsim/parser/alter_stable.sim -,,y,script,./test.sh -f tsim/parser/alter.sim -,,y,script,./test.sh -f tsim/parser/alter1.sim -,,y,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -,,y,script,./test.sh -f tsim/parser/auto_create_tb.sim -,,y,script,./test.sh -f tsim/parser/between_and.sim -,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim -,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim -,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim -,,y,script,./test.sh -f tsim/parser/columnValue_double.sim -,,y,script,./test.sh -f tsim/parser/columnValue_float.sim -,,y,script,./test.sh -f tsim/parser/columnValue_int.sim -,,y,script,./test.sh -f tsim/parser/columnValue_smallint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim -,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim -,,y,script,./test.sh -f tsim/parser/commit.sim -,,y,script,./test.sh -f tsim/parser/condition.sim -,,y,script,./test.sh -f tsim/parser/constCol.sim -,,y,script,./test.sh -f tsim/parser/create_db.sim -,,y,script,./test.sh -f tsim/parser/create_mt.sim -,,y,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim -,,y,script,./test.sh -f tsim/parser/create_tb.sim -,,y,script,./test.sh -f tsim/parser/dbtbnameValidate.sim -,,y,script,./test.sh -f tsim/parser/distinct.sim -,,y,script,./test.sh -f tsim/parser/fill_us.sim -,,y,script,./test.sh -f tsim/parser/fill.sim -,,y,script,./test.sh -f tsim/parser/first_last.sim -,,y,script,./test.sh -f tsim/parser/fill_stb.sim -,,y,script,./test.sh -f tsim/parser/interp.sim -,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim -,,y,script,./test.sh -f tsim/parser/function.sim -,,y,script,./test.sh -f tsim/parser/groupby-basic.sim -,,y,script,./test.sh -f tsim/parser/groupby.sim -,,y,script,./test.sh -f tsim/parser/having_child.sim -,,y,script,./test.sh -f tsim/parser/having.sim -,,y,script,./test.sh -f tsim/parser/import_commit1.sim -,,y,script,./test.sh -f tsim/parser/import_commit2.sim -,,y,script,./test.sh -f tsim/parser/import_commit3.sim -,,y,script,./test.sh -f tsim/parser/import_file.sim -,,y,script,./test.sh -f tsim/parser/import.sim -,,y,script,./test.sh -f tsim/parser/insert_multiTbl.sim -,,y,script,./test.sh -f tsim/parser/insert_tb.sim -,,y,script,./test.sh -f tsim/parser/join_manyblocks.sim -,,y,script,./test.sh -f tsim/parser/join_multitables.sim -,,y,script,./test.sh -f tsim/parser/join_multivnode.sim -,,y,script,./test.sh -f tsim/parser/join.sim -,,y,script,./test.sh -f tsim/parser/last_cache.sim -,,y,script,./test.sh -f tsim/parser/last_groupby.sim -,,y,script,./test.sh -f tsim/parser/lastrow.sim -,,y,script,./test.sh -f tsim/parser/lastrow2.sim -,,y,script,./test.sh -f tsim/parser/like.sim -,,y,script,./test.sh -f tsim/parser/limit.sim -,,y,script,./test.sh -f tsim/parser/limit1.sim -,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim -,,y,script,./test.sh -f tsim/parser/nchar.sim -,,y,script,./test.sh -f tsim/parser/nestquery.sim -,,y,script,./test.sh -f tsim/parser/null_char.sim -,,y,script,./test.sh -f tsim/parser/precision_ns.sim -,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim -,,y,script,./test.sh -f tsim/parser/regex.sim -,,y,script,./test.sh -f tsim/parser/regressiontest.sim -,,y,script,./test.sh -f tsim/parser/select_across_vnodes.sim -,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim -,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim -,,y,script,./test.sh -f tsim/parser/select_with_tags.sim -,,y,script,./test.sh -f tsim/parser/selectResNum.sim -,,y,script,./test.sh -f tsim/parser/set_tag_vals.sim -,,y,script,./test.sh -f tsim/parser/single_row_in_tb.sim -,,y,script,./test.sh -f tsim/parser/sliding.sim -,,y,script,./test.sh -f tsim/parser/slimit_alter_tags.sim -,,y,script,./test.sh -f tsim/parser/slimit.sim -,,y,script,./test.sh -f tsim/parser/slimit1.sim -,,y,script,./test.sh -f tsim/parser/stableOp.sim -,,y,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim -,,y,script,./test.sh -f tsim/parser/tags_filter.sim -,,y,script,./test.sh -f tsim/parser/tbnameIn.sim -,,y,script,./test.sh -f tsim/parser/timestamp.sim -,,y,script,./test.sh -f tsim/parser/top_groupby.sim -,,y,script,./test.sh -f tsim/parser/topbot.sim -,,y,script,./test.sh -f tsim/parser/union.sim -,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim -,,y,script,./test.sh -f tsim/parser/where.sim -,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim -,,y,script,./test.sh -f tsim/query/charScalarFunction.sim -,,y,script,./test.sh -f tsim/query/explain.sim -,,y,script,./test.sh -f tsim/query/interval-offset.sim -,,y,script,./test.sh -f tsim/query/interval.sim -,,y,script,./test.sh -f tsim/query/scalarFunction.sim -,,y,script,./test.sh -f tsim/query/scalarNull.sim -,,y,script,./test.sh -f tsim/query/session.sim -,,y,script,./test.sh -f tsim/query/udf.sim -,,y,script,./test.sh -f tsim/query/udf_with_const.sim -,,y,script,./test.sh -f tsim/query/sys_tbname.sim -,,y,script,./test.sh -f tsim/query/groupby.sim -,,y,script,./test.sh -f tsim/query/event.sim -,,y,script,./test.sh -f tsim/query/forceFill.sim -,,y,script,./test.sh -f tsim/query/emptyTsRange.sim -,,y,script,./test.sh -f tsim/query/partitionby.sim -,,y,script,./test.sh -f tsim/qnode/basic1.sim -,,y,script,./test.sh -f tsim/snode/basic1.sim -,,y,script,./test.sh -f tsim/mnode/basic1.sim -,,y,script,./test.sh -f tsim/mnode/basic2.sim -,,y,script,./test.sh -f tsim/mnode/basic3.sim -,,y,script,./test.sh -f tsim/mnode/basic4.sim -,,y,script,./test.sh -f tsim/mnode/basic5.sim -,,y,script,./test.sh -f tsim/show/basic.sim -,,y,script,./test.sh -f tsim/table/autocreate.sim -,,y,script,./test.sh -f tsim/table/basic1.sim -,,y,script,./test.sh -f tsim/table/basic2.sim -,,y,script,./test.sh -f tsim/table/basic3.sim -,,y,script,./test.sh -f tsim/table/bigint.sim -,,y,script,./test.sh -f tsim/table/binary.sim -,,y,script,./test.sh -f tsim/table/bool.sim -,,y,script,./test.sh -f tsim/table/column_name.sim -,,y,script,./test.sh -f tsim/table/column_num.sim -,,y,script,./test.sh -f tsim/table/column_value.sim -,,y,script,./test.sh -f tsim/table/column2.sim -,,y,script,./test.sh -f tsim/table/createmulti.sim -,,y,script,./test.sh -f tsim/table/date.sim -,,y,script,./test.sh -f tsim/table/db.table.sim -,,y,script,./test.sh -f tsim/table/delete_reuse1.sim -,,y,script,./test.sh -f tsim/table/delete_reuse2.sim -,,y,script,./test.sh -f tsim/table/delete_writing.sim -,,y,script,./test.sh -f tsim/table/describe.sim -,,y,script,./test.sh -f tsim/table/double.sim -,,y,script,./test.sh -f tsim/table/float.sim -,,y,script,./test.sh -f tsim/table/hash.sim -,,y,script,./test.sh -f tsim/table/int.sim -,,y,script,./test.sh -f tsim/table/limit.sim -,,y,script,./test.sh -f tsim/table/smallint.sim -,,y,script,./test.sh -f tsim/table/table_len.sim -,,y,script,./test.sh -f tsim/table/table.sim -,,y,script,./test.sh -f tsim/table/tinyint.sim -,,y,script,./test.sh -f tsim/table/vgroup.sim -,,n,script,./test.sh -f tsim/stream/basic0.sim -g -,,y,script,./test.sh -f tsim/stream/basic1.sim -,,y,script,./test.sh -f tsim/stream/basic2.sim -,,y,script,./test.sh -f tsim/stream/drop_stream.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim -,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim -,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim -,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim -,,y,script,./test.sh -f tsim/stream/distributeSession0.sim -,,y,script,./test.sh -f tsim/stream/session0.sim -,,y,script,./test.sh -f tsim/stream/session1.sim -,,y,script,./test.sh -f tsim/stream/state0.sim -,,y,script,./test.sh -f tsim/stream/triggerInterval0.sim -,,y,script,./test.sh -f tsim/stream/triggerSession0.sim -,,y,script,./test.sh -f tsim/stream/partitionby.sim -,,y,script,./test.sh -f tsim/stream/partitionby1.sim -,,y,script,./test.sh -f tsim/stream/schedSnode.sim -,,y,script,./test.sh -f tsim/stream/windowClose.sim -,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim -,,y,script,./test.sh -f tsim/stream/sliding.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim -,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim -,,y,script,./test.sh -f tsim/stream/deleteInterval.sim -,,y,script,./test.sh -f tsim/stream/deleteSession.sim -,,y,script,./test.sh -f tsim/stream/deleteState.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalLinear.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim -,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim -,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim -,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim -,,y,script,./test.sh -f tsim/trans/lossdata1.sim -,,y,script,./test.sh -f tsim/trans/create_db.sim -,,y,script,./test.sh -f tsim/tmq/basic1.sim -,,y,script,./test.sh -f tsim/tmq/basic2.sim -,,y,script,./test.sh -f tsim/tmq/basic3.sim -,,y,script,./test.sh -f tsim/tmq/basic4.sim -,,y,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim -,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim -,,y,script,./test.sh -f tsim/tmq/topic.sim -,,y,script,./test.sh -f tsim/tmq/snapshot.sim -,,y,script,./test.sh -f tsim/tmq/snapshot1.sim -,,y,script,./test.sh -f tsim/stable/alter_comment.sim -,,y,script,./test.sh -f tsim/stable/alter_count.sim -,,y,script,./test.sh -f tsim/stable/alter_import.sim -,,y,script,./test.sh -f tsim/stable/alter_insert1.sim -,,y,script,./test.sh -f tsim/stable/alter_insert2.sim -,,y,script,./test.sh -f tsim/stable/alter_metrics.sim -,,y,script,./test.sh -f tsim/stable/column_add.sim -,,y,script,./test.sh -f tsim/stable/column_drop.sim -,,y,script,./test.sh -f tsim/stable/column_modify.sim -,,y,script,./test.sh -f tsim/stable/disk.sim -,,y,script,./test.sh -f tsim/stable/dnode3.sim -,,y,script,./test.sh -f tsim/stable/metrics.sim -,,y,script,./test.sh -f tsim/stable/refcount.sim -,,y,script,./test.sh -f tsim/stable/tag_add.sim -,,y,script,./test.sh -f tsim/stable/tag_drop.sim -,,y,script,./test.sh -f tsim/stable/tag_filter.sim -,,y,script,./test.sh -f tsim/stable/tag_modify.sim -,,y,script,./test.sh -f tsim/stable/tag_rename.sim -,,y,script,./test.sh -f tsim/stable/values.sim -,,y,script,./test.sh -f tsim/stable/vnode3.sim -,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim -,,y,script,./test.sh -f tsim/sma/sma_leak.sim -,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim -,,n,script,./test.sh -f tsim/valgrind/checkError1.sim -,,n,script,./test.sh -f tsim/valgrind/checkError2.sim -,,n,script,./test.sh -f tsim/valgrind/checkError3.sim -,,n,script,./test.sh -f tsim/valgrind/checkError4.sim -,,n,script,./test.sh -f tsim/valgrind/checkError5.sim -,,n,script,./test.sh -f tsim/valgrind/checkError6.sim -,,n,script,./test.sh -f tsim/valgrind/checkError7.sim -,,n,script,./test.sh -f tsim/valgrind/checkError8.sim -,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim -,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim -,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim -,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim -,,y,script,./test.sh -f tsim/vnode/replica3_many.sim -,,y,script,./test.sh -f tsim/vnode/replica3_import.sim -,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode2.sim -,,y,script,./test.sh -f tsim/vnode/stable_dnode3.sim -,,y,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim -,,y,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim -,,y,script,./test.sh -f tsim/sync/3Replica1VgElect.sim -,,y,script,./test.sh -f tsim/sync/3Replica5VgElect.sim -,,y,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim -,,y,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim -,,y,script,./test.sh -f tsim/catalog/alterInCurrent.sim -,,y,script,./test.sh -f tsim/scalar/in.sim -,,y,script,./test.sh -f tsim/scalar/scalar.sim -,,y,script,./test.sh -f tsim/scalar/filter.sim -,,y,script,./test.sh -f tsim/scalar/caseWhen.sim -,,y,script,./test.sh -f tsim/scalar/tsConvert.sim -,,y,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim -,,y,script,./test.sh -f tsim/alter/dnode.sim -,,y,script,./test.sh -f tsim/alter/table.sim -,,y,script,./test.sh -f tsim/cache/new_metrics.sim -,,y,script,./test.sh -f tsim/cache/restart_table.sim -,,y,script,./test.sh -f tsim/cache/restart_metrics.sim -,,y,script,./test.sh -f tsim/column/commit.sim -,,y,script,./test.sh -f tsim/column/metrics.sim -,,y,script,./test.sh -f tsim/column/table.sim -,,y,script,./test.sh -f tsim/compress/commitlog.sim -,,y,script,./test.sh -f tsim/compress/compress2.sim -,,y,script,./test.sh -f tsim/compress/compress.sim -,,y,script,./test.sh -f tsim/compress/uncompress.sim -,,y,script,./test.sh -f tsim/compute/avg.sim -,,y,script,./test.sh -f tsim/compute/block_dist.sim -,,y,script,./test.sh -f tsim/compute/bottom.sim -,,y,script,./test.sh -f tsim/compute/count.sim -,,y,script,./test.sh -f tsim/compute/diff.sim -,,y,script,./test.sh -f tsim/compute/diff2.sim -,,y,script,./test.sh -f tsim/compute/first.sim -,,y,script,./test.sh -f tsim/compute/interval.sim -,,y,script,./test.sh -f tsim/compute/last_row.sim -,,y,script,./test.sh -f tsim/compute/last.sim -,,y,script,./test.sh -f tsim/compute/leastsquare.sim -,,y,script,./test.sh -f tsim/compute/max.sim -,,y,script,./test.sh -f tsim/compute/min.sim -,,y,script,./test.sh -f tsim/compute/null.sim -,,y,script,./test.sh -f tsim/compute/percentile.sim -,,y,script,./test.sh -f tsim/compute/stddev.sim -,,y,script,./test.sh -f tsim/compute/sum.sim -,,y,script,./test.sh -f tsim/compute/top.sim -,,y,script,./test.sh -f tsim/field/2.sim -,,y,script,./test.sh -f tsim/field/3.sim -,,y,script,./test.sh -f tsim/field/4.sim -,,y,script,./test.sh -f tsim/field/5.sim -,,y,script,./test.sh -f tsim/field/6.sim -,,y,script,./test.sh -f tsim/field/binary.sim -,,y,script,./test.sh -f tsim/field/bigint.sim -,,y,script,./test.sh -f tsim/field/bool.sim -,,y,script,./test.sh -f tsim/field/double.sim -,,y,script,./test.sh -f tsim/field/float.sim -,,y,script,./test.sh -f tsim/field/int.sim -,,y,script,./test.sh -f tsim/field/single.sim -,,y,script,./test.sh -f tsim/field/smallint.sim -,,y,script,./test.sh -f tsim/field/tinyint.sim -,,y,script,./test.sh -f tsim/field/unsigined_bigint.sim -,,y,script,./test.sh -f tsim/vector/metrics_field.sim -,,y,script,./test.sh -f tsim/vector/metrics_mix.sim -,,y,script,./test.sh -f tsim/vector/metrics_query.sim -,,y,script,./test.sh -f tsim/vector/metrics_tag.sim -,,y,script,./test.sh -f tsim/vector/metrics_time.sim -,,y,script,./test.sh -f tsim/vector/multi.sim -,,y,script,./test.sh -f tsim/vector/single.sim -,,y,script,./test.sh -f tsim/vector/table_field.sim -,,y,script,./test.sh -f tsim/vector/table_mix.sim -,,y,script,./test.sh -f tsim/vector/table_query.sim -,,y,script,./test.sh -f tsim/vector/table_time.sim -,,y,script,./test.sh -f tsim/wal/kill.sim -,,y,script,./test.sh -f tsim/tag/3.sim -,,y,script,./test.sh -f tsim/tag/4.sim -,,y,script,./test.sh -f tsim/tag/5.sim -,,y,script,./test.sh -f tsim/tag/6.sim -,,y,script,./test.sh -f tsim/tag/add.sim -,,y,script,./test.sh -f tsim/tag/bigint.sim -,,y,script,./test.sh -f tsim/tag/binary_binary.sim -,,y,script,./test.sh -f tsim/tag/binary.sim -,,y,script,./test.sh -f tsim/tag/bool_binary.sim -,,y,script,./test.sh -f tsim/tag/bool_int.sim -,,y,script,./test.sh -f tsim/tag/bool.sim -,,y,script,./test.sh -f tsim/tag/change.sim -,,y,script,./test.sh -f tsim/tag/column.sim -,,y,script,./test.sh -f tsim/tag/commit.sim -,,y,script,./test.sh -f tsim/tag/create.sim -,,y,script,./test.sh -f tsim/tag/delete.sim -,,y,script,./test.sh -f tsim/tag/double.sim -,,y,script,./test.sh -f tsim/tag/filter.sim -,,y,script,./test.sh -f tsim/tag/float.sim -,,y,script,./test.sh -f tsim/tag/int_binary.sim -,,y,script,./test.sh -f tsim/tag/int_float.sim -,,y,script,./test.sh -f tsim/tag/int.sim -,,y,script,./test.sh -f tsim/tag/set.sim -,,y,script,./test.sh -f tsim/tag/smallint.sim -,,y,script,./test.sh -f tsim/tag/tinyint.sim -,,y,script,./test.sh -f tsim/tag/drop_tag.sim -,,y,script,./test.sh -f tsim/tag/tbNameIn.sim -,,y,script,./test.sh -f tmp/monitor.sim +#system test +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb3.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 + + +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 + +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb4.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb4.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/db.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/raw_block_interface_test.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -#system test ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py @@ -433,6 +129,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py +,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py +,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py @@ -452,8 +150,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py @@ -558,6 +256,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py @@ -636,7 +335,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py -,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py @@ -646,11 +344,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py @@ -698,68 +393,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb2.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb3.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeDb4.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb2.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb3.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb4.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/db.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqError.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/schema.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbFilter.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqCheckData1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAlterSchema.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py -,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 @@ -819,11 +452,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 @@ -852,6 +481,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 @@ -859,6 +489,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 + ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 @@ -874,7 +505,6 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3 @@ -917,11 +547,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 @@ -950,6 +576,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 @@ -1015,13 +642,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 4 @@ -1049,6 +672,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 @@ -1059,27 +683,429 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -R -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py +#tsim test +,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +,,y,script,./test.sh -f tsim/parser/where.sim +,,y,script,./test.sh -f tsim/parser/join_manyblocks.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +,,y,script,./test.sh -f tsim/parser/limit1.sim +,,y,script,./test.sh -f tsim/parser/union.sim +,,y,script,./test.sh -f tsim/parser/commit.sim +,,y,script,./test.sh -f tsim/parser/nestquery.sim +,,n,script,./test.sh -f tsim/valgrind/checkError7.sim +,,y,script,./test.sh -f tsim/parser/groupby.sim +,,y,script,./test.sh -f tsim/parser/sliding.sim +,,y,script,./test.sh -f tsim/dnode/balance2.sim +,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim +,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim +,,y,script,./test.sh -f tsim/trans/create_db.sim +,,y,script,./test.sh -f tsim/dnode/balance3.sim +,,y,script,./test.sh -f tsim/vnode/replica3_many.sim +,,y,script,./test.sh -f tsim/stable/metrics_idx.sim +,,y,script,./test.sh -f tsim/db/alter_replica_13.sim +,,y,script,./test.sh -f tsim/sync/3Replica1VgElect.sim +,,y,script,./test.sh -f tsim/sync/3Replica5VgElect.sim +,,n,script,./test.sh -f tsim/valgrind/checkError6.sim + +,,y,script,./test.sh -f tsim/user/basic.sim +,,y,script,./test.sh -f tsim/user/password.sim +,,y,script,./test.sh -f tsim/user/privilege_db.sim +,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,y,script,./test.sh -f tsim/user/privilege_topic.sim +,,y,script,./test.sh -f tsim/db/alter_option.sim +,,y,script,./test.sh -f tsim/db/alter_replica_31.sim +,,y,script,./test.sh -f tsim/db/basic1.sim +,,y,script,./test.sh -f tsim/db/basic2.sim +,,y,script,./test.sh -f tsim/db/basic3.sim +,,y,script,./test.sh -f tsim/db/basic4.sim +,,y,script,./test.sh -f tsim/db/basic5.sim +,,y,script,./test.sh -f tsim/db/basic6.sim +,,y,script,./test.sh -f tsim/db/commit.sim +,,y,script,./test.sh -f tsim/db/create_all_options.sim +,,y,script,./test.sh -f tsim/db/delete_reuse1.sim +,,y,script,./test.sh -f tsim/db/delete_reuse2.sim +,,y,script,./test.sh -f tsim/db/delete_reusevnode.sim +,,y,script,./test.sh -f tsim/db/delete_reusevnode2.sim +,,y,script,./test.sh -f tsim/db/delete_writing1.sim +,,y,script,./test.sh -f tsim/db/delete_writing2.sim +,,y,script,./test.sh -f tsim/db/error1.sim +,,y,script,./test.sh -f tsim/db/keep.sim +,,y,script,./test.sh -f tsim/db/len.sim +,,y,script,./test.sh -f tsim/db/repeat.sim +,,y,script,./test.sh -f tsim/db/show_create_db.sim +,,y,script,./test.sh -f tsim/db/show_create_table.sim +,,y,script,./test.sh -f tsim/db/tables.sim +,,y,script,./test.sh -f tsim/db/taosdlog.sim +,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim +,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim +,,y,script,./test.sh -f tsim/dnode/balance1.sim +,,y,script,./test.sh -f tsim/dnode/balancex.sim +,,y,script,./test.sh -f tsim/dnode/create_dnode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim +,,y,script,./test.sh -f tsim/dnode/offline_reason.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim +,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim +,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim +,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim +,,y,script,./test.sh -f tsim/import/basic.sim +,,y,script,./test.sh -f tsim/import/commit.sim +,,y,script,./test.sh -f tsim/import/large.sim +,,y,script,./test.sh -f tsim/import/replica1.sim +,,y,script,./test.sh -f tsim/insert/backquote.sim +,,y,script,./test.sh -f tsim/insert/basic.sim +,,y,script,./test.sh -f tsim/insert/basic0.sim +,,y,script,./test.sh -f tsim/insert/basic1.sim +,,y,script,./test.sh -f tsim/insert/basic2.sim +,,y,script,./test.sh -f tsim/insert/commit-merge0.sim +,,y,script,./test.sh -f tsim/insert/insert_drop.sim +,,y,script,./test.sh -f tsim/insert/insert_select.sim +,,y,script,./test.sh -f tsim/insert/null.sim +,,y,script,./test.sh -f tsim/insert/query_block1_file.sim +,,y,script,./test.sh -f tsim/insert/query_block1_memory.sim +,,y,script,./test.sh -f tsim/insert/query_block2_file.sim +,,y,script,./test.sh -f tsim/insert/query_block2_memory.sim +,,y,script,./test.sh -f tsim/insert/query_file_memory.sim +,,y,script,./test.sh -f tsim/insert/query_multi_file.sim +,,y,script,./test.sh -f tsim/insert/tcp.sim +,,y,script,./test.sh -f tsim/insert/update0.sim +,,y,script,./test.sh -f tsim/insert/delete0.sim +,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/insert/update2.sim +,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim +,,y,script,./test.sh -f tsim/parser/alter_column.sim +,,y,script,./test.sh -f tsim/parser/alter_stable.sim +,,y,script,./test.sh -f tsim/parser/alter.sim +,,y,script,./test.sh -f tsim/parser/alter1.sim +,,y,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +,,y,script,./test.sh -f tsim/parser/auto_create_tb.sim +,,y,script,./test.sh -f tsim/parser/between_and.sim +,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim +,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim +,,y,script,./test.sh -f tsim/parser/columnValue_double.sim +,,y,script,./test.sh -f tsim/parser/columnValue_float.sim +,,y,script,./test.sh -f tsim/parser/columnValue_int.sim +,,y,script,./test.sh -f tsim/parser/columnValue_smallint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim +,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/constCol.sim +,,y,script,./test.sh -f tsim/parser/create_db.sim +,,y,script,./test.sh -f tsim/parser/create_mt.sim +,,y,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim +,,y,script,./test.sh -f tsim/parser/create_tb.sim +,,y,script,./test.sh -f tsim/parser/dbtbnameValidate.sim +,,y,script,./test.sh -f tsim/parser/distinct.sim +,,y,script,./test.sh -f tsim/parser/fill_us.sim +,,y,script,./test.sh -f tsim/parser/fill.sim +,,y,script,./test.sh -f tsim/parser/first_last.sim +,,y,script,./test.sh -f tsim/parser/fill_stb.sim +,,y,script,./test.sh -f tsim/parser/interp.sim +,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim +,,y,script,./test.sh -f tsim/parser/function.sim +,,y,script,./test.sh -f tsim/parser/groupby-basic.sim +,,y,script,./test.sh -f tsim/parser/having_child.sim +,,y,script,./test.sh -f tsim/parser/having.sim +,,y,script,./test.sh -f tsim/parser/import_commit1.sim +,,y,script,./test.sh -f tsim/parser/import_commit2.sim +,,y,script,./test.sh -f tsim/parser/import_commit3.sim +,,y,script,./test.sh -f tsim/parser/import_file.sim +,,y,script,./test.sh -f tsim/parser/import.sim +,,y,script,./test.sh -f tsim/parser/insert_multiTbl.sim +,,y,script,./test.sh -f tsim/parser/insert_tb.sim +,,y,script,./test.sh -f tsim/parser/join_multitables.sim +,,y,script,./test.sh -f tsim/parser/join_multivnode.sim +,,y,script,./test.sh -f tsim/parser/join.sim +,,y,script,./test.sh -f tsim/parser/last_cache.sim +,,y,script,./test.sh -f tsim/parser/last_groupby.sim +,,y,script,./test.sh -f tsim/parser/lastrow.sim +,,y,script,./test.sh -f tsim/parser/lastrow2.sim +,,y,script,./test.sh -f tsim/parser/like.sim +,,y,script,./test.sh -f tsim/parser/limit.sim +,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim +,,y,script,./test.sh -f tsim/parser/nchar.sim +,,y,script,./test.sh -f tsim/parser/null_char.sim +,,y,script,./test.sh -f tsim/parser/precision_ns.sim +,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim +,,y,script,./test.sh -f tsim/parser/regex.sim +,,y,script,./test.sh -f tsim/parser/regressiontest.sim +,,y,script,./test.sh -f tsim/parser/select_across_vnodes.sim +,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim +,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim +,,y,script,./test.sh -f tsim/parser/select_with_tags.sim +,,y,script,./test.sh -f tsim/parser/selectResNum.sim +,,y,script,./test.sh -f tsim/parser/set_tag_vals.sim +,,y,script,./test.sh -f tsim/parser/single_row_in_tb.sim +,,y,script,./test.sh -f tsim/parser/slimit_alter_tags.sim +,,y,script,./test.sh -f tsim/parser/slimit.sim +,,y,script,./test.sh -f tsim/parser/slimit1.sim +,,y,script,./test.sh -f tsim/parser/stableOp.sim +,,y,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +,,y,script,./test.sh -f tsim/parser/tags_filter.sim +,,y,script,./test.sh -f tsim/parser/tbnameIn.sim +,,y,script,./test.sh -f tsim/parser/timestamp.sim +,,y,script,./test.sh -f tsim/parser/top_groupby.sim +,,y,script,./test.sh -f tsim/parser/topbot.sim +,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim +,,y,script,./test.sh -f tsim/parser/slimit_limit.sim +,,y,script,./test.sh -f tsim/parser/table_merge_limit.sim +,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim +,,y,script,./test.sh -f tsim/query/charScalarFunction.sim +,,y,script,./test.sh -f tsim/query/explain.sim +,,y,script,./test.sh -f tsim/query/interval-offset.sim +,,y,script,./test.sh -f tsim/query/interval.sim +,,y,script,./test.sh -f tsim/query/scalarFunction.sim +,,y,script,./test.sh -f tsim/query/scalarNull.sim +,,y,script,./test.sh -f tsim/query/session.sim +,,y,script,./test.sh -f tsim/query/udf.sim +,,n,script,./test.sh -f tsim/query/udfpy.sim +,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/join_interval.sim +,,y,script,./test.sh -f tsim/query/unionall_as_table.sim +,,y,script,./test.sh -f tsim/query/multi_order_by.sim +,,y,script,./test.sh -f tsim/query/sys_tbname.sim +,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/event.sim +,,y,script,./test.sh -f tsim/query/forceFill.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/partitionby.sim +,,y,script,./test.sh -f tsim/query/tableCount.sim +,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/qnode/basic1.sim +,,y,script,./test.sh -f tsim/snode/basic1.sim +,,y,script,./test.sh -f tsim/mnode/basic1.sim +,,y,script,./test.sh -f tsim/mnode/basic2.sim +,,y,script,./test.sh -f tsim/mnode/basic3.sim +,,y,script,./test.sh -f tsim/mnode/basic4.sim +,,y,script,./test.sh -f tsim/mnode/basic5.sim +,,y,script,./test.sh -f tsim/show/basic.sim +,,y,script,./test.sh -f tsim/table/autocreate.sim +,,y,script,./test.sh -f tsim/table/basic1.sim +,,y,script,./test.sh -f tsim/table/basic2.sim +,,y,script,./test.sh -f tsim/table/basic3.sim +,,y,script,./test.sh -f tsim/table/bigint.sim +,,y,script,./test.sh -f tsim/table/binary.sim +,,y,script,./test.sh -f tsim/table/bool.sim +,,y,script,./test.sh -f tsim/table/column_name.sim +,,y,script,./test.sh -f tsim/table/column_num.sim +,,y,script,./test.sh -f tsim/table/column_value.sim +,,y,script,./test.sh -f tsim/table/column2.sim +,,y,script,./test.sh -f tsim/table/createmulti.sim +,,y,script,./test.sh -f tsim/table/date.sim +,,y,script,./test.sh -f tsim/table/db.table.sim +,,y,script,./test.sh -f tsim/table/delete_reuse1.sim +,,y,script,./test.sh -f tsim/table/delete_reuse2.sim +,,y,script,./test.sh -f tsim/table/delete_writing.sim +,,y,script,./test.sh -f tsim/table/describe.sim +,,y,script,./test.sh -f tsim/table/double.sim +,,y,script,./test.sh -f tsim/table/float.sim +,,y,script,./test.sh -f tsim/table/hash.sim +,,y,script,./test.sh -f tsim/table/int.sim +,,y,script,./test.sh -f tsim/table/limit.sim +,,y,script,./test.sh -f tsim/table/smallint.sim +,,y,script,./test.sh -f tsim/table/table_len.sim +,,y,script,./test.sh -f tsim/table/table.sim +,,y,script,./test.sh -f tsim/table/tinyint.sim +,,y,script,./test.sh -f tsim/table/vgroup.sim +,,n,script,./test.sh -f tsim/stream/basic0.sim -g +,,y,script,./test.sh -f tsim/stream/basic1.sim +,,y,script,./test.sh -f tsim/stream/basic2.sim +,,y,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim +,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,y,script,./test.sh -f tsim/stream/distributeSession0.sim +,,y,script,./test.sh -f tsim/stream/session0.sim +,,y,script,./test.sh -f tsim/stream/session1.sim +,,y,script,./test.sh -f tsim/stream/state0.sim +,,y,script,./test.sh -f tsim/stream/triggerInterval0.sim +,,y,script,./test.sh -f tsim/stream/triggerSession0.sim +,,y,script,./test.sh -f tsim/stream/partitionby.sim +,,y,script,./test.sh -f tsim/stream/partitionby1.sim +,,y,script,./test.sh -f tsim/stream/schedSnode.sim +,,y,script,./test.sh -f tsim/stream/windowClose.sim +,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim +,,y,script,./test.sh -f tsim/stream/sliding.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim +,,y,script,./test.sh -f tsim/stream/deleteInterval.sim +,,y,script,./test.sh -f tsim/stream/deleteSession.sim +,,y,script,./test.sh -f tsim/stream/deleteState.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalLinear.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim +,,y,script,./test.sh -f tsim/trans/lossdata1.sim +,,y,script,./test.sh -f tsim/tmq/basic1.sim +,,y,script,./test.sh -f tsim/tmq/basic2.sim +,,y,script,./test.sh -f tsim/tmq/basic3.sim +,,y,script,./test.sh -f tsim/tmq/basic4.sim +,,y,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/topic.sim +,,y,script,./test.sh -f tsim/tmq/snapshot.sim +,,y,script,./test.sh -f tsim/tmq/snapshot1.sim +,,y,script,./test.sh -f tsim/stable/alter_comment.sim +,,y,script,./test.sh -f tsim/stable/alter_count.sim +,,y,script,./test.sh -f tsim/stable/alter_import.sim +,,y,script,./test.sh -f tsim/stable/alter_insert1.sim +,,y,script,./test.sh -f tsim/stable/alter_insert2.sim +,,y,script,./test.sh -f tsim/stable/alter_metrics.sim +,,y,script,./test.sh -f tsim/stable/column_add.sim +,,y,script,./test.sh -f tsim/stable/column_drop.sim +,,y,script,./test.sh -f tsim/stable/column_modify.sim +,,y,script,./test.sh -f tsim/stable/disk.sim +,,y,script,./test.sh -f tsim/stable/dnode3.sim +,,y,script,./test.sh -f tsim/stable/metrics.sim +,,y,script,./test.sh -f tsim/stable/refcount.sim +,,y,script,./test.sh -f tsim/stable/tag_add.sim +,,y,script,./test.sh -f tsim/stable/tag_drop.sim +,,y,script,./test.sh -f tsim/stable/tag_filter.sim +,,y,script,./test.sh -f tsim/stable/tag_modify.sim +,,y,script,./test.sh -f tsim/stable/tag_rename.sim +,,y,script,./test.sh -f tsim/stable/values.sim +,,y,script,./test.sh -f tsim/stable/vnode3.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,y,script,./test.sh -f tsim/sma/sma_leak.sim +,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +,,n,script,./test.sh -f tsim/valgrind/checkError1.sim +,,n,script,./test.sh -f tsim/valgrind/checkError2.sim +,,n,script,./test.sh -f tsim/valgrind/checkError3.sim +,,n,script,./test.sh -f tsim/valgrind/checkError4.sim +,,n,script,./test.sh -f tsim/valgrind/checkError5.sim +,,n,script,./test.sh -f tsim/valgrind/checkError8.sim +,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim +,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim +,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim +,,y,script,./test.sh -f tsim/vnode/replica3_import.sim +,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode2.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode3.sim +,,y,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +,,y,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim +,,y,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim +,,y,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim +,,y,script,./test.sh -f tsim/catalog/alterInCurrent.sim +,,y,script,./test.sh -f tsim/scalar/in.sim +,,y,script,./test.sh -f tsim/scalar/scalar.sim +,,y,script,./test.sh -f tsim/scalar/filter.sim +,,y,script,./test.sh -f tsim/scalar/caseWhen.sim +,,y,script,./test.sh -f tsim/scalar/tsConvert.sim +,,y,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim +,,y,script,./test.sh -f tsim/alter/dnode.sim +,,y,script,./test.sh -f tsim/alter/table.sim +,,y,script,./test.sh -f tsim/cache/new_metrics.sim +,,y,script,./test.sh -f tsim/cache/restart_table.sim +,,y,script,./test.sh -f tsim/cache/restart_metrics.sim +,,y,script,./test.sh -f tsim/column/commit.sim +,,y,script,./test.sh -f tsim/column/metrics.sim +,,y,script,./test.sh -f tsim/column/table.sim +,,y,script,./test.sh -f tsim/compress/commitlog.sim +,,y,script,./test.sh -f tsim/compress/compress2.sim +,,y,script,./test.sh -f tsim/compress/compress.sim +,,y,script,./test.sh -f tsim/compress/uncompress.sim +,,y,script,./test.sh -f tsim/compute/avg.sim +,,y,script,./test.sh -f tsim/compute/block_dist.sim +,,y,script,./test.sh -f tsim/compute/bottom.sim +,,y,script,./test.sh -f tsim/compute/count.sim +,,y,script,./test.sh -f tsim/compute/diff.sim +,,y,script,./test.sh -f tsim/compute/diff2.sim +,,y,script,./test.sh -f tsim/compute/first.sim +,,y,script,./test.sh -f tsim/compute/interval.sim +,,y,script,./test.sh -f tsim/compute/last_row.sim +,,y,script,./test.sh -f tsim/compute/last.sim +,,y,script,./test.sh -f tsim/compute/leastsquare.sim +,,y,script,./test.sh -f tsim/compute/max.sim +,,y,script,./test.sh -f tsim/compute/min.sim +,,y,script,./test.sh -f tsim/compute/null.sim +,,y,script,./test.sh -f tsim/compute/percentile.sim +,,y,script,./test.sh -f tsim/compute/stddev.sim +,,y,script,./test.sh -f tsim/compute/sum.sim +,,y,script,./test.sh -f tsim/compute/top.sim +,,y,script,./test.sh -f tsim/field/2.sim +,,y,script,./test.sh -f tsim/field/3.sim +,,y,script,./test.sh -f tsim/field/4.sim +,,y,script,./test.sh -f tsim/field/5.sim +,,y,script,./test.sh -f tsim/field/6.sim +,,y,script,./test.sh -f tsim/field/binary.sim +,,y,script,./test.sh -f tsim/field/bigint.sim +,,y,script,./test.sh -f tsim/field/bool.sim +,,y,script,./test.sh -f tsim/field/double.sim +,,y,script,./test.sh -f tsim/field/float.sim +,,y,script,./test.sh -f tsim/field/int.sim +,,y,script,./test.sh -f tsim/field/single.sim +,,y,script,./test.sh -f tsim/field/smallint.sim +,,y,script,./test.sh -f tsim/field/tinyint.sim +,,y,script,./test.sh -f tsim/field/unsigined_bigint.sim +,,y,script,./test.sh -f tsim/vector/metrics_field.sim +,,y,script,./test.sh -f tsim/vector/metrics_mix.sim +,,y,script,./test.sh -f tsim/vector/metrics_query.sim +,,y,script,./test.sh -f tsim/vector/metrics_tag.sim +,,y,script,./test.sh -f tsim/vector/metrics_time.sim +,,y,script,./test.sh -f tsim/vector/multi.sim +,,y,script,./test.sh -f tsim/vector/single.sim +,,y,script,./test.sh -f tsim/vector/table_field.sim +,,y,script,./test.sh -f tsim/vector/table_mix.sim +,,y,script,./test.sh -f tsim/vector/table_query.sim +,,y,script,./test.sh -f tsim/vector/table_time.sim +,,y,script,./test.sh -f tsim/wal/kill.sim +,,y,script,./test.sh -f tsim/tag/3.sim +,,y,script,./test.sh -f tsim/tag/4.sim +,,y,script,./test.sh -f tsim/tag/5.sim +,,y,script,./test.sh -f tsim/tag/6.sim +,,y,script,./test.sh -f tsim/tag/add.sim +,,y,script,./test.sh -f tsim/tag/bigint.sim +,,y,script,./test.sh -f tsim/tag/binary_binary.sim +,,y,script,./test.sh -f tsim/tag/binary.sim +,,y,script,./test.sh -f tsim/tag/bool_binary.sim +,,y,script,./test.sh -f tsim/tag/bool_int.sim +,,y,script,./test.sh -f tsim/tag/bool.sim +,,y,script,./test.sh -f tsim/tag/change.sim +,,y,script,./test.sh -f tsim/tag/column.sim +,,y,script,./test.sh -f tsim/tag/commit.sim +,,y,script,./test.sh -f tsim/tag/create.sim +,,y,script,./test.sh -f tsim/tag/delete.sim +,,y,script,./test.sh -f tsim/tag/double.sim +,,y,script,./test.sh -f tsim/tag/filter.sim +,,y,script,./test.sh -f tsim/tag/float.sim +,,y,script,./test.sh -f tsim/tag/int_binary.sim +,,y,script,./test.sh -f tsim/tag/int_float.sim +,,y,script,./test.sh -f tsim/tag/int.sim +,,y,script,./test.sh -f tsim/tag/set.sim +,,y,script,./test.sh -f tsim/tag/smallint.sim +,,y,script,./test.sh -f tsim/tag/tinyint.sim +,,y,script,./test.sh -f tsim/tag/drop_tag.sim +,,y,script,./test.sh -f tsim/tag/tbNameIn.sim +,,y,script,./test.sh -f tmp/monitor.sim #develop test ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py ,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py @@ -1092,7 +1118,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index edb9b4ab3cae80af399f3ed3b22d4416bf1902d8..80236cf604d63e4d9c8fdddfa81aeaf16fb64944 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -68,8 +68,8 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j || exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1" + # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then echo "delete ${WORKDIR}/debugNoSan" @@ -97,7 +97,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 " + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=true;make -j || exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 43533d4f3688922d0f7670e5a74218a606c11f12..de343752c69801c76d1296f0e81685af591e046c 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -167,8 +167,10 @@ function run_thread() { local case_build_san=`echo "$line"|cut -d, -f3` if [ "${case_build_san}" == "y" ]; then case_build_san="y" + DEBUGPATH="debugSan" elif [[ "${case_build_san}" == "n" ]] || [[ "${case_build_san}" == "" ]]; then case_build_san="n" + DEBUGPATH="debugNoSan" else usage exit 1 @@ -274,6 +276,7 @@ function run_thread() { # echo "$thread_no ${line} DONE" if [ $ret -eq 0 ]; then echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" + flock -x $lock_file -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" else if [ ! -z ${web_server} ]; then flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" @@ -300,10 +303,10 @@ function run_thread() { if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/TDengine/debug/build" - if [ $ent -ne 0 ]; then - remote_build_dir="${workdirs[index]}/TDinternal/debug/build" - fi + local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + # if [ $ent -ne 0 ]; then + # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + # fi mkdir $build_dir 2>/dev/null if [ $? -eq 0 ]; then # scp build binary @@ -365,6 +368,8 @@ lock_file=$log_dir/$$.lock index_file=$log_dir/case_index.txt stat_file=$log_dir/stat.txt failed_case_file=$log_dir/failed.txt +success_case_file=$log_dir/success.txt + echo "0" >$index_file i=0 diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 95115bc4371cf0c0d7d17089e067a4615fb264c5..ffc23ba6d2baa83b534532d1eac9cb154d007ecb 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -51,6 +51,9 @@ if [ $ent -eq 0 ]; then ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null ln -s /home/TDengine/include/client/taos.h /usr/include/taos.h 2>/dev/null + ln -s /home/TDengine/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null + ln -s /home/TDengine/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null + ln -s /home/TDengine/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null CONTAINER_TESTDIR=/home/TDengine else export PATH=$PATH:/home/TDinternal/debug/build/bin @@ -58,6 +61,9 @@ else ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null ln -s /home/TDinternal/community/include/client/taos.h /usr/include/taos.h 2>/dev/null + ln -s /home/TDinternal/community/include/common/taosdef.h /usr/include/taosdef.h 2>/dev/null + ln -s /home/TDinternal/community/include/util/taoserror.h /usr/include/taoserror.h 2>/dev/null + ln -s /home/TDinternal/community/include/libs/function/taosudf.h /usr/include/taosudf.h 2>/dev/null CONTAINER_TESTDIR=/home/TDinternal/community fi mkdir -p /var/lib/taos/subscribe @@ -81,6 +87,7 @@ echo "cmd exit code: $RET" md5sum /usr/lib/libtaos.so.1 md5sum /home/TDinternal/debug/build/lib/libtaos.so + if [ $RET -ne 0 ]; then pwd fi diff --git a/tests/parallel_test/split_case.sh b/tests/parallel_test/split_case.sh new file mode 100755 index 0000000000000000000000000000000000000000..af601ed9a6a1c4d371e56d94aa45bc56b8715793 --- /dev/null +++ b/tests/parallel_test/split_case.sh @@ -0,0 +1,58 @@ +case_file="cases_temp_file" + +parm_path=$(dirname $0) +parm_path=$(pwd ${parm_path}) +echo "execute path:${parm_path}" +cd ${parm_path} +cp cases.task ${case_file} +sed -i '/^$/d' ${case_file} +sed -i '$a\%%FINISHED%%' ${case_file} + +utest="unit-test" +tsimtest="script" +systest="system-test" +devtest="develop-test" +doctest="docs-examples-test" +rm -rf win-${utest}.log win-${tsimtest}.log win-${systest}.log win-${devtest}.log win-${doctest}.log +rm -rf ${parm_path}/../${utest}/win-test-file ${parm_path}/../${tsimtest}/win-test-file ${parm_path}/../${systest}/win-test-file ${parm_path}/../${devtest}/win-test-file +while read -r line +do + echo "$line"|grep -q "^#" + if [ $? -eq 0 ]; then + continue + fi + exec_dir=$(echo "$line"|cut -d ',' -f4) + case_cmd=$(echo "$line"|cut -d ',' -f5) + if [[ "${exec_dir}" == "${utest}" ]]; then + echo ${case_cmd} >> win-${utest}.log + continue + fi + if [[ "${exec_dir}" == "${tsimtest}" ]]; then + echo ${case_cmd} >> win-${tsimtest}.log + continue + fi + if [[ "${exec_dir}" == "${systest}" ]]; then + if [[ "${case_cmd}" =~ "pytest.sh" ]]; then + case_cmd=$(echo "$case_cmd"|cut -d ' ' -f 2-) + echo ${case_cmd} >> win-${systest}.log + else + echo ${case_cmd} >> win-${systest}.log + fi + continue + fi + if [[ "${exec_dir}" == "${devtest}" ]]; then + echo ${case_cmd} >> win-${devtest}.log + continue + fi + if [[ "${exec_dir}" == "${doctest}" ]]; then + echo ${case_cmd} >> win-${doctest}.log + continue + fi +done < ${case_file} +mv win-${utest}.log ${parm_path}/../${utest}/win-test-file +mv win-${tsimtest}.log ${parm_path}/../${tsimtest}/win-test-file +mv win-${systest}.log ${parm_path}/../${systest}/win-test-file +mv win-${devtest}.log ${parm_path}/../${devtest}/win-test-file + + +rm -rf ${case_file} diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py new file mode 100644 index 0000000000000000000000000000000000000000..90cd347b9aed49816bc0de4242bacff8891173bc --- /dev/null +++ b/tests/pytest/util/autogen.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +import threading +import random +import string +import time + + +# +# Auto Gen class +# +class AutoGen: + def __init__(self): + self.ts = 1600000000000 + self.batch_size = 100 + seed = time.clock_gettime(time.CLOCK_REALTIME) + random.seed(seed) + + # set start ts + def set_start_ts(self, ts): + self.ts = ts + + # set batch size + def set_batch_size(self, batch_size): + self.batch_size = batch_size + + # _columns_sql + def gen_columns_sql(self, pre, cnt, binary_len, nchar_len): + types = [ + 'timestamp', + 'tinyint', + 'smallint', + 'tinyint unsigned', + 'smallint unsigned', + 'int', + 'bigint', + 'int unsigned', + 'bigint unsigned', + 'float', + 'double', + 'bool', + f'varchar({binary_len})', + f'nchar({nchar_len})' + ] + + sqls = "" + metas = [] + for i in range(cnt): + colname = f"{pre}{i}" + sel = i % len(types) + coltype = types[sel] + sql = f"{colname} {coltype}" + if sqls != "": + sqls += "," + sqls += sql + metas.append(sel) + + return metas, sqls; + + # gen tags data + def gen_data(self, i, marr): + datas = "" + for c in marr: + data = "" + if c == 0 : # timestamp + data = "%d" % (self.ts + i) + elif c <= 4 : # small + data = "%d"%(i%128) + elif c <= 8 : # int + data = f"{i}" + elif c <= 10 : # float + data = "%f"%(i+i/1000) + elif c <= 11 : # bool + data = "%d"%(i%2) + elif c == 12 : # binary + data = '"' + self.random_string(self.bin_len) + '"' + elif c == 13 : # binary + data = '"' + self.random_string(self.nch_len) + '"' + + if datas != "": + datas += "," + datas += data + + return datas + + # generate specail wide random string + def random_string(self, count): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(count)) + + # create db + def create_db(self, dbname): + self.dbname = dbname + tdSql.execute(f'create database {dbname}') + tdSql.execute(f'use {dbname}') + + # create table or stable + def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): + self.bin_len = binary_len + self.nch_len = nchar_len + self.stbname = stbname + self.mtags, tags = self.gen_columns_sql("t", tag_cnt, binary_len, nchar_len) + self.mcols, cols = self.gen_columns_sql("c", column_cnt - 1, binary_len, nchar_len) + + sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})" + tdSql.execute(sql) + + # create child table + def create_child(self, stbname, prename, cnt): + self.child_cnt = cnt + self.child_name = prename + for i in range(cnt): + tags_data = self.gen_data(i, self.mtags) + sql = f"create table {prename}{i} using {stbname} tags({tags_data})" + tdSql.execute(sql) + + tdLog.info(f"create child tables {cnt} ok") + + def insert_data_child(self, child_name, cnt, batch_size, step): + values = "" + print("insert child data") + ts = self.ts + + # loop do + for i in range(cnt): + value = self.gen_data(i, self.mcols) + ts += step + values += f"({ts},{value}) " + if batch_size == 1 or (i > 0 and i % batch_size == 0) : + sql = f"insert into {child_name} values {values}" + tdSql.execute(sql) + if batch_size > 40: + tdLog.info(f" insert data i={i}") + values = "" + + # end batch + if values != "": + sql = f"insert into {child_name} values {values}" + tdSql.execute(sql) + tdLog.info(f" insert data i={i}") + values = "" + + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + + # insert data + def insert_data(self, cnt): + for i in range(self.child_cnt): + name = f"{self.child_name}{i}" + self.insert_data_child(name, cnt, self.batch_size, 1) + + tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") + + # insert same timestamp to all childs + def insert_samets(self, cnt): + for i in range(self.child_cnt): + name = f"{self.child_name}{i}" + self.insert_data_child(name, cnt, self.batch_size, 0) + + tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") + + diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 82aa5523596c8d62557bad7d41d02e1ca897c9a9..536b8f30d3582875e85d1260861d6839f8503f6f 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -17,6 +17,7 @@ import time import datetime import inspect import importlib +import traceback from util.log import * @@ -75,6 +76,7 @@ class TDCases: case.run() except Exception as e: tdLog.notice(repr(e)) + traceback.print_exc() tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 @@ -94,14 +96,14 @@ class TDCases: tdLog.notice("total %d Windows test case(s) executed" % (runNum)) - def runOneWindows(self, conn, fileName): + def runOneWindows(self, conn, fileName, replicaVar=1): testModule = self.__dynamicLoadModule(fileName) runNum = 0 for tmp in self.windowsCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() - case.init(conn, self._logSql) + case.init(conn, self._logSql,replicaVar) try: case.run() except Exception as e: diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 5b73989d6f22b25c056cba802328d8e3d583ef87..6d813a416644661bf83881b753afe0d55d97d5f2 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -739,6 +739,11 @@ class TDCom: else: os.system("unset LD_PRELOAD; pkill %s " % processorName) + def gen_tag_col_str(self, gen_type, data_type, count): + """ + gen multi tags or cols by gen_type + """ + return ','.join(map(lambda i: f'{gen_type}{i} {data_type}', range(count))) def is_json(msg): if isinstance(msg, str): @@ -775,4 +780,5 @@ def dict2toml(in_dict: dict, file:str): with open(file, 'w') as f: toml.dump(in_dict, f) + tdCom = TDCom() diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py index 8da36f30748251f307a9152fd8907bdebc9e1405..6809c1082ca8db3c099f4d047be46104c0dccf5b 100644 --- a/tests/pytest/util/dnodes-default.py +++ b/tests/pytest/util/dnodes-default.py @@ -46,11 +46,11 @@ class TDSimClient: self.path = path def getLogDir(self): - self.logDir = "%s/sim/psim/log" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") return self.logDir def getCfgDir(self): - self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") return self.cfgDir def setTestCluster(self, value): @@ -65,9 +65,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") + self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -131,11 +131,10 @@ class TDDnode: return totalSize def deploy(self): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( - self.path, self.index) + self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") + self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -325,11 +324,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = os.path.join(self.path,"sim","psim") return dnodesRootDir diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py index a973f8da52d63aa04ecc3eb4afea47c93419e0c5..0e433e9664ef7ca05652e6fbb03949a9311c0bdf 100644 --- a/tests/pytest/util/dnodes-no-random-fail.py +++ b/tests/pytest/util/dnodes-no-random-fail.py @@ -44,11 +44,11 @@ class TDSimClient: self.path = path def getLogDir(self): - self.logDir = "%s/sim/psim/log" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") return self.logDir def getCfgDir(self): - self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") return self.cfgDir def setTestCluster(self, value): @@ -63,9 +63,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") + self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -129,11 +129,10 @@ class TDDnode: return totalSize def deploy(self): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( - self.path, self.index) + self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") + self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -323,11 +322,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = os.path.join(self.path,"sim","psim") return dnodesRootDir diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 9b653844f880f6f0667c2b96bf9a264c04d5c613..1c527290ec3c0a24b0a0175da104eeca2ffcfdfb 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -41,11 +41,11 @@ class TDSimClient: self.path = path def getLogDir(self): - self.logDir = "%s/sim/psim/log" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") return self.logDir def getCfgDir(self): - self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") return self.cfgDir def setTestCluster(self, value): @@ -60,9 +60,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") + self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -126,11 +126,10 @@ class TDDnode: return totalSize def deploy(self): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( - self.path, self.index) + self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") + self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -320,11 +319,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = os.path.join(self.path,"sim","psim") return dnodesRootDir diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 360843c46c3568836d8e7cd5c8b33a49b3a0e99f..80ab5baa9a8a23132abf75686ccdc6bebb6d8238 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -50,11 +50,11 @@ class TDSimClient: } def getLogDir(self): - self.logDir = "%s/sim/psim/log" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") return self.logDir def getCfgDir(self): - self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") return self.cfgDir def setTestCluster(self, value): @@ -69,9 +69,9 @@ class TDSimClient: tdLog.exit(cmd) def deploy(self, *updatecfgDict): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + self.logDir = os.path.join(self.path,"sim","psim","log") + self.cfgDir = os.path.join(self.path,"sim","psim","cfg") + self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -207,11 +207,10 @@ class TDDnode: self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): - self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) - self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) - self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) - self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( - self.path, self.index) + self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") + self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") cmd = "rm -rf " + self.dataDir if os.system(cmd) != 0: @@ -472,20 +471,25 @@ class TDDnode: tdLog.exit("dnode:%d is not deployed" % (self.index)) if self.valgrind == 0: - if self.asan: - asanDir = "%s/sim/asan/dnode%d.asan" % ( - self.path, self.index) - cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( - binPath, self.cfgDir, asanDir) + if platform.system().lower() == 'windows': + cmd = "mintty -h never %s -c %s" % (binPath, self.cfgDir) else: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) + if self.asan: + asanDir = "%s/sim/asan/dnode%d.asan" % ( + self.path, self.index) + cmd = "nohup %s -c %s > /dev/null 2> %s & " % ( + binPath, self.cfgDir, asanDir) + else: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) else: valgrindCmdline = "valgrind --log-file=\"%s/../log/valgrind.log\" --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"%self.cfgDir - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - + if platform.system().lower() == 'windows': + cmd = "mintty -h never %s %s -c %s" % ( + valgrindCmdline, binPath, self.cfgDir) + else: + cmd = "nohup %s %s -c %s 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) print(cmd) if (self.remoteIP == ""): @@ -568,6 +572,8 @@ class TDDnode: while(processID): if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): killCmd = "kill -INT %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -INT %s > nul 2>&1" % processID os.system(killCmd) onlyKillOnceWindows = 1 time.sleep(1) @@ -635,11 +641,11 @@ class TDDnode: tdLog.exit(cmd) def getDnodeRootDir(self, index): - dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) return dnodeRootDir def getDnodesRootDir(self): - dnodesRootDir = "%s/sim/psim" % (self.path) + dnodesRootDir = os.path.join(self.path,"sim","psim") return dnodesRootDir @@ -789,15 +795,21 @@ class TDDnodes: processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() while(processID): print(processID) - if platform.system().lower() == 'windows': - killCmd = "kill -9 %s > nul 2>&1" % processID - else: - killCmd = "kill -9 %s > /dev/null 2>&1" % processID + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8").strip() + elif platform.system().lower() == 'windows': + psCmd = "for /f %a in ('wmic process where \"name='taosd.exe'\" get processId ^| xargs echo ^| awk '{print $2}' ^&^& echo aa') do @(ps | grep %a | awk '{print $1}' | xargs)" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + while(processID): + print(processID) + killCmd = "kill -9 %s > nul 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8").strip() - else: psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() @@ -809,10 +821,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}' | xargs" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() while(processID): - if platform.system().lower() == 'windows': - killCmd = "kill -9 %s > nul 2>&1" % processID - else: - killCmd = "kill -9 %s > /dev/null 2>&1" % processID + killCmd = "kill -9 %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( diff --git a/tests/pytest/util/gettime.py b/tests/pytest/util/gettime.py index 94eed384789b85b746a523c3a0875bf1f5b711ea..d4a5e18dc9e9fb8f3ee032219d05c8a6d53becfb 100644 --- a/tests/pytest/util/gettime.py +++ b/tests/pytest/util/gettime.py @@ -18,6 +18,9 @@ class GetTime: def get_ms_timestamp(self,ts_str): _ts_str = ts_str + if "+" in _ts_str: + timestamp = datetime.fromisoformat(_ts_str) + return int((timestamp-datetime.fromtimestamp(0,timestamp.tzinfo)).total_seconds())*1000+int(timestamp.microsecond / 1000) if " " in ts_str: p = ts_str.split(" ")[1] if len(p) > 15 : diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index bdf3f20e1546e029dfbf7b41b280e857fb732020..c041282bfce6d5cf1ba6723ddf476a50bc1f2c7a 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -23,6 +23,15 @@ import pandas as pd from util.log import * from util.constant import * +# from datetime import timezone +import time + +def _parse_ns_timestamp(timestr): + dt_obj = datetime.datetime.strptime(timestr[:len(timestr)-3], "%Y-%m-%d %H:%M:%S.%f") + tz = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1e9) + int(dt_obj.microsecond * 1000) + int(timestr[-3:]) + return tz + + def _parse_datetime(timestr): try: return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') @@ -227,28 +236,102 @@ class TDSql: self.checkRowCol(row, col) return self.cursor.istype(col, dataType) + def checkData(self, row, col, data): + if row >= self.queryRows: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row+1, self.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col >= self.queryCols: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, col+1, self.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + self.checkRowCol(row, col) + if self.queryResult[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): - # suppose user want to check nanosecond timestamp if a longer data passed - if (len(data) >= 28): - if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # suppose user want to check nanosecond timestamp if a longer data passed`` + if isinstance(data,str) : + if (len(data) >= 28): + if self.queryResult[row][col] == _parse_ns_timestamp(data): + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{pd.to_datetime(resultData)} == expect:{data}") + tdLog.info("check successfully") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + else: + if self.queryResult[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc): + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return + elif isinstance(data,int): + if len(str(data)) == 16: + precision = 'us' + elif len(str(data)) == 13: + precision = 'ms' + elif len(str(data)) == 19: + precision = 'ns' + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return + success = False + if precision == 'ms': + dt_obj = self.queryResult[row][col] + ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1000) + int(dt_obj.microsecond/1000) + if ts == data: + success = True + elif precision == 'us': + dt_obj = self.queryResult[row][col] + ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1e6) + int(dt_obj.microsecond) + if ts == data: + success = True + elif precision == 'ns': + if data == self.queryResult[row][col]: + success = True + if success: + tdLog.info("check successfully") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return + elif isinstance(data,datetime.datetime): + dt_obj = self.queryResult[row][col] + delt_data = data-datetime.datetime.fromtimestamp(0,data.tzinfo) + delt_result = self.queryResult[row][col] - datetime.datetime.fromtimestamp(0,self.queryResult[row][col].tzinfo) + if delt_data == delt_result: + tdLog.info("check successfully") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + return else: - if self.queryResult[row][col] == _parse_datetime(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") - return + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) if str(self.queryResult[row][col]) == str(data): - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") return elif isinstance(data, float): if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") + else: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) @@ -258,8 +341,7 @@ class TDSql: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) - - tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + tdLog.info("check successfully") # return true or false replace exit, no print out def checkRowColNoExit(self, row, col): @@ -349,7 +431,7 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queryTimes=10): + def execute(self, sql,queryTimes=30): self.sql = sql i=1 while i <= queryTimes: diff --git a/tests/pytest/util/taosadapter.py b/tests/pytest/util/taosadapter.py index 79ea86c5bd820c1327cf844b35eb8800fefdfe60..1f1c38672e130f54478d1fed408f2948d2ec892a 100644 --- a/tests/pytest/util/taosadapter.py +++ b/tests/pytest/util/taosadapter.py @@ -1,4 +1,4 @@ -import socket +import requests from fabric2 import Connection from util.log import * from util.common import * @@ -132,9 +132,9 @@ class TAdapter: tdLog.exit(cmd) def deploy(self, *update_cfg_dict): - self.log_dir = f"{self.path}/sim/dnode1/log" - self.cfg_dir = f"{self.path}/sim/dnode1/cfg" - self.cfg_path = f"{self.cfg_dir}/taosadapter.toml" + self.log_dir = os.path.join(self.path,"sim","dnode1","log") + self.cfg_dir = os.path.join(self.path,"sim","dnode1","cfg") + self.cfg_path = os.path.join(self.cfg_dir,"taosadapter.toml") cmd = f"touch {self.cfg_path}" if os.system(cmd) != 0: @@ -162,7 +162,7 @@ class TAdapter: tdLog.info(f"taosadapter found: {bin_path}") if platform.system().lower() == 'windows': - cmd = f"mintty -h never {bin_path} -c {self.cfg_dir}" + cmd = f"mintty -h never {bin_path} -c {self.cfg_path}" else: cmd = f"nohup {bin_path} -c {self.cfg_path} > /dev/null & " @@ -170,7 +170,7 @@ class TAdapter: self.remote_exec(self.taosadapter_cfg_dict, f"tAdapter.deployed=1\ntAdapter.log_dir={self.log_dir}\ntAdapter.cfg_dir={self.cfg_dir}\ntAdapter.start()") self.running = 1 else: - os.system(f"rm -rf {self.log_dir}/taosadapter*") + os.system(f"rm -rf {self.log_dir}{os.sep}taosadapter*") if os.system(cmd) != 0: tdLog.exit(cmd) self.running = 1 @@ -179,22 +179,19 @@ class TAdapter: time.sleep(0.1) taosadapter_port = self.taosadapter_cfg_dict["port"] - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(3) - try: - res = s.connect_ex((self.remoteIP, taosadapter_port)) - s.shutdown(2) - if res == 0: - tdLog.info(f"the taosadapter has been started, using port:{taosadapter_port}") - else: - tdLog.info(f"the taosadapter do not started!!!") - except socket.error as e: - tdLog.notice("socket connect error!") - finally: - if s: - s.close() - # tdLog.debug("the taosadapter has been started.") - time.sleep(1) + for i in range(5): + ip = 'localhost' + if self.remoteIP != "": + ip = self.remoteIP + url = f'http://{ip}:{taosadapter_port}/-/ping' + try: + r = requests.get(url) + if r.status_code == 200: + tdLog.info(f"the taosadapter has been started, using port:{taosadapter_port}") + break + except Exception: + tdLog.info(f"the taosadapter do not started!!!") + time.sleep(1) def start_taosadapter(self): """ @@ -228,33 +225,36 @@ class TAdapter: def stop(self, force_kill=False): signal = "-9" if force_kill else "-15" - if self.remoteIP: self.remote_exec(self.taosadapter_cfg_dict, "tAdapter.running=1\ntAdapter.stop()") tdLog.info("stop taosadapter") return - toBeKilled = "taosadapter" - - if self.running != 0: + if platform.system().lower() == 'windows': psCmd = f"ps -ef|grep -w {toBeKilled}| grep -v grep | awk '{{print $2}}'" - # psCmd = f"pgrep {toBeKilled}" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - while(processID): - killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) + while(processID): + killCmd = "kill %s %s > nul 2>&1" % (signal, processID) os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - if not platform.system().lower() == 'windows': + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + self.running = 0 + tdLog.debug(f"taosadapter is stopped by kill {signal}") + + else: + if self.running != 0: + psCmd = f"ps -ef|grep -w {toBeKilled}| grep -v grep | awk '{{print $2}}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + while(processID): + killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() port = 6041 fuserCmd = f"fuser -k -n tcp {port} > /dev/null" os.system(fuserCmd) - # for port in range(6030, 6041): - # fuserCmd = f"fuser -k -n tcp {port} > /dev/null" - # os.system(fuserCmd) - - self.running = 0 - tdLog.debug(f"taosadapter is stopped by kill {signal}") + self.running = 0 + tdLog.debug(f"taosadapter is stopped by kill {signal}") diff --git a/tests/requirements.txt b/tests/requirements.txt index c6e27fd3be6b35c268c8f7b7b97da048224561e3..5cdd9e02beed40b73a083041e1b4d043dcd48adb 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,3 +8,4 @@ distro requests pexpect faker +pyopenssl diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 0903095dc98dcf3652546859003fa65fd88e1569..99507ef5c3ae58d786dbfe8f91e72400d471ec3f 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -122,9 +122,11 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, + TTYPE_INSERT_NG, TTYPE_QUERY, }; @@ -187,6 +189,8 @@ CaseCfg gCase[] = { {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -250,7 +254,7 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = 24, + .caseIdx = 26, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, @@ -2191,6 +2195,47 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { } +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + char *buf = "tbnexist"; + code = bpSetTableNameTags(&data, 0, buf, stmt); + if (code == 0){ + printf("!!!taos_stmt_set_tbname expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_bind_param_batch(stmt, data.pBind)) { + printf("!!!taos_stmt_bind_param_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_execute(stmt)) { + printf("!!!taos_stmt_execute expected error not occurred\n"); + exit(1); + } + + destroyData(&data); + + return 0; +} + + + int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2213,6 +2258,10 @@ int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { } void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expected, bool silent) { + if (TTYPE_INSERT_NG == gCurCase->testType) { + return; + } + char sql[255] = "SELECT * FROM "; int32_t rows = 0; diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 6739794cc8e09025ec18ee92100453edce082f77..6d55d8a75f7efe4383c81781165b2246e7a893a0 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -15,8 +15,11 @@ exe: gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) + gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) clean: rm $(ROOT)batchprepare rm $(ROOT)stopquery rm $(ROOT)dbTableRoute + rm $(ROOT)insertSameTs + rm $(ROOT)passwdTest diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c new file mode 100644 index 0000000000000000000000000000000000000000..8a2b0a0390dad3acef1fca93abc89095164bb401 --- /dev/null +++ b/tests/script/api/passwdTest.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +/** + * passwdTest.c + * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + */ + +#include +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +#define nDup 1 +#define nRoot 10 +#define nUser 10 +#define USER_LEN 24 + +void Test(TAOS *taos, char *qstr); +void createUers(TAOS *taos, const char *host, char *qstr); +void passVerTestMulti(const char *host, char *qstr); + +int nPassVerNotified = 0; +TAOS *taosu[nRoot] = {0}; +char users[nUser][USER_LEN] = {0}; + +void __taos_notify_cb(void *param, void *ext, int type) { + switch (type) { + case TAOS_NOTIFY_PASSVER: { + ++nPassVerNotified; + printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + break; + } + default: + printf("%s:%d unknown type:%d\n", __func__, __LINE__, type); + break; + } +} + +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < nDup; ++i) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "success to run: %s\n", command); + } + + taos_free_result(pSql); +} + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + createUers(taos, argv[1], qstr); + passVerTestMulti(argv[1], qstr); + + taos_close(taos); + taos_cleanup(); +} + +void createUers(TAOS *taos, const char *host, char *qstr) { + // users + for (int i = 0; i < nUser; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "CREATE USER %s PASS 'taosdata'", users[i]); + queryDB(taos, qstr); + + taosu[i] = taos_connect(host, users[i], "taosdata", NULL, 0); + if (taosu[i] == NULL) { + printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taosu[i], __taos_notify_cb, users[i], TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb for user:%s since %d\n", users[i], code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb for user:%s\n", users[i]); + } + + // alter pass for users + sprintf(qstr, "alter user %s pass 'taos'", users[i]); + queryDB(taos, qstr); + } +} + +void passVerTestMulti(const char *host, char *qstr) { + // root + TAOS *taos[nRoot] = {0}; + char userName[USER_LEN] = "root"; + + for (int i = 0; i < nRoot; ++i) { + taos[i] = taos_connect(host, "root", "taosdata", NULL, 0); + if (taos[i] == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taos[i], __taos_notify_cb, userName, TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb since %d\n", code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb\n"); + } + } + + queryDB(taos[0], "create database if not exists demo1 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo2 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo3 vgroups 1 minrows 10"); + + queryDB(taos[0], "create table demo1.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo2.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo3.stb (ts timestamp, c1 int) tags(t1 int)"); + + strcpy(qstr, "alter user root pass 'taos'"); + queryDB(taos[0], qstr); + + // calculate the nPassVerNotified for root and users + int nConn = nRoot + nUser; + + for (int i = 0; i < 15; ++i) { + if (nPassVerNotified >= nConn) break; + sleep(1); + } + + // close the taos_conn + for (int i = 0; i < nRoot; ++i) { + taos_close(taos[i]); + printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); + sleep(1); + } + + for (int i = 0; i < nUser; ++i) { + taos_close(taosu[i]); + printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); + sleep(1); + } + + if (nPassVerNotified >= nConn) { + fprintf(stderr, "succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, nConn); + } else { + fprintf(stderr, "failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); + } + // sleep(300); +} \ No newline at end of file diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index a74e6dae737722f04a38f11b3e7c244b57a34441..535b6802f25c1a94549ae9ea5fbb67f08601f258 100755 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -12,6 +12,7 @@ fi today=`date +"%Y%m%d"` TDENGINE_DIR=/root/TDengine JDBC_DIR=/root/taos-connector-jdbc +TAOSKEEPER_DIR=/root/taoskeeper TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log # Color setting @@ -184,6 +185,24 @@ function runJDBCCases() { echo -e "### JDBC test result: $summary ###" | tee -a $TDENGINE_COVERAGE_REPORT } +function runTaosKeeperCases() { + echo "=== Run taoskeeper cases ===" + + cd $TAOSKEEPER_DIR + git checkout -- . + git reset --hard HEAD + git checkout master + git pull + + stopTaosd + stopTaosadapter + + taosd -c /etc/taos >> /dev/null 2>&1 & + taosadapter >> /dev/null 2>&1 & + + go mod tidy && go test -v ./... +} + function runTest() { echo "run Test" @@ -195,6 +214,7 @@ function runTest() { runSimCases runPythonCases runJDBCCases + runTaosKeeperCases stopTaosd cd $TDENGINE_DIR/tests/script @@ -212,7 +232,7 @@ function lcovFunc { lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info # remove exclude paths - if [ "$branch" == "3.0" ]; then + if [ "$branch" == "main" ] ; then lcov --remove coverage.info \ '*/contrib/*' '*/tests/*' '*/test/*' '*/tools/*' '*/libs/sync/*'\ '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c'\ @@ -222,6 +242,8 @@ function lcovFunc { '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' '*/catalog.c'\ '*/tqSnapshot.c' '*/tsdbSnapshot.c''*/metaSnapshot.c' '*/smaSnapshot.c' '*/tqOffsetSnapshot.c'\ '*/vnodeSnapshot.c' '*/metaSnapshot.c' '*/tsdbSnapshot.c' '*/mndGrant.c' '*/mndSnode.c' '*/streamRecover.c'\ + '*/osAtomic.c' '*/osDir.c' '*/osFile.c' '*/osMath.c' '*/osSignal.c' '*/osSleep.c' '*/osString.c' '*/osSystem.c'\ + '*/osThread.c' '*/osTime.c' '*/osTimezone.c' \ --rc lcov_branch_coverage=1 -o coverage.info else lcov --remove coverage.info \ diff --git a/tests/script/sh/bit_and.c b/tests/script/sh/bit_and.c index 2f2e48fdb0882db2a40c9aae019402e031343d13..f3bf71ce946b5b5d62ac6d63de4c2532a799721e 100644 --- a/tests/script/sh/bit_and.c +++ b/tests/script/sh/bit_and.c @@ -1,61 +1,47 @@ -#include -#include #include +#include +#include #include "taosudf.h" +DLL_EXPORT int32_t bit_and_init() { return 0; } -DLL_EXPORT int32_t bit_and_init() { - return 0; -} - -DLL_EXPORT int32_t bit_and_destroy() { - return 0; -} - -DLL_EXPORT int32_t bit_and(SUdfDataBlock* block, SUdfColumn *resultCol) { +DLL_EXPORT int32_t bit_and_destroy() { return 0; } - if (block->numOfCols < 2) { - return TSDB_CODE_UDF_INVALID_INPUT; - } +DLL_EXPORT int32_t bit_and(SUdfDataBlock* block, SUdfColumn* resultCol) { + if (block->numOfCols < 2) { + return TSDB_CODE_UDF_INVALID_INPUT; + } - for (int32_t i = 0; i < block->numOfCols; ++i) { - SUdfColumn* col = block->udfCols[i]; - if (!(col->colMeta.type == TSDB_DATA_TYPE_INT)) { - return TSDB_CODE_UDF_INVALID_INPUT; - } + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT)) { + return TSDB_CODE_UDF_INVALID_INPUT; } + } - SUdfColumnMeta *meta = &resultCol->colMeta; - meta->bytes = 4; - meta->type = TSDB_DATA_TYPE_INT; - meta->scale = 0; - meta->precision = 0; - - - SUdfColumnData *resultData = &resultCol->colData; - - resultData->numOfRows = block->numOfRows; - - for (int32_t i = 0; i < resultData->numOfRows; ++i) { - if (udfColDataIsNull(block->udfCols[0], i)) { - udfColDataSetNull(resultCol, i); - continue; - } - int32_t result = *(int32_t*)udfColDataGetData(block->udfCols[0], i); - int j = 1; - for (; j < block->numOfCols; ++j) { - if (udfColDataIsNull(block->udfCols[j], i)) { - udfColDataSetNull(resultCol, i); - break; - } - - char* colData = udfColDataGetData(block->udfCols[j], i); - result &= *(int32_t*)colData; - } - if (j == block->numOfCols) { - udfColDataSet(resultCol, i, (char*)&result, false); - } + SUdfColumnData* resultData = &resultCol->colData; + for (int32_t i = 0; i < block->numOfRows; ++i) { + if (udfColDataIsNull(block->udfCols[0], i)) { + udfColDataSetNull(resultCol, i); + continue; } - return TSDB_CODE_SUCCESS; + int32_t result = *(int32_t*)udfColDataGetData(block->udfCols[0], i); + int j = 1; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + udfColDataSetNull(resultCol, i); + break; + } + + char* colData = udfColDataGetData(block->udfCols[j], i); + result &= *(int32_t*)colData; + } + if (j == block->numOfCols) { + udfColDataSet(resultCol, i, (char*)&result, false); + } + } + resultData->numOfRows = block->numOfRows; + + return TSDB_CODE_SUCCESS; } diff --git a/tests/script/sh/checkValgrind.bat b/tests/script/sh/checkValgrind.bat new file mode 100644 index 0000000000000000000000000000000000000000..cd80750fde70b434e97113be34785c675f4b1211 --- /dev/null +++ b/tests/script/sh/checkValgrind.bat @@ -0,0 +1,2 @@ +@echo off +echo 0 \ No newline at end of file diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 38e7022edec2c5dae9053e11934a65e108a75b93..fd76b3007874ef5ccdcbd6cacb5681587cec9a9d 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -54,7 +54,7 @@ if %NODE% == 6 set NODE=7600 if %NODE% == 7 set NODE=7700 if %NODE% == 8 set NODE=7800 -rem set "fqdn=" +set "fqdn=localhost" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 88dd43349ed9d49774faafe827b77126286c741d..75957882922a0bcf20dee10d5824f3c488b0697f 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -50,7 +50,7 @@ if %EXEC_OPTON% == start ( goto :finish ) echo check taosd online - tail -n +0 %TAOS_LOG% | grep -q "TDengine initialized successfully" || goto :check_online + tail -n +0 %TAOS_LOG% | grep -E "TDengine initialized successfully|from offline to online" || goto :check_online echo finish goto :finish ) diff --git a/tests/script/sh/l2norm.c b/tests/script/sh/l2norm.c index 8ccdffb8d69d1ba27410ad61a74ee3d7ca156be2..0b7f5bf7f6756c11331cae3140a9bf6b5cba024b 100644 --- a/tests/script/sh/l2norm.c +++ b/tests/script/sh/l2norm.c @@ -16,7 +16,7 @@ DLL_EXPORT int32_t l2norm_destroy() { DLL_EXPORT int32_t l2norm_start(SUdfInterBuf *buf) { *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); - buf->numOfResult = 0; + buf->numOfResult = 1; return 0; } @@ -58,20 +58,11 @@ DLL_EXPORT int32_t l2norm(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInte *(double*)(newInterBuf->buf) = sumSquares; newInterBuf->bufLen = sizeof(double); - - if (interBuf->numOfResult == 0 && numNotNull == 0) { - newInterBuf->numOfResult = 0; - } else { - newInterBuf->numOfResult = 1; - } + newInterBuf->numOfResult = 1; return 0; } DLL_EXPORT int32_t l2norm_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { - if (buf->numOfResult == 0) { - resultData->numOfResult = 0; - return 0; - } double sumSquares = *(double*)(buf->buf); *(double*)(resultData->buf) = sqrt(sumSquares); resultData->bufLen = sizeof(double); diff --git a/tests/script/sh/max_vol.c b/tests/script/sh/max_vol.c new file mode 100644 index 0000000000000000000000000000000000000000..4f9ecd33a7e3471a6502857a81db09470236b059 --- /dev/null +++ b/tests/script/sh/max_vol.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +#define STR_MAX_LEN 256 // inter buffer length + +// init +DLL_EXPORT int32_t max_vol_init() +{ + return 0; +} + +// destory +DLL_EXPORT int32_t max_vol_destroy() +{ + return 0; +} + +// start +DLL_EXPORT int32_t max_vol_start(SUdfInterBuf *buf) +{ + memset(buf->buf, 0, sizeof(float) + STR_MAX_LEN); + // set init value + *((float*)buf->buf) = -10000000; + buf->bufLen = sizeof(float) + STR_MAX_LEN; + buf->numOfResult = 0; + return 0; +} + +DLL_EXPORT int32_t max_vol(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + float maxValue = *(float *)interBuf->buf; + char strBuff[STR_MAX_LEN] = "inter1buf"; + + if (block->numOfCols < 2) + { + return TSDB_CODE_UDF_INVALID_INPUT; + } + + // check data type + for (int32_t i = 0; i < block->numOfCols; ++i) + { + SUdfColumn *col = block->udfCols[i]; + if( i == block->numOfCols - 1) { + // last column is device id , must varchar + if (col->colMeta.type != TSDB_DATA_TYPE_VARCHAR ) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } else { + if (col->colMeta.type != TSDB_DATA_TYPE_FLOAT) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + } + + // calc max voltage + SUdfColumn *lastCol = block->udfCols[block->numOfCols - 1]; + for (int32_t i = 0; i < (block->numOfCols - 1); ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn *col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + char *data = udfColDataGetData(col, j); + float voltage = *(float *)data; + if (voltage > maxValue) { + maxValue = voltage; + char *valData = udfColDataGetData(lastCol, j); + // get device id + char *deviceId = valData + sizeof(uint16_t); + sprintf(strBuff, "%s_(%d,%d)_%f", deviceId, j, i, maxValue); + } + } + } + + *(float*)newInterBuf->buf = maxValue; + strcpy(newInterBuf->buf + sizeof(float), strBuff); + newInterBuf->bufLen = sizeof(float) + strlen(strBuff)+1; + newInterBuf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t max_vol_finish(SUdfInterBuf *buf, SUdfInterBuf *resultData) +{ + char * str = buf->buf + sizeof(float); + // copy to des + char * des = resultData->buf + sizeof(uint16_t); + strcpy(des, str); + + // set binary type len + uint16_t len = strlen(str); + *((uint16_t*)resultData->buf) = len; + + // set buf len + resultData->bufLen = len + sizeof(uint16_t); + // set row count + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/sh/prepare_pyudf.sh b/tests/script/sh/prepare_pyudf.sh new file mode 100755 index 0000000000000000000000000000000000000000..b72f8c9f77f94cf35a6bf1daef93ee773e53acfe --- /dev/null +++ b/tests/script/sh/prepare_pyudf.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set +e + +FILE=/usr/local/lib/libtaospyudf.so +if [ ! -f "$FILE" ]; then + echo "$FILE does not exist." + apt install -y python3 python3-dev python3-venv + /usr/bin/python3 -m venv /udfenv + source /udfenv/bin/activate + pip3 install taospyudf + ldconfig + deactivate +else + echo "show dependencies of $FILE" + ldd $FILE +fi diff --git a/tests/script/sh/pybitand.py b/tests/script/sh/pybitand.py new file mode 100644 index 0000000000000000000000000000000000000000..1df724794ec20c6a133f3800046c897c722a173e --- /dev/null +++ b/tests/script/sh/pybitand.py @@ -0,0 +1,21 @@ +def init(): + pass + +def process(block): + (rows, cols) = block.shape() + result = [] + for i in range(rows): + r = 2 ** 32 - 1 + for j in range(cols): + cell = block.data(i,j) + if cell is None: + result.append(None) + break + else: + r = r & cell + else: + result.append(r) + return result + +def destroy(): + pass diff --git a/tests/script/sh/pyl2norm.py b/tests/script/sh/pyl2norm.py new file mode 100644 index 0000000000000000000000000000000000000000..105f546b7d186f466f4fcad5eca38ecf5ff7b5e1 --- /dev/null +++ b/tests/script/sh/pyl2norm.py @@ -0,0 +1,27 @@ +import json +import math + +def init(): + pass + +def destroy(): + pass + +def start(): + return json.dumps(0.0).encode('utf-8') + +def finish(buf): + sum_squares = json.loads(buf) + result = math.sqrt(sum_squares) + return result + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + sum_squares = json.loads(buf) + + for i in range(rows): + for j in range(cols): + cell = datablock.data(i,j) + if cell is not None: + sum_squares += cell * cell + return json.dumps(sum_squares).encode('utf-8') diff --git a/tests/script/test-win.bat b/tests/script/test-win.bat new file mode 100644 index 0000000000000000000000000000000000000000..66bef2008d168c4a2286a8068d4d196230164fc9 --- /dev/null +++ b/tests/script/test-win.bat @@ -0,0 +1,67 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +echo Windows Taosd Full Test +set /a exitNum=0 +rm -rf failed.txt +set caseFile="win-test-file" +if not "%2" == "" ( + set caseFile="%2" +) +for /F "usebackq tokens=*" %%i in (!caseFile!) do ( + set line=%%i + call :CheckSkipCase %%i + if !skipCase! == false ( + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt || set /a errorlevel=8 + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) + ) +) +exit /b !exitNum! + +:colorEcho +set timeNow=%time% +call :GetTimeSeconds %timeNow% +set time2=%_timeTemp% +set /a interTime=%time2% - %time1% +echo End at %timeNow% , cast %interTime%s +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i +goto :eof + +:GetTimeSeconds +set tt=%1 +set tt=%tt:.= % +set tt=%tt::= % +set tt=%tt: 0= % +set /a index=1 +for %%a in (%tt%) do ( + if !index! EQU 1 ( + set /a hh=%%a + )^ + else if !index! EQU 2 ( + set /a mm=%%a + + )^ + else if !index! EQU 3 ( + set /a ss=%%a + ) + set /a index=index+1 +) +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +goto :eof + +:CheckSkipCase +set skipCase=false +@REM if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true ) +echo %* | grep valgrind && set skipCase=true +:goto eof \ No newline at end of file diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index f4392fbca4727a9a8b2adfba3b24ebf2a5c9ed1a..f20f861bd0d40df47ab9c6e1792651410678f515 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -285,14 +285,12 @@ sql_error alter database db keep -1 print ============== modify minrows sql_error alter database db minrows 8 -sql_error alter database db minrows 200 -sql_error alter database db minrows 11 -sql_error alter database db minrows 8000 -sql_error alter database db minrows 8001 +sql_error alter database db minrows 8000000 +sql_error alter database db minrows 8001000 print ============== modify maxrows -sql_error alter database db maxrows 1000 -sql_error alter database db maxrows 2000 +sql_error alter database db maxrows 10000001 +sql_error alter database db maxrows 20000000 sql_error alter database db maxrows 11 # equal minrows sql_error alter database db maxrows 10 # little than minrows diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 382cc986e241d3108268491ae1b0ca9ef1c51bdb..abba824e827eace92d8e2c84685e07fdfbaa4b8d 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -298,11 +298,11 @@ sql drop database db sql_error create database db MAXROWS -1 sql_error create database db MAXROWS 0 sql_error create database db MAXROWS 199 -sql_error create database db MAXROWS 10001 +sql_error create database db MAXROWS 10000001 sql_error create database db MINROWS -1 sql_error create database db MINROWS 0 sql_error create database db MINROWS 9 -sql_error create database db MINROWS 1001 +sql_error create database db MINROWS 1000001 sql_error create database db MAXROWS 500 MINROWS 1000 print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] diff --git a/tests/script/tsim/dnode/drop_dnode_force.sim b/tests/script/tsim/dnode/drop_dnode_force.sim index 425e5f48b2f278f3ef2f2bad51b25f81b8133fc2..bce51a98887dd4aa9bfe6a4bb444424fdf8b006d 100644 --- a/tests/script/tsim/dnode/drop_dnode_force.sim +++ b/tests/script/tsim/dnode/drop_dnode_force.sim @@ -50,6 +50,21 @@ endi print =============== step2 create database sql create database d1 vgroups 1 replica 3 sql use d1 + +$wt = 0 +stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + sql create table d1.st0 (ts timestamp, i int) tags (j int) sql create table d1.c0 using st0 tags(0) sql create table d1.c1 using st0 tags(1) diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim index 0fa7320ff6cd515452748c6c81e82011eb4f8dcb..ef5001dcee28012ca40d2d09cd933b1e5ee1ec34 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -170,6 +170,20 @@ if $leaderExist != 1 then goto step35 endi +$wt = 0 +stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + print =============== step36: create table sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) diff --git a/tests/script/tsim/insert/delete0.sim b/tests/script/tsim/insert/delete0.sim new file mode 100644 index 0000000000000000000000000000000000000000..56538536439bbf3d7e9937353ee65acaea34fb43 --- /dev/null +++ b/tests/script/tsim/insert/delete0.sim @@ -0,0 +1,161 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database with different precision +sql create database d0 keep 365 +sql create database d1 keep 365 precision 'ms' +sql create database d2 keep 365 precision 'us' +sql create database d3 keep 365 precision 'ns' + +sql select * from information_schema.ins_databases +if $rows != 6 then + return -1 +endi + +print $data00 $data01 $data02 + + +sql create table if not exists d0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d3.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d0.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d1.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d2.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d3.ntb (ts timestamp, c1 int, c2 float, c3 double) + +sql create table d0.ct1 using d0.stb tags(1000) +sql create table d1.ct1 using d1.stb tags(1000) +sql create table d2.ct1 using d2.stb tags(1000) +sql create table d3.ct1 using d3.stb tags(1000) +sql create table d0.ct2 using d0.stb tags(1000) +sql create table d1.ct2 using d1.stb tags(1000) +sql create table d2.ct2 using d2.stb tags(1000) +sql create table d3.ct2 using d3.stb tags(1000) + + +sql insert into d0.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ntb values(now+0s, 10, 2.0, 3.0) + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 2 then + return -1 +endi + +print =============== delete from child table +sql delete from d0.ct1 where ts < now() +sql delete from d1.ct1 where ts < now() +sql delete from d2.ct1 where ts < now() +sql delete from d3.ct1 where ts < now() + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 1 then + return -1 +endi +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 1 then + return -1 +endi + +print =============== delete from super table +sql delete from d0.stb where ts < now() +sql delete from d1.stb where ts < now() +sql delete from d2.stb where ts < now() +sql delete from d3.stb where ts < now() + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 0 then + return -1 +endi + +print =============== delete from normal table +sql delete from d0.ntb where ts < now() +sql delete from d1.ntb where ts < now() +sql delete from d2.ntb where ts < now() +sql delete from d3.ntb where ts < now() + +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/col_arithmetic_operation.sim b/tests/script/tsim/parser/col_arithmetic_operation.sim index 9a2ba34c85e552585770bb42913b8c83ddd58131..a11b99d3d60329f33bd5e191d32ce1af4be772b8 100644 --- a/tests/script/tsim/parser/col_arithmetic_operation.sim +++ b/tests/script/tsim/parser/col_arithmetic_operation.sim @@ -9,7 +9,7 @@ $dbPrefix = ca_db $tbPrefix = ca_tb $stbPrefix = ca_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -18,6 +18,7 @@ $i = 0 $db = $dbPrefix . $i $stb = $stbPrefix . $i +print drop database $db -x step1 sql drop database $db -x step1 step1: sql create database $db diff --git a/tests/script/tsim/parser/col_arithmetic_query.sim b/tests/script/tsim/parser/col_arithmetic_query.sim index b77dcbe4982744a59b61687f70d5a650229361e4..1537db9a8365b0eb104f62ae74367d53bf9a10bd 100644 --- a/tests/script/tsim/parser/col_arithmetic_query.sim +++ b/tests/script/tsim/parser/col_arithmetic_query.sim @@ -4,7 +4,7 @@ $dbPrefix = ca_db $tbPrefix = ca_tb $stbPrefix = ca_stb -$rowNum = 10000 +$rowNum = 1000 $i = 0 $db = $dbPrefix . $i @@ -33,7 +33,7 @@ endi # asc/desc order [d.2] ====================================================== sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0.000000000 then @@ -57,7 +57,7 @@ if $data91 != 1.000000000 then endi sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc; -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 2.200000000 then @@ -151,7 +151,7 @@ sql select top(c1,1) - 88 from $tb # all data types [d.6] ================================================================ sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0.000000000 then @@ -221,7 +221,7 @@ if $data90 != 76.000000000 then return -1 endi -sql select c4 / 99.123 from $tb limit 10 offset 9999; +sql select c4 / 99.123 from $tb limit 10 offset 999; if $rows != 1 then return -1 endi @@ -237,7 +237,7 @@ sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12); # constant column. [d.13]============================================================== sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0 then @@ -261,7 +261,7 @@ endi # column value filter [d.14]=========================================================== sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2 -if $rows != 2000 then +if $rows != 200 then return -1 endi if $data00 != 0 then @@ -338,13 +338,14 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb if $rows != 1 then return -1 endi -if $data00 != 1.800000000 then +print $data00 +if $data00 != 6.500000000 then return -1 endi -if $data01 != 100000.000000000 then +if $data01 != 10000.000000000 then return -1 endi -if $data02 != 200000.000000000 then +if $data02 != 20000.000000000 then return -1 endi @@ -371,7 +372,7 @@ endi if $data00 != 0.000000000 then return -1 endi -if $data01 != 225000.000000000 then +if $data01 != 22500.000000000 then return -1 endi if $data02 != 8.077777778 then @@ -383,7 +384,7 @@ endi if $data04 != 0.444444444 then return -1 endi -if $data05 != 450000.000000000 then +if $data05 != 45000.000000000 then return -1 endi @@ -484,10 +485,10 @@ endi if $data10 != 0.000000000 then return -1 endi -if $data20 != 0.997600000 then +if $data20 != 0.976000000 then return -1 endi -if $data90 != 7.980800000 then +if $data90 != 7.808000000 then return -1 endi @@ -496,7 +497,7 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0; if $rows != 1 then return -1 endi -if $data00 != -449988.000000000 then +if $data00 != -44988.000000000 then return -1 endi @@ -546,7 +547,7 @@ endi # interval query [d.17]=============================================================== sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from ca_stb0 interval(1s) -if $rows != 10000 then +if $rows != 1000 then return -1 endi diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index a9bf8b26ebda178ed789f3119eac7203a6889f9b..84610ad379e614bb50aefb90691fc10e287d8ba2 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -7,7 +7,7 @@ $dbPrefix = sc_db $tbPrefix = sc_tb $stbPrefix = sc_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $loops = 5 $log = 1 diff --git a/tests/script/tsim/parser/groupby.sim b/tests/script/tsim/parser/groupby.sim index 4ee9c530a79c72ccac12a99922af1eeefc7485ed..637b7553fd741d8bf89fce3baf67d817391ea0e5 100644 --- a/tests/script/tsim/parser/groupby.sim +++ b/tests/script/tsim/parser/groupby.sim @@ -7,7 +7,7 @@ $dbPrefix = group_db $tbPrefix = group_tb $mtPrefix = group_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== groupby.sim @@ -88,7 +88,7 @@ if $row != 20 then return -1 endi -if $data00 != 100 then +if $data00 != 10 then return -1 endi @@ -96,7 +96,7 @@ if $data01 != 0 then return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi @@ -186,7 +186,7 @@ if $data04 != 0.00000 then return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi @@ -227,7 +227,7 @@ if $row != 20 then return -1 endi -if $data00 != 800 then +if $data00 != 80 then return -1 endi @@ -235,7 +235,7 @@ if $data01 != 0 then return -1 endi -if $data10 != 800 then +if $data10 != 80 then return -1 endi @@ -243,7 +243,7 @@ if $data11 != 1 then return -1 endi -if $data90 != 800 then +if $data90 != 80 then return -1 endi @@ -297,7 +297,7 @@ if $data00 != $data03 then return -1 endi -if $data01 != @70-01-01 08:01:49.900@ then +if $data01 != @70-01-01 08:01:40.900@ then return -1 endi @@ -305,7 +305,7 @@ if $data02 != @70-01-01 08:01:40.000@ then return -1 endi -if $data07 != 800 then +if $data07 != 80 then return -1 endi @@ -313,7 +313,7 @@ if $data10 != $data13 then return -1 endi -if $data11 != @70-01-01 08:01:49.901@ then +if $data11 != @70-01-01 08:01:40.901@ then return -1 endi @@ -321,7 +321,7 @@ if $data12 != @70-01-01 08:01:40.001@ then return -1 endi -if $data17 != 800 then +if $data17 != 80 then return -1 endi @@ -329,7 +329,7 @@ if $data90 != $data93 then return -1 endi -if $data91 != @70-01-01 08:01:49.909@ then +if $data91 != @70-01-01 08:01:40.909@ then return -1 endi @@ -337,11 +337,11 @@ if $data92 != @70-01-01 08:01:40.009@ then return -1 endi -if $data97 != 800 then +if $data97 != 80 then return -1 endi -if $data95 != 7200 then +if $data95 != 720 then return -1 endi @@ -358,7 +358,7 @@ if $data00 != 0 then return -1 endi -if $data11 != 800 then +if $data11 != 80 then return -1 endi @@ -372,7 +372,7 @@ if $data00 != 0 then return -1 endi -if $data01 != @70-01-01 08:01:49.900@ then +if $data01 != @70-01-01 08:01:40.900@ then return -1 endi @@ -389,13 +389,13 @@ if $data04 != 0 then return -1 endi -if $data06 != 100 then +if $data06 != 10 then return -1 endi sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 order by c4; -if $rows != 10000 then +if $rows != 1000 then return -1 endi @@ -415,18 +415,18 @@ if $data03 != 0 then return -1 endi -sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 1; +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 1; if $rows != 1 then return -1 endi -sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 20 offset 9990; +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 slimit 20 soffset 990; if $rows != 10 then return -1 endi sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4; -if $rows != 10000 then +if $rows != 1000 then return -1 endi @@ -441,32 +441,32 @@ if $rows != 100 then return -1 endi -if $data00 != 100 then +if $data00 != 10 then return -1 endi -if $data01 != 495000 then +if $data01 != 4500 then return -1 endi -if $data02 != 100 then +if $data02 != 10 then return -1 endi -if $data03 != 4950.000000000 then - print expect 4950.000000000 , acutal $data03 +if $data03 != 450.000000000 then + print expect 450.000000000 , acutal $data03 return -1 endi -if $data10 != 100 then +if $data10 != 10 then return -1 endi -if $data11 != 495100 then +if $data11 != 4510 then return -1 endi -if $data13 != 4951.000000000 then +if $data13 != 451.000000000 then return -1 endi @@ -481,19 +481,19 @@ if $rows != 1 then return -1 endi -#if $data00 != 79200.000000000 then +#if $data00 != 2160.000000000 then # return -1 #endi -#if $data01 != @binary99@ then +#if $data01 != @binary27@ then # return -1 #endi -#if $data02 != 99.000000000 then +#if $data02 != 27.000000000 then # return -1 #endi -#if $data03 != 99.000000000 then +#if $data03 != 27.000000000 then # return -1 #endi @@ -503,7 +503,7 @@ if $rows != 100 then return -1 endi -if $data00 != 4851.000000000 then +if $data00 != 441.000000000 then return -1 endi @@ -511,19 +511,19 @@ if $data01 != 0 then return -1 endi -if $data02 != 9900 then +if $data02 != 900 then return -1 endi -if $data03 != 4950.000000000 then +if $data03 != 450.000000000 then return -1 endi -if $data04 != 2886.607004772 then +if $data04 != 287.228132327 then return -1 endi -if $data10 != 4852.000000000 then +if $data10 != 442.000000000 then return -1 endi @@ -531,15 +531,15 @@ if $data11 != 1 then return -1 endi -if $data12 != 9901 then +if $data12 != 901 then return -1 endi -if $data13 != 4951.000000000 then +if $data13 != 451.000000000 then return -1 endi -if $data14 != 2886.607004772 then +if $data14 != 287.228132327 then return -1 endi diff --git a/tests/script/tsim/parser/interp.sim b/tests/script/tsim/parser/interp.sim index e6512a22d79da5ca5970665f08e369e582eed361..f74a2dc6243ad6d2bf7acf3a5f57d80df3d53cbb 100644 --- a/tests/script/tsim/parser/interp.sim +++ b/tests/script/tsim/parser/interp.sim @@ -7,7 +7,7 @@ $dbPrefix = intp_db $tbPrefix = intp_tb $stbPrefix = intp_stb $tbNum = 4 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index 154316a03f694d7ea375f1cc7d3cc4ad903cd9b9..a40a75f50c54cb9ffdfed24edd2a7e03e0c13e5c 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -7,7 +7,7 @@ $dbPrefix = join_m_db $tbPrefix = join_tb $mtPrefix = join_mt $tbNum = 3 -$rowNum = 20000 +$rowNum = 2000 $totalNum = $tbNum * $rowNum print =============== join_manyBlocks.sim @@ -78,8 +78,8 @@ print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; print $row -if $row != 60000 then - print expect 60000, actual: $row +if $row != 6000 then + print expect 6000, actual: $row return -1 endi diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 9a41a9f5aa6de119eddfc7cf31eeddccacd1fe5e..3f1a29d928eed421ee23aab79444495ecbeed6f2 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -54,6 +54,7 @@ sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) run tsim/parser/last_cache_query.sim +sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index c35617fc4fc64cedbe667a4153bac1a9064f4a91..bae5eba7a4395099ad50105b6259cc69a7b5947e 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -7,7 +7,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -55,6 +55,7 @@ print ====== tables created run tsim/parser/limit1_tb.sim run tsim/parser/limit1_stb.sim +sql flush database $db print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim index be0963c0fdf3b15750512715b8fa752184436df6..731a218de5d11f11b623a8348344597fb84f78b8 100644 --- a/tests/script/tsim/parser/limit1_stb.sim +++ b/tests/script/tsim/parser/limit1_stb.sim @@ -4,7 +4,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -453,13 +453,14 @@ endi ### [TBASE-361] $offset = $rowNum / 2 $offset = $offset + 1 +print === select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset sql select _wstart, max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset $val = $rowNum - $offset if $rows != $val then print expect $val, actual:$rows return -1 endi -if $data00 != @18-10-22 02:30:00.000@ then +if $data00 != @18-09-20 20:30:00.000@ then return -1 endi if $data01 != 1 then @@ -483,6 +484,7 @@ if $rows != 2 then return -1 endi +print === select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) limit 1 offset 0 sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 5 and c1 > 0 and c2 < 9 and c3 > 1 and c4 < 7 and c5 > 4 partition by t1 interval(5m) limit 1 offset 0 if $rows != 3 then return -1 diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 0ba82654187ab773dd0d84d488e3e1b2e5f50381..e6407e3024e4747f591570689e2ce77c031158df 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -4,7 +4,7 @@ $dbPrefix = lm1_db $tbPrefix = lm1_tb $stbPrefix = lm1_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $ts0 = 1537146000000 $delta = 600000 @@ -13,7 +13,7 @@ $i = 0 $db = $dbPrefix . $i $stb = $stbPrefix . $i -print ====== use db +print ====== use $db sql use $db ##### select from table @@ -664,9 +664,9 @@ endi if $data21 != 4.027681991 then return -1 endi - +print select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m) sql select _wstart, count(c1), count(c2), count(c3), count(c4), count(c5), count(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(27m) -if $rows != 3704 then +if $rows != 371 then return -1 endi if $data01 != 2 then @@ -994,11 +994,14 @@ endi $offset = $rowNum / 10 $offset = $offset * 3 $offset = $offset - 1 + +print === select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset sql select * from $tb where c1 < 5 and c1 > 1 order by ts asc limit 3 offset $offset if $rows != 1 then return -1 endi -if $data00 != @18-11-25 18:40:00.000@ then +print $data00 +if $data00 != @18-09-24 06:40:00.000@ then return -1 endi if $data01 != 4 then @@ -1012,19 +1015,19 @@ sql select * from $tb order by ts desc limit 5 if $rows != 5 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then +if $data00 != @18-09-24 07:30:00.000@ then return -1 endi if $data01 != 9 then return -1 endi -if $data10 != @18-11-25 19:20:00.000@ then +if $data10 != @18-09-24 07:20:00.000@ then return -1 endi if $data12 != 8 then return -1 endi -if $data20 != @18-11-25 19:10:00.000@ then +if $data20 != @18-09-24 07:10:00.000@ then return -1 endi if $data23 != 7.00000 then @@ -1048,7 +1051,7 @@ endi if $data39 != nchar6 then return -1 endi -if $data40 != @18-11-25 18:50:00.000@ then +if $data40 != @18-09-24 06:50:00.000@ then return -1 endi @@ -1056,19 +1059,19 @@ sql select * from $tb order by ts desc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data00 != @18-11-25 19:20:00.000@ then +if $data00 != @18-09-24 07:20:00.000@ then return -1 endi if $data01 != 8 then return -1 endi -if $data10 != @18-11-25 19:10:00.000@ then +if $data10 != @18-09-24 07:10:00.000@ then return -1 endi if $data12 != 7 then return -1 endi -if $data20 != @18-11-25 19:00:00.000@ then +if $data20 != @18-09-24 07:00:00.000@ then return -1 endi if $data23 != 6.00000 then @@ -1092,12 +1095,13 @@ endi if $data39 != nchar5 then return -1 endi -if $data40 != @18-11-25 18:40:00.000@ then +if $data40 != @18-09-24 06:40:00.000@ then return -1 endi $offset = $rowNum $offset = $offset - 2 +print ==== select * from $tb order by ts desc limit 5 offset $offset sql select * from $tb order by ts desc limit 5 offset $offset if $rows != 2 then return -1 @@ -1119,16 +1123,16 @@ sql select * from $tb where c1 < 8 order by ts desc limit 3 offset 2 if $rows != 3 then return -1 endi -if $data00 != @18-11-25 18:50:00.000@ then +if $data00 != @18-09-24 06:50:00.000@ then return -1 endi if $data01 != 5 then return -1 endi -if $data10 != @18-11-25 18:40:00.000@ then +if $data10 != @18-09-24 06:40:00.000@ then return -1 endi -if $data20 != @18-11-25 18:30:00.000@ then +if $data20 != @18-09-24 06:30:00.000@ then return -1 endi if $data12 != 4 then diff --git a/tests/script/tsim/parser/nestquery.sim b/tests/script/tsim/parser/nestquery.sim index 2a363de43df0f774cbbd6b163c3bfd550f10ea1d..c2189d0bddbe91d001ccca378214935df317f2a8 100644 --- a/tests/script/tsim/parser/nestquery.sim +++ b/tests/script/tsim/parser/nestquery.sim @@ -8,7 +8,7 @@ print ======================== dnode1 start $dbPrefix = nest_db $tbPrefix = nest_tb $mtPrefix = nest_mt -$tbNum = 10 +$tbNum = 3 $rowNum = 10000 $totalNum = $tbNum * $rowNum @@ -23,7 +23,7 @@ sql create database if not exists $db sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -$half = $tbNum / 2 +$half = 2 $i = 0 while $i < $half @@ -69,7 +69,7 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname) if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != 4 then return -1 endi @@ -77,7 +77,7 @@ sql select count(*) from (select count(*) from nest_mt0 partition by tbname inte if $rows != 1 then return -1 endi -if $data00 != 170 then +if $data00 != 68 then return -1 endi @@ -85,7 +85,7 @@ sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname inte if $rows != 1 then return -1 endi -if $data00 != 100000 then +if $data00 != 40000 then return -1 endi diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index 669ff3a179c67bce6970868b1fac1bbefccc9d0f..2d99b0a296621ca0b29f908dba8714afde47076c 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -7,7 +7,7 @@ $dbPrefix = group_db $tbPrefix = group_tb $mtPrefix = group_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== projection_limit_offset.sim @@ -81,199 +81,199 @@ $ts2 = $tb2 . .ts sql select ts from group_mt0 print $rows -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 0; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 0; if $rows != 4008 then print expect 4008, actual:$rows return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 1; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 1; if $rows != 4007 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 101; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 101; print $rows if $rows != 3907 then return -1 endi -if $data00 != @70-01-01 08:01:43.101@ then +if $data00 != @70-01-01 08:01:40.101@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 902; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 902; if $rows != 3106 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 400; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 400; if $rows != 3608 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 8000 offset 4007; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 8000 offset 4007; if $rows != 1 then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' limit 2000 offset 4008; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' limit 2000 offset 4008; if $rows != 0 then return -1 endi #==================================order by desc, multi vnode, limit/offset=================================== -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 0; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 0; if $rows != 4008 then return -1 endi -if $data00 != @70-01-01 08:01:43.500@ then +if $data00 != @70-01-01 08:01:40.500@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 1; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 1; if $rows != 4007 then return -1 endi -if $data00 != @70-01-01 08:01:43.500@ then +if $data00 != @70-01-01 08:01:40.500@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 101; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 101; print $rows if $rows != 3907 then return -1 endi -if $data00 != @70-01-01 08:01:43.488@ then +if $data00 != @70-01-01 08:01:40.488@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 902; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 902; if $rows != 3106 then return -1 endi -if $data00 != @70-01-01 08:01:43.388@ then +if $data00 != @70-01-01 08:01:40.388@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 400; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 400; if $rows != 3608 then return -1 endi -if $data00 != @70-01-01 08:01:43.450@ then +if $data00 != @70-01-01 08:01:40.450@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 8000 offset 4007; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 8000 offset 4007; if $rows != 1 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.500' order by ts desc limit 2000 offset 4008; +sql select ts from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.500' order by ts desc limit 2000 offset 4008; if $rows != 0 then return -1 endi #=================================single value filter====================================== -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 0; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 0; if $row != 8 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 1; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 1; if $row != 7 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 2; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 2; if $row != 6 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 4; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 4; if $row != 4 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 7; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 7; if $row != 1 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 8; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 8; if $row != 0 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts asc limit 10 offset 9; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts asc limit 10 offset 9; if $row != 0 then return -1 endi #===============================single value filter, order by desc============================ -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 0; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 0; if $row != 8 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 1; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 1; if $row != 7 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 2; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 2; if $row != 6 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 4; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 4; if $row != 4 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 7; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 7; if $row != 1 then return -1 endi -if $data00 != @70-01-01 08:01:43.000@ then +if $data00 != @70-01-01 08:01:40.000@ then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 8; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 8; if $row != 0 then return -1 endi -sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:43' and ts<='1970-1-1 8:1:43.00' order by ts desc limit 10 offset 9; +sql select ts,tbname from group_mt0 where ts>='1970-1-1 8:1:40' and ts<='1970-1-1 8:1:40.00' order by ts desc limit 10 offset 9; if $row != 0 then return -1 endi #[tbase-695] -sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000 +sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:40.500' and c1<99999999 limit 10000 offset 500 print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 -if $row != 35000 then +if $row != 3500 then return -1 endi @@ -301,7 +301,7 @@ print tbase-722 sql select spread(ts) from group_tb0; print $data00 -if $data00 != 9999.000000000 then +if $data00 != 999.000000000 then return -1 endi diff --git a/tests/script/tsim/parser/selectResNum.sim b/tests/script/tsim/parser/selectResNum.sim index 43365b2af2c8fe52a3ce3e36d057ba9dcdc25a5d..11310782402b8e98fadcbaa9bf740c806c52443a 100644 --- a/tests/script/tsim/parser/selectResNum.sim +++ b/tests/script/tsim/parser/selectResNum.sim @@ -7,10 +7,10 @@ $dbPrefix = sc_db $tbPrefix = sc_tb $stbPrefix = sc_stb $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum $loops = 200000 -$log = 10000 +$log = 1000 $ts0 = 1537146000000 $delta = 600000 print ========== selectResNum.sim diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 45a49fbc4e28662499ef71bcb1de9859e944b57d..1cb4cb5340f993ee25b4f73e8b076375ea28a445 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -7,7 +7,7 @@ $dbPrefix = sliding_db $tbPrefix = sliding_tb $mtPrefix = sliding_mt $tbNum = 8 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== sliding.sim @@ -88,25 +88,25 @@ $ts1 = $tb1 . .ts $ts2 = $tb2 . .ts print ===============================interval_sliding query -sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s); +sql select _wstart, count(*) from sliding_tb0 interval(3s) sliding(3s); if $row != 10 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi -if $data10 != @00-01-01 00:00:30.000@ then +if $data10 != @00-01-01 00:00:03.000@ then return -1 endi -if $data11 != 1000 then +if $data11 != 100 then return -1 endi sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a); -if $row != 10000 then +if $row != 1000 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then @@ -123,10 +123,10 @@ if $data91 != 0.000000000 then endi sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc; -if $row != 10000 then +if $row != 1000 then return -1 endi -if $data00 != @00-01-01 00:04:59.970@ then +if $data00 != @00-01-01 00:00:29.970@ then return -1 endi if $data01 != 0.000000000 then @@ -141,7 +141,7 @@ endi if $data04 != 99 then return -1 endi -if $data90 != @00-01-01 00:04:59.700@ then +if $data90 != @00-01-01 00:00:29.700@ then return -1 endi if $data91 != 0.000000000 then @@ -157,41 +157,41 @@ if $data94 != 90 then return -1 endi -sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc; +sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(3s) sliding(1s) order by _wstart asc; if $row != 32 then return -1 endi -if $data00 != @99-12-31 23:59:40.000@ then - print expect 12-31 23:59:40.000, actual: $data00 +if $data00 != @99-12-31 23:59:58.000@ then + print expect 12-31 23:59:58.000, actual: $data00 return -1 endi -if $data01 != 334 then +if $data01 != 34 then return -1 endi if $data02 != 33 then return -1 endi -sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc; +sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc; if $row != 2 then return -1 endi if $data04 != 99 then return -1 endi -if $data01 != 999 then +if $data01 != 99 then return -1 endi -if $data02 != 28.837977152 then +if $data02 != 28.577380332 then return -1 endi #interval offset + limit -sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990; +sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 90; if $row != 10 then return -1 endi -if $data00 != @00-01-01 00:04:30.270@ then +if $data00 != @00-01-01 00:00:27.270@ then return -1 endi if $data01 != 1 then @@ -203,7 +203,7 @@ endi if $data03 != 0.000000000 then return -1 endi -if $data90 != @00-01-01 00:04:30.000@ then +if $data90 != @00-01-01 00:00:27.000@ then return -1 endi if $data91 != 1 then @@ -217,43 +217,43 @@ if $data93 != 0.000000000 then endi #interval offset test -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(3s) order by _wstart asc limit 100 offset 1; if $row != 9 then return -1 endi -if $data00 != @00-01-01 00:00:30.000@ then +if $data00 != @00-01-01 00:00:03.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi if $data02 != 99 then return -1 endi -if $data80 != @00-01-01 00:04:30.000@ then +if $data80 != @00-01-01 00:00:27.000@ then return -1 endi -if $data81 != 1000 then +if $data81 != 100 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:4' interval(3s) sliding(3s) order by _wstart asc limit 100 offset 0; if $row != 2 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 999 then +if $data01 != 99 then return -1 endi if $data02 != 99 then return -1 endi -if $data03 != 28.837977152 then +if $data03 != 28.577380332 then return -1 endi -if $data10 != @00-01-01 00:00:30.000@ then +if $data10 != @00-01-01 00:00:03.000@ then return -1 endi if $data11 != 34 then @@ -266,14 +266,14 @@ if $data13 != 9.810708435 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 1; if $row != 15 then return -1 endi if $data00 != @00-01-01 00:00:00.000@ then return -1 endi -if $data01 != 1000 then +if $data01 != 100 then return -1 endi if $data02 != 99 then @@ -282,85 +282,85 @@ endi if $data03 != 28.866070048 then return -1 endi -if $data90 != @00-01-01 00:03:00.000@ then +if $data90 != @00-01-01 00:00:18.000@ then return -1 endi -if $data91 != 1000 then +if $data91 != 100 then return -1 endi if $data92 != 99 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 5; if $row != 11 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 6; if $row != 10 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 7; if $row != 9 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 8; if $row != 8 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 9; if $row != 7 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 10; if $row != 6 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 11; if $row != 5 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 12; if $row != 4 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 13; if $row != 3 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 14; if $row != 2 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 15; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(3s) sliding(2s) order by _wstart asc limit 100 offset 16; if $row != 0 then return -1 endi -sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc; +sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) order by _wstart desc; if $row != 10 then return -1 endi -#00-01-01 00:04:30.000| 10| 0| 0.000000000| 0.000000000| -if $data00 != @00-01-01 00:04:30.000@ then +#00-01-01 00:00:27.000 | 1 | 0 | 0.000000000 | 0.000000000 | +if $data00 != @00-01-01 00:00:27.000@ then return -1 endi -if $data01 != 10 then +if $data01 != 1 then return -1 endi if $data02 != 0 then @@ -370,18 +370,18 @@ if $data03 != 0.000000000 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15; +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 14; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16; +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(3s) sliding(2s) order by _wstart desc limit 1 offset 15; if $row != 0 then return -1 endi sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2; -if $data00 != @00-01-01 00:04:59.910@ then +if $data00 != @00-01-01 00:00:29.910@ then return -1 endi diff --git a/tests/script/tsim/parser/slimit1_query.sim b/tests/script/tsim/parser/slimit1_query.sim index 9a27d5523b04c3466e2a6223dd10695f05e9662a..1167fe0b3dba3d25349fdd2401ece10f0ea09afa 100644 --- a/tests/script/tsim/parser/slimit1_query.sim +++ b/tests/script/tsim/parser/slimit1_query.sim @@ -70,7 +70,7 @@ endi ### empty result set sql select count(*) from stb partition by t2,t1 order by t2 asc slimit 0 soffset 0 -if $rows != 9 then +if $rows != 0 then return -1 endi diff --git a/tests/script/tsim/parser/slimit_limit.sim b/tests/script/tsim/parser/slimit_limit.sim new file mode 100644 index 0000000000000000000000000000000000000000..546ef576871b00c14e7a90d7244c2a4f40dc70c7 --- /dev/null +++ b/tests/script/tsim/parser/slimit_limit.sim @@ -0,0 +1,113 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 1; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql create table tba3 using sta tags(3, 3, 3); +sql create table tba4 using sta tags(4, 4, 4); +sql create table tba5 using sta tags(5, 5, 5); +sql create table tba6 using sta tags(6, 6, 6); +sql create table tba7 using sta tags(7, 7, 7); +sql create table tba8 using sta tags(8, 8, 8); +sql create index index1 on sta (t2); +sql insert into tba1 values ('2022-04-26 15:15:01', 1, "a"); +sql insert into tba1 values ('2022-04-26 15:15:02', 11, "a"); +sql insert into tba2 values ('2022-04-26 15:15:01', 2, "a"); +sql insert into tba2 values ('2022-04-26 15:15:02', 22, "a"); +sql insert into tba3 values ('2022-04-26 15:15:01', 3, "a"); +sql insert into tba4 values ('2022-04-26 15:15:01', 4, "a"); +sql insert into tba5 values ('2022-04-26 15:15:01', 5, "a"); +sql insert into tba6 values ('2022-04-26 15:15:01', 6, "a"); +sql insert into tba7 values ('2022-04-26 15:15:01', 7, "a"); +sql insert into tba8 values ('2022-04-26 15:15:01', 8, "a"); + +sql select t1,count(*) from sta group by t1 limit 1; +if $rows != 8 then + return -1 +endi +sql select t1,count(*) from sta group by t1 slimit 1; +if $rows != 1 then + return -1 +endi +sql select f1,count(*) from sta group by f1 limit 1; +if $rows != 10 then + return -1 +endi +sql select f1,count(*) from sta group by f1 slimit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by t1, f1 limit 1; +if $rows != 10 then + return -1 +endi +sql select t1,f1,count(*) from sta group by t1, f1 slimit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by f1, t1 limit 1; +if $rows != 10 then + return -1 +endi +sql select t1,f1,count(*) from sta group by f1, t1 slimit 1; +if $rows != 1 then + return -1 +endi + +sql select t1,count(*) from sta group by t1 order by t1 limit 1; +if $rows != 1 then + return -1 +endi +sql select t1,count(*) from sta group by t1 order by t1 slimit 1; +if $rows != 8 then + return -1 +endi +sql select f1,count(*) from sta group by f1 order by f1 limit 1; +if $rows != 1 then + return -1 +endi +sql select f1,count(*) from sta group by f1 order by f1 slimit 1; +if $rows != 10 then + return -1 +endi +sql select t1,f1,count(*) from sta group by t1, f1 order by t1,f1 limit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by t1, f1 order by t1,f1 slimit 1; +if $rows != 10 then + return -1 +endi +sql select t1,f1,count(*) from sta group by f1, t1 order by f1,t1 limit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by f1, t1 order by f1,t1 slimit 1; +if $rows != 10 then + return -1 +endi + +sql select t1,count(*) from sta group by t1 slimit 1 limit 1; +if $rows != 1 then + return -1 +endi +sql select f1,count(*) from sta group by f1 slimit 1 limit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by t1, f1 slimit 1 limit 1; +if $rows != 1 then + return -1 +endi +sql select t1,f1,count(*) from sta group by f1, t1 slimit 1 limit 1; +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/table_merge_limit.sim b/tests/script/tsim/parser/table_merge_limit.sim new file mode 100644 index 0000000000000000000000000000000000000000..7d67b6d3a270c8587c85123963b564c82ea18762 --- /dev/null +++ b/tests/script/tsim/parser/table_merge_limit.sim @@ -0,0 +1,47 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +$dbPrefix = m_fl_db +$tbPrefix = m_fl_tb +$mtPrefix = m_fl_mt +$tbNum = 2 +$rowNum = 513 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== fill.sim +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db vgroups 1 +sql use $db +sql create table $mt (ts timestamp, c1 int) tags(tgcol int) + +$i = 0 +$ts = $ts0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $xs = $x * $delta + $ts = $ts0 + $xs + sql insert into $tb values ( $ts , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql select * from $mt order by ts limit 10 +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 9e7c6f77cc90f9612e9d258880322c218892c828..dee5da96e8488a14cc615d7ab3944ccf6558f93e 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -8,7 +8,7 @@ $tbPrefix = union_tb $tbPrefix1 = union_tb_ $mtPrefix = union_mt $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== union.sim @@ -65,7 +65,7 @@ sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c $j = 0 $t = 1578203484000 -$rowNum = 1000 +$rowNum = 100 $tbNum = 5 $i = 0 @@ -117,7 +117,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l # sql with parenthese sql (select c1 from union_tb0) -if $rows != 10000 then +if $rows != 1000 then return -1 endi if $data00 != 0 then @@ -187,7 +187,7 @@ endi if $data10 != 1 then return -1 endi -if $data20 != 4950000 then +if $data20 != 495000 then return -1 endi @@ -202,7 +202,7 @@ endi if $data10 != 1 then return -1 endi -if $data20 != 495000 then +if $data20 != 49500 then return -1 endi @@ -211,7 +211,7 @@ sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1 if $rows != 11 then return -1 endi -if $data00 != 10000 then +if $data00 != 1000 then return -1 endi if $data10 != 9 then @@ -227,7 +227,7 @@ endi print ===========================================tags union # two super table tag union, limit is not active during retrieve tags query sql (select t1 from union_mt0) union all (select t1 from union_mt0) -if $rows != 200000 then +if $rows != 20000 then return -1 endi @@ -248,7 +248,7 @@ sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (sele if $rows != 2 then return -1 endi -#if $data00 != 495000 then +#if $data00 != 49500 then # return -1 #endi @@ -272,7 +272,7 @@ endi # multi-vnode projection query sql (select c1 from union_mt0) union all select c1 from union_mt0; -if $rows != 200000 then +if $rows != 20000 then return -1 endi @@ -330,10 +330,10 @@ sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a if $rows != 2 then return -1 endi -if $data00 != 495000 then +if $data00 != 49500 then return -1 endi -if $data10 != 495000 then +if $data10 != 49500 then return -1 endi diff --git a/tests/script/tsim/parser/where.sim b/tests/script/tsim/parser/where.sim index c1660883b6189a74c38b59b95c4c4fc1fcad8a90..8ab47dacafe9ae39f49db82d045574b2a3448884 100644 --- a/tests/script/tsim/parser/where.sim +++ b/tests/script/tsim/parser/where.sim @@ -7,7 +7,7 @@ $dbPrefix = wh_db $tbPrefix = wh_tb $mtPrefix = wh_mt $tbNum = 10 -$rowNum = 10000 +$rowNum = 1000 $totalNum = $tbNum * $rowNum print =============== where.sim @@ -62,7 +62,7 @@ sql select count(*) from $tb where c1<10 and c1<>2 if $rows != 1 then return -1 endi -if $data00 != 900 then +if $data00 != 90 then return -1 endi @@ -308,9 +308,9 @@ if $row != 0 then endi sql select * from wh_mt0 where c3 = 1; -print $rows -> 1000 +print $rows -> 100 print $data00 $data01 $data02 -if $row != 1000 then +if $row != 100 then return -1 endi diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index eb6b102bd9a8d85352dc1312d906fcbabb22b960..2a8f07178804a166fc9e4c0e2163840825bd4459 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start sql connect print ======== step1 -sql create database db1 vgroups 3; +sql create database db1 vgroups 3 cachesize 10 cachemodel 'both'; sql use db1; sql select * from information_schema.ins_databases; sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); @@ -60,6 +60,15 @@ sql explain verbose true select * from information_schema.ins_stables where db_n sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0; sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0; sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s); +sql explain verbose true select distinct tbname, table_name from information_schema.ins_tables; +sql explain verbose true select diff(f1) as f11 from tb1 order by f11; +sql explain verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear); +sql explain verbose true select count(*) from st1 partition by tbname; +sql explain verbose true select count(*) from information_schema.ins_tables group by stable_name; +sql explain verbose true select last(*) from st1; +sql explain verbose true select last_row(*) from st1; +sql explain verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev); +sql explain verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10; print ======== step4 sql explain analyze select ts from st1 where -2; @@ -96,6 +105,13 @@ sql explain analyze verbose true select * from information_schema.ins_stables wh sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0; sql explain analyze verbose true select count(f1) from st1 group by tbname; sql explain analyze verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0; +sql explain analyze verbose true select diff(f1) as f11 from tb1 order by f11; +sql explain analyze verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear); +sql explain analyze verbose true select count(*) from information_schema.ins_tables group by stable_name; +sql explain analyze verbose true select last(*) from st1; +sql explain analyze verbose true select last_row(*) from st1; +sql explain analyze verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev); +sql explain analyze verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10; #not pass case #sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa; diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index f4d95df083ca9f80ad5ed68d71992689aa7f66be..0d796af0a0ed1d154667c0d647131829625b34d3 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -212,10 +212,10 @@ print ===> rows2: $data20 $data21 $data22 $data23 $data24 if $rows != 3 then return -1 endi -if $data01 != 2 then +if $data01 != 4 then return -1 endi -if $data04 != 2 then +if $data04 != 4 then return -1 endi diff --git a/tests/script/tsim/query/join_interval.sim b/tests/script/tsim/query/join_interval.sim new file mode 100644 index 0000000000000000000000000000000000000000..14994a5cc1b77909ebcc40fae93880f9fda065f0 --- /dev/null +++ b/tests/script/tsim/query/join_interval.sim @@ -0,0 +1,42 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step create databases +sql create database d1 +sql create database d2 +sql create table d1.t1(ts timestamp, i int) tags(t int); +sql create table d2.t1(ts timestamp, i int); +sql insert into d1.t11 using d1.t1 tags(1) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t12 using d1.t1 tags(2) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t13 using d1.t1 tags(3) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) + +sql insert into d2.t1 values(1500000000000,0)(1500000000001,1)(1500000000002,2) + +sql select _wstart,_wend,count((a.ts)),count(b.ts) from d1.t1 a, d2.t1 b where a.ts is not null and a.ts = b.ts interval(1a) ; +if $data02 != 3 then + return -1 +endi + +if $data03 != 3 then + return -1 +endi + +if $data12 != 3 then + return -1 +endi + +if $data13 != 3 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +if $data23 != 3 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/multi_order_by.sim b/tests/script/tsim/query/multi_order_by.sim new file mode 100644 index 0000000000000000000000000000000000000000..7b7f2abe365333b2f63a5825ed7945450bddc0d5 --- /dev/null +++ b/tests/script/tsim/query/multi_order_by.sim @@ -0,0 +1,57 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql create table t(ts timestamp, f int); +sql insert into t values(now,0)(now+1s, 1)(now+2s, 2)(now+3s,3)(now+4s,4)(now+5s,5)(now+6s,6)(now+7s,7)(now+8s,8)(now+9s,9) +sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts; +print $data01 $data11 $data21 +if $data01 != 5 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data21 != 7 then + return -1 +endi +sql select * from (select * from t order by ts limit 3 offset 2) order by ts desc; +print $data01 $data11 $data21 +if $data01 != 4 then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data21 != 2 then + return -1 +endi +sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts desc; +print $data01 $data11 $data21 +if $data01 != 7 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data21 != 5 then + return -1 +endi +sql select * from (select * from t order by ts limit 3 offset 2) order by ts; +print $data01 $data11 $data21 +if $data01 != 2 then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data21 != 4 then + return -1 +endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/nullColSma.sim b/tests/script/tsim/query/nullColSma.sim new file mode 100644 index 0000000000000000000000000000000000000000..886274e7e59ca22a557a8ed861585592d5402fbb --- /dev/null +++ b/tests/script/tsim/query/nullColSma.sim @@ -0,0 +1,139 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +$dbPrefix = m_in_db +$tbPrefix = m_in_tb +$mtPrefix = m_in_mt +$tbNum = 1 +$rowNum = 200 +$totalNum = 400 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database if exists $db +sql create database $db vgroups 1 maxrows 200 minrows 10; +sql use $db +sql create table $mt (ts timestamp, f1 int, f2 float) TAGS(tgcol int) + +print ====== start create child tables and insert data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $cc = $x * 1 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , NULL , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +$i = 1 +$tb = $tbPrefix . $i +sql create table $tb using $mt tags( $i ) + +$x = 0 +while $x < $rowNum + $cc = $x * 1 + $ms = 1601481600000 + $cc + + sql insert into $tb values ($ms , $x , NULL ) + $x = $x + 1 +endw + +sql flush database $db + +print =============== step2 +$i = 0 +$tb = $tbPrefix . $i +sql select max(f1) from $tb +if $rows != 1 then + return -1 +endi +if $data00 != NULL then + return -1 +endi + +$i = 1 +$tb = $tbPrefix . $i +sql select max(f2) from $tb +if $rows != 1 then + return -1 +endi +if $data00 != NULL then + return -1 +endi + +$rowNum = 10 + +print ====== insert more data +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + while $x < $rowNum + $cc = $x * 1 + $ms = 1601481700000 + $cc + + sql insert into $tb values ($ms , $x , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +$i = 1 +$tb = $tbPrefix . $i +$x = 0 +while $x < $rowNum + $cc = $x * 1 + $ms = 1601481700000 + $cc + + sql insert into $tb values ($ms , $x , $x ) + $x = $x + 1 +endw + +sql flush database $db + +print =============== step3 +$i = 0 +$tb = $tbPrefix . $i +sql select max(f1) from $tb +if $rows != 1 then + return -1 +endi +if $data00 != 9 then + return -1 +endi + +$i = 1 +$tb = $tbPrefix . $i +sql select max(f2) from $tb +if $rows != 1 then + return -1 +endi +if $data00 != 9.00000 then + print $data00 + return -1 +endi + + +print =============== clear +#sql drop database $db +#sql select * from information_schema.ins_databases +#if $rows != 0 then +# return -1 +#endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index 7b3953129a81d6fa56b093c8b048b2071b6fbe70..c676f2b1e0b7168c5e83e9b5d2c9b07a1164effb 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -51,6 +51,11 @@ if $data00 != @ins_stables@ then return -1 endi +sql select * from information_schema.ins_tables where table_name=''; +if $rows != 0 then + return -1 +endi + sql select tbname from information_schema.ins_tables; print $rows $data00 if $rows != 33 then diff --git a/tests/script/tsim/query/tableCount.sim b/tests/script/tsim/query/tableCount.sim new file mode 100644 index 0000000000000000000000000000000000000000..d8d9bb9b03e227ce3e54d0d1c84071268b768774 --- /dev/null +++ b/tests/script/tsim/query/tableCount.sim @@ -0,0 +1,107 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql create database db1; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql create table tba3 using sta tags(3, 3, 3); +sql create table tba4 using sta tags(3, 3, 3); +sql create table tba5 using sta tags(3, 3, 3); +sql create table tba6 using sta tags(3, 3, 3); +sql create table tba7 using sta tags(3, 3, 3); +sql create table tba8 using sta tags(3, 3, 3); +sql create table tbb1 using stb tags(4, 4, 4); +sql create table tbb2 using stb tags(5, 5, 5); +sql create table tbb3 using stb tags(6, 6, 6); +sql create table tbb4 using stb tags(4, 4, 4); +sql create table tbb5 using stb tags(5, 5, 5); +sql create table tbb6 using stb tags(6, 6, 6); +sql create table tbb7 using stb tags(7, 7, 7); +sql create table tbb8 using stb tags(8, 8, 8); +sql create table tbn1 (ts timestamp, f1 int); +sql create database db2 vgroups 3; +sql create database db2; +sql use db2; +sql create stable sta (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create stable stb (ts timestamp, f1 int, f2 binary(200)) tags(t1 int, t2 int, t3 int); +sql create table tba1 using sta tags(1, 1, 1); +sql create table tba2 using sta tags(2, 2, 2); +sql create table tbb1 using stb tags(4, 4, 4); +sql create table tbb2 using stb tags(5, 5, 5); +sql create table tbb3 using stb tags(6, 6, 6); + +sql select count(table_name) from information_schema.ins_tables group by stable_name; +if $rows != 3 then + return -1 +endi +sql select count(table_name) from information_schema.ins_tables group by db_name; +if $rows != 4 then + return -1 +endi +sql select count(table_name) from information_schema.ins_tables group by db_name, stable_name; +if $rows != 7 then + return -1 +endi +sql select stable_name,count(table_name) from information_schema.ins_tables group by stable_name order by stable_name; +if $rows != 3 then + return -1 +endi +if $data01 != 30 then + return -1 +endi +if $data11 != 10 then + return -1 +endi +if $data21 != 11 then + return -1 +endi +sql select db_name,count(table_name) from information_schema.ins_tables group by db_name order by db_name; +if $rows != 4 then + return -1 +endi +if $data01 != 17 then + return -1 +endi +if $data11 != 5 then + return -1 +endi +if $data21 != 24 then + return -1 +endi +if $data31 != 5 then + return -1 +endi +sql select db_name,stable_name,count(table_name) from information_schema.ins_tables group by db_name, stable_name order by db_name, stable_name; +if $rows != 7 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 8 then + return -1 +endi +if $data22 != 8 then + return -1 +endi +if $data32 != 2 then + return -1 +endi +if $data42 != 3 then + return -1 +endi +if $data52 != 24 then + return -1 +endi +if $data62 != 5 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 0b48a815e21900c4a80e3ea883e6b95f26bcf695..e539f115317abcd2b64279ec476f8f3464a559d5 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -23,12 +23,15 @@ if $system_content == Windows_NT then return 0; endi if $system_content == Windows_NT then - sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int bufSize 8; + sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int; sql create aggregate function l2norm as 'C:\\Windows\\Temp\\l2norm.dll' outputtype double bufSize 8; else - sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int bufSize 8; + sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int; sql create aggregate function l2norm as '/tmp/udf/libl2norm.so' outputtype double bufSize 8; endi + +sql_error create function bit_and as '/tmp/udf/libbitand.so' oputtype json; + sql show functions; if $rows != 2 then return -1 diff --git a/tests/script/tsim/query/udf_with_const.sim b/tests/script/tsim/query/udf_with_const.sim index 7a2a3389bd155e1c384823ce94c3fdd96b14139e..4a12dd68682a1afe39bf31be1b5b249d2433ee72 100644 --- a/tests/script/tsim/query/udf_with_const.sim +++ b/tests/script/tsim/query/udf_with_const.sim @@ -23,9 +23,9 @@ if $system_content == Windows_NT then endi if $system_content == Windows_NT then - sql create function gpd as 'C:\\Windows\\Temp\\gpd.dll' outputtype int bufSize 8; + sql create function gpd as 'C:\\Windows\\Temp\\gpd.dll' outputtype int; else - sql create function gpd as '/tmp/udf/libgpd.so' outputtype int bufSize 8; + sql create function gpd as '/tmp/udf/libgpd.so' outputtype int; endi sql show functions; if $rows != 1 then diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim new file mode 100644 index 0000000000000000000000000000000000000000..9e0492ffd95e3baed20a77ddf3423a1e8872c1d0 --- /dev/null +++ b/tests/script/tsim/query/udfpy.sim @@ -0,0 +1,328 @@ +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 udf +system sh/compile_udf.sh +system sh/prepare_pyudf.sh +system mkdir -p /tmp/pyudf +system cp sh/pybitand.py /tmp/pyudf/ +system cp sh/pyl2norm.py /tmp/pyudf/ +system ls /tmp/pyudf + +sql create database udf vgroups 3; +sql use udf; +sql select * from information_schema.ins_databases; + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi +if $system_content == Windows_NT then + sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int; + sql create aggregate function l2norm as 'C:\\Windows\\Temp\\l2norm.dll' outputtype double bufSize 8; +else + sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int; + sql create aggregate function l2norm as '/tmp/udf/libl2norm.so' outputtype double bufSize 8; +endi +sql create function pybitand as '/tmp/pyudf/pybitand.py' outputtype int language 'python'; +sql create aggregate function pyl2norm as '/tmp/pyudf/pyl2norm.py' outputtype double bufSize 128 language 'python'; + +sql show functions; +if $rows != 4 then + return -1 +endi + +sql select func_language, func_body,name from information_schema.ins_functions order by name +if $rows != 4 then + return -1 +endi + +if $data00 != @C@ then + return -1 +endi +if $data10 != @C@ then + return -1 +endi +if $data20 != @Python@ then + return -1 +endi +if $data30 != @Python@ then + return -1 +endi + +sql select bit_and(f, f) from t; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +sql select pybitand(f, f) from t; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +sql select l2norm(f) from t; +if $rows != 1 then + print expect 1, actual $rows + return -1 +endi +if $data00 != 2.236067977 then + return -1 +endi + +sql select pyl2norm(f) from t; +if $rows != 1 then + print expect 1, actual $rows + return -1 +endi +if $data00 != 2.236067977 then + return -1 +endi + +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select bit_and(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select l2norm(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + +sql select pybitand(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select pyl2norm(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + +sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2); +sql select bit_and(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +sql select l2norm(f1, f2) from t2; +print $rows, $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 2.645751311 then + return -1 +endi + +sql select pybitand(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +sql select pyl2norm(f1, f2) from t2; +print $rows, $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 2.645751311 then + return -1 +endi + + +sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9); +sql select l2norm(f1-f2), l2norm(f1+f2) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1; +endi +if $data00 != 5.656854249 then + return -1 +endi +if $data01 != 18.547236991 then + return -1 +endi + +sql select l2norm(bit_and(f2, f1)), l2norm(bit_and(f1, f2)) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi +if $data01 != 1.414213562 then + return -1 +endi + +sql select l2norm(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2); +print $rows , $data00 , $data10 , $data20 +if $rows != 3 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 9.055385138 then + return -1 +endi +if $data20 != 8.000000000 then + return -1 +endi + +sql select pyl2norm(f1-f2), pyl2norm(f1+f2) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1; +endi +if $data00 != 5.656854249 then + return -1 +endi +if $data01 != 18.547236991 then + return -1 +endi + +sql select pyl2norm(pybitand(f2, f1)), pyl2norm(pybitand(f1, f2)) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi +if $data01 != 1.414213562 then + return -1 +endi + +sql select pyl2norm(f2) from udf.t2 group by 1-pybitand(f1, f2) order by 1-pybitand(f1,f2); +print $rows , $data00 , $data10 , $data20 +if $rows != 3 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 9.055385138 then + return -1 +endi +if $data20 != 8.000000000 then + return -1 +endi + +sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int +sql select func_version from information_schema.ins_functions where name='bit_and' +if $data00 != 1 then + return -1 +endi +sql select bit_and(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 , $data40 , $data50 +if $rows != 6 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +if $data40 != 0 then + return -1 +endi +if $data50 != 1 then + return -1 +endi +#sql drop function bit_and; +#sql show functions; +#if $rows != 1 then +# return -1 +#endi +#if $data00 != @l2norm@ then +# return -1 +# endi +#sql drop function l2norm; +#sql show functions; +#if $rows != 0 then +# return -1 +#endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim new file mode 100644 index 0000000000000000000000000000000000000000..dc3d2cbec4e7677e98a1a038fc9b57b9f371d2ff --- /dev/null +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -0,0 +1,28 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; +sql CREATE STABLE bw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE aw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE dw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql insert into t1 using dw_yc_h_substation_mea tags('1234567890','testa','0021001','abc01') values(now,'2023-03-27','00:01:00',now,2.3,3.3,4.4,5.5); +sql insert into t2 using dw_yc_h_substation_mea tags('2234567890','testb','0022001','abc02') values(now,'2023-03-27','00:01:00',now,2.3,2.3,2.4,2.5); +sql insert into t3 using aw_yc_h_substation_mea tags('2234567890','testc','0023001','abc03') values(now,'2023-03-27','00:15:00',now,2.3,2.3,2.4,2.5); +sql insert into t4 using bw_yc_h_substation_mea tags('4234567890','testd','0021001','abc03') values(now,'2023-03-27','00:45:00',now,2.3,2.3,2.4,2.5); +sql insert into t5 using bw_yc_h_substation_mea tags('5234567890','testd','0021001','abc03') values(now,'2023-03-27','00:00:00',now,2.3,2.3,2.4,2.5); +sql select t.ts,t.id,t.name,t.create_date,t.create_time,t.datasource,t.sum_p_value from (select ts,id,name,create_date,create_time,datasource,sum_p_value from bw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from aw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from dw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45')) t where t.datasource='0021001' and t.id='4234567890' order by t.create_time; + +if $rows != 1 then + return -1 +endi +if $data01 != @4234567890@ then + return -1 +endi +if $data05 != @0021001@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index cae7a66589bac72c352ed855902f3b23bb798046..8990e492fa9aa3805cc8427827484958a7fc4aaf 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -244,5 +244,15 @@ if $rows <= 0 then return -1 endi +sql show cluster alive; +if $rows <= 0 then + return -1 +endi + +sql show db.alive; +if $rows <= 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index e69875d69f20b01f91ea2b90fd912c84b88d2455..15ca6bf7c924bb7cffba8f0607cb0a5c1fea95cb 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -37,7 +37,7 @@ if $loop_count == 20 then endi if $rows != 4 then - print =====rows=$rows + print =====rows=$rows, expect 4 goto loop0 endi @@ -53,7 +53,7 @@ if $data02 != 2 then endi if $data03 != 5 then - print =====data03=$data03 + print =====data03=$data03, expect:5 goto loop0 endi diff --git a/tests/script/tsim/stream/checkStreamSTable1.sim b/tests/script/tsim/stream/checkStreamSTable1.sim new file mode 100644 index 0000000000000000000000000000000000000000..495e1cf35854b4e1f434c793b29091b7ec5b034d --- /dev/null +++ b/tests/script/tsim/stream/checkStreamSTable1.sim @@ -0,0 +1,71 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2 from st interval(1s) ; +sql insert into t1 values(1648791211000,1,2,3); +sql insert into t1 values(1648791212000,2,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print 1 select * from streamt1; +sql select * from streamt1; + +if $rows != 2 then + print rows=$rows + goto loop0 +endi + +print drop stream streams1 +sql drop stream streams1; + +print alter table streamt1 add column c3 double +sql alter table streamt1 add column c3 double; + +print create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; +sql create stream streams1 trigger at_once into streamt1 as select _wstart, count(*) c1, count(a) c2, avg(b) c3 from st interval(1s) ; + +sql insert into t2 values(1648791213000,1,2,3); +sql insert into t1 values(1648791214000,1,2,3); + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print 2 select * from streamt1; +sql select * from streamt1; + +if $rows != 4 then + print rows=$rows + goto loop1 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim index dd74b73dcefc76f69a19b424b5f544cbf0232559..c84e52067cade9cf9df2452d73a5d41260bfc3a9 100644 --- a/tests/script/tsim/stream/deleteState.sim +++ b/tests/script/tsim/stream/deleteState.sim @@ -51,13 +51,8 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then - print =====data01=$data01 - goto loop1 -endi - -if $data02 != NULL then - print =====data02=$data02 +if $rows != 0 then + print =====rows=$rows goto loop1 endi diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 1d58922928c7313aa97bff2ed29e26e9e1035952..1559d3d32b5752ce9ec3be66117daeab354ca1ca 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -272,4 +272,122 @@ if $data12 != 2 then goto loop3 endi +print ===== step3 + +sql drop database if exists test4; +sql create database test4 vgroups 10; +sql use test4; +sql create stable st(ts timestamp,a int,b int,c varchar(250) ) tags(ta int,tb int,tc int); +sql create table aaa using st tags(1,1,1); +sql create table bbb using st tags(2,2,2); +sql create table ccc using st tags(3,2,2); +sql create table ddd using st tags(4,2,2); + + +sql create stream streams1 ignore expired 0 fill_history 0 watermark 3s into streamst subtable(c) as select _wstart, c , count(*) c1, last_row(b) c2 from st partition by c interval(1s) ; + +sql insert into aaa values(1648791221001,2,2,"/a1/aa/aa"); +sql insert into bbb values(1648791221001,2,2,"/a1/aa/aa"); +sql insert into ccc values(1648791221001,2,2,"/a1/aa/aa"); +sql insert into ddd values(1648791221001,2,2,"/a1/aa/aa"); + +sql insert into aaa values(1648791222002,2,2,"/a2/aa/aa"); +sql insert into bbb values(1648791222002,2,2,"/a2/aa/aa"); +sql insert into ccc values(1648791222002,2,2,"/a2/aa/aa"); +sql insert into ddd values(1648791222002,2,2,"/a2/aa/aa"); + +sql insert into aaa values(1648791223003,2,2,"/a3/aa/aa"); +sql insert into bbb values(1648791223003,2,2,"/a3/aa/aa"); +sql insert into ccc values(1648791223003,2,2,"/a3/aa/aa"); +sql insert into ddd values(1648791223003,2,2,"/a3/aa/aa"); + +sql insert into aaa values(1648791224003,2,2,"/a4/aa/aa"); +sql insert into bbb values(1648791224003,2,2,"/a4/aa/aa"); +sql insert into ccc values(1648791224003,2,2,"/a4/aa/aa"); +sql insert into ddd values(1648791224003,2,2,"/a4/aa/aa"); + + +sql insert into aaa values(1648791225003,2,2,"/a5/aa/aa"); +sql insert into bbb values(1648791225003,2,2,"/a5/aa/aa"); +sql insert into ccc values(1648791225003,2,2,"/a5/aa/aa"); +sql insert into ddd values(1648791225003,2,2,"/a5/aa/aa"); + +sql insert into aaa values(1648791226003,2,2,"/a6/aa/aa"); +sql insert into bbb values(1648791226003,2,2,"/a6/aa/aa"); +sql insert into ccc values(1648791226003,2,2,"/a6/aa/aa"); +sql insert into ddd values(1648791226003,2,2,"/a6/aa/aa"); + +$loop_count = 0 + +loop4: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamst; + +if $rows == 0 then + goto loop4 +endi + +sql delete from aaa where ts = 1648791223003 ; + +$loop_count = 0 + +loop5: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamst; + +if $rows == 0 then + goto loop5 +endi + + +sql delete from ccc; + +$loop_count = 0 + +loop6: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamst; + +if $rows == 0 then + goto loop6 +endi + +sql delete from ddd; + +$loop_count = 0 + +loop7: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamst; + +if $rows == 0 then + goto loop7 +endi + +print ===== over + system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim index 529a2a1b309b1351bd0515c2597e327a58d8574f..a2d9c4ab458219beb81d53a4ad1148806e068e71 100644 --- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -3,7 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start -#==system sh/exec.sh -n dnode1 -s start -v sleep 50 sql connect @@ -11,7 +10,6 @@ sql connect sql create dnode $hostname2 port 7200 system sh/exec.sh -n dnode2 -s start -#==system sh/exec.sh -n dnode2 -s start -v print ===== step1 $x = 0 @@ -37,14 +35,14 @@ endi print ===== step2 -sql create database test vgroups 4; +sql create database test vgroups 10; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); -sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); +sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 delete_mark 10s into streamtST1 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s); sleep 1000 @@ -235,6 +233,29 @@ endi print loop3 over +sql insert into ts1 values(1648791200001,1,12,3,1.0); +sql insert into ts2 values(1648791200001,1,12,3,1.0); +sql insert into ts3 values(1648791200001,1,12,3,1.0); +sql insert into ts4 values(1648791200001,1,12,3,1.0); + +$loop_count = 0 +loop31: +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamtST1; + +if $rows <= 4 then + print =====rows=$rows + goto loop31 +endi + +print loop31 over + sql drop stream if exists streams1; sql drop database if exists test1; @@ -243,7 +264,7 @@ sql use test1; sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); -sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 delete_mark 20s into streamt1 as select _wstart as c0, count(*) c1, count(a) c2 from st interval(10s) ; sql insert into t1 values(1648791211000,1,2,3); diff --git a/tests/script/tsim/stream/partitionbyColumnState.sim b/tests/script/tsim/stream/partitionbyColumnState.sim index 62262a490ce739a463142d2b91a5ffecb1fe8f26..b69ab2df52f6a43ffc36a7dbc91c853d4902c00b 100644 --- a/tests/script/tsim/stream/partitionbyColumnState.sim +++ b/tests/script/tsim/stream/partitionbyColumnState.sim @@ -27,13 +27,8 @@ if $loop_count == 10 then return -1 endi -if $data01 != 1 then - print =====data01=$data01 - goto loop0 -endi - -if $data02 != NULL then - print =====data02=$data02 +if $rows != 0 then + print =====rows=$rows goto loop0 endi diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim new file mode 100644 index 0000000000000000000000000000000000000000..67e02c0890821dc2a57776e64faa95ee9a89c997 --- /dev/null +++ b/tests/script/tsim/stream/state1.sim @@ -0,0 +1,157 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print step 1 +print =============== create database +sql create database test vgroups 4; +sql select * from information_schema.ins_databases; +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double, id int); + +print create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a); + +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 into streamt1 as select _wstart, count(*) c1 from t1 state_window(a); + +sql insert into t1(ts) values(1648791213000); + +$loop_count = 0 +loop0: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +print $data00 $data01 +print $data10 $data11 + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791214000,1,2,3,1.0,3); +$loop_count = 0 +loop1: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop1 +endi + +sql insert into t1 values(1648791215000,2,2,3,1.0,4); + +$loop_count = 0 +loop2: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; + +if $rows != 2 then + print =====rows=$rows + goto loop2 +endi + +sql insert into t1(ts) values(1648791216000); + +$loop_count = 0 +loop3: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt1; +if $rows != 2 then + print =====rows=$rows + goto loop3 +endi + +print step 1 over +print step 2 + +sql create database test2 vgroups 1; +sql use test2; +sql create table t1(ts timestamp, a int, b int , c int, d double); +print create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a) +sql create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213010,1,2,3,1.1); + +$loop_count = 0 +loop4: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop4 +endi + +print insert into t1 values(1648791213005,2,2,3,1.1) +sql insert into t1 values(1648791213005,2,2,3,1.1); + +$loop_count = 0 +loop5: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt2 +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 + +if $rows != 3 then + print =====rows=$rows + goto loop5 +endi + +print step 2 over + +print state1 end + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index f069ff40e597df927d7162d5d628a7d8a50a23ca..aae1b25636b46a738096b93f8325e2d6f8a1306c 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -55,7 +55,7 @@ $loop_cnt = 0 check_db_ready: $loop_cnt = $loop_cnt + 1 sleep 200 -if $loop_cnt == 100 then +if $loop_cnt == 500 then print ====> db not ready! return -1 endi diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c5fbf41b66fcba4088e6480c721d4a6e21811d7a..0abe56ab3c3d7d09b6f441aed813445be0d2df92 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -114,6 +114,7 @@ run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim run tsim/insert/basic0.sim run tsim/insert/update0.sim +run tsim/insert/delete0.sim run tsim/insert/backquote.sim run tsim/insert/null.sim run tsim/catalog/alterInCurrent.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 6880f290f5fda796cbf5b90f98cf5d40c1976eff..4551228f2f67e6da9beed93f984e93113a77055f 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -111,8 +117,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -134,7 +140,7 @@ if $data[0][2] != $expectmsgcnt then print expect $expectmsgcnt , actual $data02 return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -155,6 +161,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -183,8 +192,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -202,7 +211,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -226,6 +235,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -254,8 +266,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -273,7 +285,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 11b645c4d1a008dc82af07999b86ce32c5484947..51d39e8d110f0be00e3a16cf7cdd47e6be226a78 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for stb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -111,11 +117,11 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -142,14 +148,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -186,6 +192,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -214,10 +223,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -244,14 +253,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -288,6 +297,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -316,10 +328,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -346,14 +358,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index dce73be592e459264916f67b4390141115c05bc8..8356a60b672c54feb37663d5c3b8f3391c99a456 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,8 +86,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -105,7 +108,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -118,6 +121,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -140,8 +146,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -159,7 +165,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -175,6 +181,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -197,8 +206,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -216,7 +225,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 87559305ba19cb30b7c8f7928a1437f3246cc139..63e7e2dcf4e600e340f6c3767ab91dfd19fc5338 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -82,10 +85,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -112,14 +115,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -147,6 +150,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -169,10 +175,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -199,14 +205,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -234,6 +240,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -256,10 +265,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -286,14 +295,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index dda5e0059e5c760925e7944b13089b1d3897c0f8..cfdae059dc862d4177d12891ffd9499900ba3b41 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -80,8 +83,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_stb_all @@ -89,7 +92,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -118,40 +121,40 @@ endi # $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][2] > $totalMsgOfStb then - return -1 -endi -if $data[1][2] < $totalMsgOfOneTopic then +#if $data[0][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[0][2] > $expectmsgcnt then return -1 endi -if $data[1][2] > $totalMsgOfStb then +#if $data[1][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[1][2] > $expectmsgcnt then return -1 endi -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +#$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $totalMsgCons then - print total: $totalMsgCons +if $sumOfMsgCnt != $expectmsgcnt then + print total: $expectmsgcnt print sum: $sumOfMsgCnt return -1 endi # $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][3] > $totalMsgOfStb then - return -1 -endi -if $data[1][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][3] > $totalMsgOfStb then - return -1 -endi +#if $data[0][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[0][3] > $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] > $totalMsgOfStb then +# return -1 +#endi $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfRows = $data[0][3] + $data[1][3] @@ -168,6 +171,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -189,15 +195,15 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -226,24 +232,24 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb # or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfCtb then +if $data[0][2] == $topicNum then + if $data[1][2] == 1 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfCtb then +elif $data[0][2] == 1 then + if $data[1][2] == $topicNum then goto check_ok_0 endi endi return -1 check_ok_0: -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfCtb then +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_1 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfCtb then +elif $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then goto check_ok_1 endi endi @@ -259,6 +265,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -280,8 +289,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ntb_function @@ -289,7 +298,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -318,12 +327,12 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb # or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfNtb then +if $data[0][2] == $expectmsgcnt then + if $data[1][2] == 1 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfNtb then +elif $data[0][2] == 1 then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -334,8 +343,8 @@ if $data[0][3] == $totalMsgOfOneTopic then if $data[1][3] == $totalMsgOfNtb then goto check_ok_3 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfNtb then +elif $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_3 endi endi diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index 8d677766d74371209e53bab94b62a524318146ee..a64dd6924d7c30dec12e315924439b37019af500 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -111,8 +117,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -130,10 +136,10 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then +#if $data[0][2] != $expectmsgcnt then +# return -1 +#endi +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -154,6 +160,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -182,8 +191,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -201,7 +210,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -225,6 +234,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -253,8 +265,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -272,7 +284,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index 4921c86c4574174b06af4d906b9a3e74e8ee9c3c..4e47e3dbf9a0eeaf4380e43ceb6caef81f3c8770 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -82,6 +85,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -110,10 +116,10 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -163,19 +169,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfMsgRows = $data[0][3] + $data[1][3] -if $sumOfMsgRows != $expectmsgcnt then +if $sumOfMsgRows != $totalMsgOfStb then return -1 endi @@ -197,6 +203,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -225,10 +234,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -255,13 +264,13 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfCtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_0 endi @@ -299,6 +308,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -327,10 +339,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -357,13 +369,13 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfNtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_2 endi diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 9b418f12f2b95410994cf52fd4e140a8dcc73889..6b35342ad119431d1eece232486069c9792015d9 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -80,8 +83,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -102,7 +105,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -115,6 +118,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -137,8 +143,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -156,7 +162,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -172,6 +178,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -194,8 +203,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -213,7 +222,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index f1755f732baa365689653d83e117d2780c40f621..122a91af3627fb12d00be61a503942e32dca604d 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -79,10 +82,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -131,19 +134,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfConsRow = $data[0][3] + $data[1][3] -if $sumOfConsRow != $expectmsgcnt then +if $sumOfConsRow != $totalMsgOfStb then return -1 endi @@ -156,6 +159,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -178,10 +184,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -208,14 +214,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -244,6 +250,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -266,10 +275,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -296,14 +305,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim index de0468e6f26871cc4b5118889a0fcd1262c291e4..81fff3522499ff99ccfa9a92be5483ea42d775ba 100644 --- a/tests/script/tsim/tmq/snapshot.sim +++ b/tests/script/tsim/tmq/snapshot.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -83,6 +86,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -152,6 +158,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -223,6 +232,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index e586719db2fa7646d487658d09e71693ff557c7a..c79892ae1d943a071c58fe2236eb90ef9a5e10d4 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -34,6 +34,9 @@ $showRow = 0 sql connect sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 + print == create topics from super table sql create topic topic_stb_column as select ts, c3 from stb sql create topic topic_stb_all as select ts, c1, c2, c3 from stb @@ -147,6 +150,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ctb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) @@ -224,6 +230,9 @@ sql create database $cdbName vgroups 1 sleep 500 sql use $cdbName +print == alter database +sql alter database $cdbName wal_retention_period 3600 + print == create consume info table and consume result table for ntb sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) diff --git a/tests/script/tsim/tmq/topic.sim b/tests/script/tsim/tmq/topic.sim index cb1e74798ea072a03d2ea2570a5a245222b5c8c5..0bf0873e9f0513b08b6868bed220c5e3d74072f9 100644 --- a/tests/script/tsim/tmq/topic.sim +++ b/tests/script/tsim/tmq/topic.sim @@ -39,6 +39,8 @@ endi sql use $dbName +print == alter database +sql alter database $dbName wal_retention_period 3600 print == create super table sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) diff --git a/tests/script/tsim/user/privilege_db.sim b/tests/script/tsim/user/privilege_db.sim index b708fdab64c1931d6a86c67a8f3ec86cb880677a..50eaa1210866dcfb98f0c889a5eef9960d2faf81 100644 --- a/tests/script/tsim/user/privilege_db.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -4,13 +4,13 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create db -sql create database d1 vgroups 1; +sql create database d1 vgroups 1 wal_retention_period 3600; sql use d1 sql create table d1_stb (ts timestamp, i int) tags (j int) sql create topic d1_topic_1 as select ts, i from d1_stb -sql create database d2 vgroups 1; -sql create database d3 vgroups 1; +sql create database d2 vgroups 1 wal_retention_period 3600; +sql create database d3 vgroups 1 wal_retention_period 3600; sql select * from information_schema.ins_databases if $rows != 5 then return -1 diff --git a/tests/script/tsim/user/privilege_topic.sim b/tests/script/tsim/user/privilege_topic.sim index 9ce5bebec3d68761fb5364896f81e4a5e1428878..9d096a97807a70dfe34297c7fac688502b236a5a 100644 --- a/tests/script/tsim/user/privilege_topic.sim +++ b/tests/script/tsim/user/privilege_topic.sim @@ -4,9 +4,9 @@ system sh/exec.sh -n dnode1 -s start sql connect print =============== create db -sql create database root_d1 vgroups 1; -sql create database root_d2 vgroups 1; -sql create database root_d3 vgroups 1; +sql create database root_d1 vgroups 1 wal_retention_period 3600; +sql create database root_d2 vgroups 1 wal_retention_period 3600; +sql create database root_d3 vgroups 1 wal_retention_period 3600; sql show user privileges if $rows != 1 then diff --git a/tests/script/tsim/valgrind/checkError7.sim b/tests/script/tsim/valgrind/checkError7.sim index af42d1e76b50bc07e9c7a484bd24c9544517f980..edfe8816c9d0e83f705032c64da2dd5ccd7034f2 100644 --- a/tests/script/tsim/valgrind/checkError7.sim +++ b/tests/script/tsim/valgrind/checkError7.sim @@ -8,8 +8,9 @@ sql create database d1 sql use d1 $x = 0 -while $x < 128 +while $x < 5 $tb = d1.s . $x + print create table $tb (ts timestamp, i int) tags (j int) sql create table $tb (ts timestamp, i int) tags (j int) $x = $x + 1 endw @@ -17,7 +18,7 @@ endw print ======================== describe stables # TODO : create stable error $m = 0 -while $m < 128 +while $m < 5 $tb = s . $m $filter = ' . $tb $filter = $filter . ' @@ -36,15 +37,16 @@ print ======================== show stables sql show d1.stables print num of stables is $rows -if $rows != 128 then +if $rows != 5 then return -1 endi print ======================== create table $x = 0 -while $x < 424 +while $x < 42 $tb = d1.t . $x + print create table $tb using d1.s0 tags( $x ) sql create table $tb using d1.s0 tags( $x ) $x = $x + 1 endw @@ -54,7 +56,7 @@ print ======================== show stables sql show d1.tables print num of tables is $rows -if $rows != 424 then +if $rows != 42 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index caf316bd868a057cd8a161817c85d8664a9db82e..e316c104ed68bcc32bc4f84fa3ee803b5a91ae1a 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -15,10 +15,10 @@ sql insert into t values(now, 1)(now+1s, 2); system_content printf %OS% if $system_content == Windows_NT then - sql create function udf1 as 'C:\\Windows\\Temp\\udf1.dll' outputtype int bufSize 8; + sql create function udf1 as 'C:\\Windows\\Temp\\udf1.dll' outputtype int; sql create aggregate function udf2 as 'C:\\Windows\\Temp\\udf2.dll' outputtype double bufSize 8; else - sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8; + sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int; sql create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8; endi sql show functions; @@ -29,10 +29,10 @@ sql select udf1(f) from t; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -51,10 +51,10 @@ sql select udf1(f1, f2) from t2; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -72,10 +72,10 @@ print $rows , $data00 , $data10 , $data20 , $data30 if $rows != 4 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -114,10 +114,10 @@ print $rows , $data00 , $data01 if $rows != 1 then return -1 endi -if $data00 != 176.000000000 then +if $data00 != 2.000000000 then return -1 endi -if $data01 != 152.420471066 then +if $data01 != 1.732050808 then return -1 endi diff --git a/tests/script/win-test-file b/tests/script/win-test-file new file mode 100644 index 0000000000000000000000000000000000000000..2d5a1b3108efb2b884bf7664ff61557f09d2bca5 --- /dev/null +++ b/tests/script/win-test-file @@ -0,0 +1,402 @@ +./test.sh -f tsim/user/basic.sim +./test.sh -f tsim/user/password.sim +./test.sh -f tsim/user/privilege_db.sim +./test.sh -f tsim/user/privilege_sysinfo.sim +./test.sh -f tsim/user/privilege_topic.sim +./test.sh -f tsim/db/alter_option.sim +rem ./test.sh -f tsim/db/alter_replica_13.sim +./test.sh -f tsim/db/alter_replica_31.sim +./test.sh -f tsim/db/basic1.sim +./test.sh -f tsim/db/basic2.sim +./test.sh -f tsim/db/basic3.sim +./test.sh -f tsim/db/basic4.sim +./test.sh -f tsim/db/basic5.sim +./test.sh -f tsim/db/basic6.sim +./test.sh -f tsim/db/commit.sim +./test.sh -f tsim/db/create_all_options.sim +./test.sh -f tsim/db/delete_reuse1.sim +./test.sh -f tsim/db/delete_reuse2.sim +./test.sh -f tsim/db/delete_reusevnode.sim +./test.sh -f tsim/db/delete_reusevnode2.sim +./test.sh -f tsim/db/delete_writing1.sim +./test.sh -f tsim/db/delete_writing2.sim +./test.sh -f tsim/db/error1.sim +./test.sh -f tsim/db/keep.sim +./test.sh -f tsim/db/len.sim +./test.sh -f tsim/db/repeat.sim +./test.sh -f tsim/db/show_create_db.sim +./test.sh -f tsim/db/show_create_table.sim +./test.sh -f tsim/db/tables.sim +./test.sh -f tsim/db/taosdlog.sim +./test.sh -f tsim/dnode/balance_replica1.sim +./test.sh -f tsim/dnode/balance_replica3.sim +./test.sh -f tsim/dnode/balance1.sim +./test.sh -f tsim/dnode/balance2.sim +./test.sh -f tsim/dnode/balance3.sim +./test.sh -f tsim/dnode/balancex.sim +./test.sh -f tsim/dnode/create_dnode.sim +./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +./test.sh -f tsim/dnode/drop_dnode_force.sim +./test.sh -f tsim/dnode/offline_reason.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +./test.sh -f tsim/dnode/vnode_clean.sim +./test.sh -f tsim/dnode/use_dropped_dnode.sim +./test.sh -f tsim/dnode/split_vgroup_replica1.sim +./test.sh -f tsim/dnode/split_vgroup_replica3.sim +./test.sh -f tsim/import/basic.sim +./test.sh -f tsim/import/commit.sim +./test.sh -f tsim/import/large.sim +./test.sh -f tsim/import/replica1.sim +./test.sh -f tsim/insert/backquote.sim +./test.sh -f tsim/insert/basic.sim +./test.sh -f tsim/insert/basic0.sim +./test.sh -f tsim/insert/basic1.sim +./test.sh -f tsim/insert/basic2.sim +./test.sh -f tsim/insert/commit-merge0.sim +./test.sh -f tsim/insert/insert_drop.sim +./test.sh -f tsim/insert/insert_select.sim +./test.sh -f tsim/insert/null.sim +./test.sh -f tsim/insert/query_block1_file.sim +./test.sh -f tsim/insert/query_block1_memory.sim +./test.sh -f tsim/insert/query_block2_file.sim +./test.sh -f tsim/insert/query_block2_memory.sim +./test.sh -f tsim/insert/query_file_memory.sim +./test.sh -f tsim/insert/query_multi_file.sim +./test.sh -f tsim/insert/tcp.sim +./test.sh -f tsim/insert/update0.sim +./test.sh -f tsim/insert/update1_sort_merge.sim +./test.sh -f tsim/insert/update2.sim +./test.sh -f tsim/parser/alter__for_community_version.sim +./test.sh -f tsim/parser/alter_column.sim +./test.sh -f tsim/parser/alter_stable.sim +./test.sh -f tsim/parser/alter.sim +./test.sh -f tsim/parser/alter1.sim +./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +./test.sh -f tsim/parser/auto_create_tb.sim +./test.sh -f tsim/parser/between_and.sim +./test.sh -f tsim/parser/binary_escapeCharacter.sim +./test.sh -f tsim/parser/col_arithmetic_operation.sim +./test.sh -f tsim/parser/columnValue_bigint.sim +./test.sh -f tsim/parser/columnValue_bool.sim +./test.sh -f tsim/parser/columnValue_double.sim +./test.sh -f tsim/parser/columnValue_float.sim +./test.sh -f tsim/parser/columnValue_int.sim +./test.sh -f tsim/parser/columnValue_smallint.sim +./test.sh -f tsim/parser/columnValue_tinyint.sim +./test.sh -f tsim/parser/columnValue_unsign.sim +./test.sh -f tsim/parser/commit.sim +./test.sh -f tsim/parser/condition.sim +./test.sh -f tsim/parser/constCol.sim +./test.sh -f tsim/parser/create_db.sim +./test.sh -f tsim/parser/create_mt.sim +./test.sh -f tsim/parser/create_tb_with_tag_name.sim +./test.sh -f tsim/parser/create_tb.sim +./test.sh -f tsim/parser/dbtbnameValidate.sim +./test.sh -f tsim/parser/distinct.sim +./test.sh -f tsim/parser/fill_us.sim +./test.sh -f tsim/parser/fill.sim +./test.sh -f tsim/parser/first_last.sim +./test.sh -f tsim/parser/fill_stb.sim +./test.sh -f tsim/parser/interp.sim +./test.sh -f tsim/parser/fourArithmetic-basic.sim +./test.sh -f tsim/parser/function.sim +./test.sh -f tsim/parser/groupby-basic.sim +./test.sh -f tsim/parser/groupby.sim +./test.sh -f tsim/parser/having_child.sim +./test.sh -f tsim/parser/having.sim +./test.sh -f tsim/parser/import_commit1.sim +./test.sh -f tsim/parser/import_commit2.sim +./test.sh -f tsim/parser/import_commit3.sim +./test.sh -f tsim/parser/import_file.sim +./test.sh -f tsim/parser/import.sim +./test.sh -f tsim/parser/insert_multiTbl.sim +./test.sh -f tsim/parser/insert_tb.sim +./test.sh -f tsim/parser/join_manyblocks.sim +./test.sh -f tsim/parser/join_multitables.sim +./test.sh -f tsim/parser/join_multivnode.sim +./test.sh -f tsim/parser/join.sim +./test.sh -f tsim/parser/last_cache.sim +./test.sh -f tsim/parser/last_groupby.sim +./test.sh -f tsim/parser/lastrow.sim +./test.sh -f tsim/parser/lastrow2.sim +./test.sh -f tsim/parser/like.sim +./test.sh -f tsim/parser/limit.sim +./test.sh -f tsim/parser/limit1.sim +./test.sh -f tsim/parser/mixed_blocks.sim +./test.sh -f tsim/parser/nchar.sim +./test.sh -f tsim/parser/nestquery.sim +./test.sh -f tsim/parser/null_char.sim +./test.sh -f tsim/parser/precision_ns.sim +./test.sh -f tsim/parser/projection_limit_offset.sim +./test.sh -f tsim/parser/regex.sim +./test.sh -f tsim/parser/regressiontest.sim +./test.sh -f tsim/parser/select_across_vnodes.sim +./test.sh -f tsim/parser/select_distinct_tag.sim +./test.sh -f tsim/parser/select_from_cache_disk.sim +./test.sh -f tsim/parser/select_with_tags.sim +./test.sh -f tsim/parser/selectResNum.sim +./test.sh -f tsim/parser/set_tag_vals.sim +./test.sh -f tsim/parser/single_row_in_tb.sim +./test.sh -f tsim/parser/sliding.sim +./test.sh -f tsim/parser/slimit_alter_tags.sim +./test.sh -f tsim/parser/slimit.sim +./test.sh -f tsim/parser/slimit1.sim +./test.sh -f tsim/parser/stableOp.sim +./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +./test.sh -f tsim/parser/tags_filter.sim +./test.sh -f tsim/parser/tbnameIn.sim +./test.sh -f tsim/parser/timestamp.sim +./test.sh -f tsim/parser/top_groupby.sim +./test.sh -f tsim/parser/topbot.sim +./test.sh -f tsim/parser/union.sim +./test.sh -f tsim/parser/union_sysinfo.sim +./test.sh -f tsim/parser/where.sim +./test.sh -f tsim/query/tagLikeFilter.sim +./test.sh -f tsim/query/charScalarFunction.sim +./test.sh -f tsim/query/explain.sim +./test.sh -f tsim/query/interval-offset.sim +./test.sh -f tsim/query/interval.sim +./test.sh -f tsim/query/scalarFunction.sim +./test.sh -f tsim/query/scalarNull.sim +./test.sh -f tsim/query/session.sim +./test.sh -f tsim/query/sys_tbname.sim +./test.sh -f tsim/query/groupby.sim +./test.sh -f tsim/query/event.sim +./test.sh -f tsim/query/forceFill.sim +./test.sh -f tsim/query/emptyTsRange.sim +./test.sh -f tsim/query/partitionby.sim +./test.sh -f tsim/qnode/basic1.sim +./test.sh -f tsim/snode/basic1.sim +./test.sh -f tsim/mnode/basic1.sim +./test.sh -f tsim/mnode/basic2.sim +./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic5.sim +./test.sh -f tsim/show/basic.sim +./test.sh -f tsim/table/autocreate.sim +./test.sh -f tsim/table/basic1.sim +./test.sh -f tsim/table/basic2.sim +./test.sh -f tsim/table/basic3.sim +./test.sh -f tsim/table/bigint.sim +./test.sh -f tsim/table/binary.sim +./test.sh -f tsim/table/bool.sim +./test.sh -f tsim/table/column_name.sim +./test.sh -f tsim/table/column_num.sim +./test.sh -f tsim/table/column_value.sim +./test.sh -f tsim/table/column2.sim +./test.sh -f tsim/table/createmulti.sim +./test.sh -f tsim/table/date.sim +./test.sh -f tsim/table/db.table.sim +./test.sh -f tsim/table/delete_reuse1.sim +./test.sh -f tsim/table/delete_reuse2.sim +./test.sh -f tsim/table/delete_writing.sim +./test.sh -f tsim/table/describe.sim +./test.sh -f tsim/table/double.sim +./test.sh -f tsim/table/float.sim +./test.sh -f tsim/table/hash.sim +./test.sh -f tsim/table/int.sim +./test.sh -f tsim/table/limit.sim +./test.sh -f tsim/table/smallint.sim +./test.sh -f tsim/table/table_len.sim +./test.sh -f tsim/table/table.sim +./test.sh -f tsim/table/tinyint.sim +./test.sh -f tsim/table/vgroup.sim +./test.sh -f tsim/stream/basic0.sim -g +./test.sh -f tsim/stream/basic1.sim +./test.sh -f tsim/stream/basic2.sim +./test.sh -f tsim/stream/drop_stream.sim +./test.sh -f tsim/stream/fillHistoryBasic1.sim +./test.sh -f tsim/stream/fillHistoryBasic2.sim +./test.sh -f tsim/stream/fillHistoryBasic3.sim +./test.sh -f tsim/stream/distributeInterval0.sim +./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +./test.sh -f tsim/stream/distributeSession0.sim +./test.sh -f tsim/stream/session0.sim +./test.sh -f tsim/stream/session1.sim +./test.sh -f tsim/stream/state0.sim +./test.sh -f tsim/stream/triggerInterval0.sim +./test.sh -f tsim/stream/triggerSession0.sim +./test.sh -f tsim/stream/partitionby.sim +./test.sh -f tsim/stream/partitionby1.sim +./test.sh -f tsim/stream/schedSnode.sim +./test.sh -f tsim/stream/windowClose.sim +./test.sh -f tsim/stream/ignoreExpiredData.sim +./test.sh -f tsim/stream/sliding.sim +./test.sh -f tsim/stream/partitionbyColumnInterval.sim +./test.sh -f tsim/stream/partitionbyColumnSession.sim +./test.sh -f tsim/stream/partitionbyColumnState.sim +./test.sh -f tsim/stream/deleteInterval.sim +./test.sh -f tsim/stream/deleteSession.sim +./test.sh -f tsim/stream/deleteState.sim +./test.sh -f tsim/stream/fillIntervalDelete0.sim +./test.sh -f tsim/stream/fillIntervalDelete1.sim +./test.sh -f tsim/stream/fillIntervalLinear.sim +./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +./test.sh -f tsim/stream/fillIntervalPrevNext.sim +./test.sh -f tsim/stream/fillIntervalValue.sim +./test.sh -f tsim/stream/udTableAndTag0.sim +./test.sh -f tsim/stream/udTableAndTag1.sim +./test.sh -f tsim/trans/lossdata1.sim +./test.sh -f tsim/trans/create_db.sim +./test.sh -f tsim/tmq/basic1.sim +./test.sh -f tsim/tmq/basic2.sim +./test.sh -f tsim/tmq/basic3.sim +./test.sh -f tsim/tmq/basic4.sim +./test.sh -f tsim/tmq/basic1Of2Cons.sim +./test.sh -f tsim/tmq/basic2Of2Cons.sim +./test.sh -f tsim/tmq/basic3Of2Cons.sim +./test.sh -f tsim/tmq/basic4Of2Cons.sim +./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +./test.sh -f tsim/tmq/topic.sim +./test.sh -f tsim/tmq/snapshot.sim +./test.sh -f tsim/tmq/snapshot1.sim +./test.sh -f tsim/stable/alter_comment.sim +./test.sh -f tsim/stable/alter_count.sim +./test.sh -f tsim/stable/alter_import.sim +./test.sh -f tsim/stable/alter_insert1.sim +./test.sh -f tsim/stable/alter_insert2.sim +./test.sh -f tsim/stable/alter_metrics.sim +./test.sh -f tsim/stable/column_add.sim +./test.sh -f tsim/stable/column_drop.sim +./test.sh -f tsim/stable/column_modify.sim +./test.sh -f tsim/stable/disk.sim +./test.sh -f tsim/stable/dnode3.sim +./test.sh -f tsim/stable/metrics.sim +./test.sh -f tsim/stable/refcount.sim +./test.sh -f tsim/stable/tag_add.sim +./test.sh -f tsim/stable/tag_drop.sim +./test.sh -f tsim/stable/tag_filter.sim +./test.sh -f tsim/stable/tag_modify.sim +./test.sh -f tsim/stable/tag_rename.sim +./test.sh -f tsim/stable/values.sim +./test.sh -f tsim/stable/vnode3.sim +./test.sh -f tsim/stable/metrics_idx.sim +./test.sh -f tsim/sma/drop_sma.sim +./test.sh -f tsim/sma/sma_leak.sim +./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +./test.sh -f tsim/valgrind/checkError1.sim +./test.sh -f tsim/valgrind/checkError2.sim +./test.sh -f tsim/valgrind/checkError3.sim +./test.sh -f tsim/valgrind/checkError4.sim +./test.sh -f tsim/valgrind/checkError5.sim +./test.sh -f tsim/valgrind/checkError6.sim +./test.sh -f tsim/valgrind/checkError7.sim +./test.sh -f tsim/valgrind/checkError8.sim +./test.sh -f tsim/vnode/replica3_basic.sim +./test.sh -f tsim/vnode/replica3_repeat.sim +./test.sh -f tsim/vnode/replica3_vgroup.sim +./test.sh -f tsim/vnode/replica3_many.sim +./test.sh -f tsim/vnode/replica3_import.sim +./test.sh -f tsim/vnode/stable_balance_replica1.sim +./test.sh -f tsim/vnode/stable_dnode2_stop.sim +./test.sh -f tsim/vnode/stable_dnode2.sim +./test.sh -f tsim/vnode/stable_dnode3.sim +./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +./test.sh -f tsim/vnode/stable_replica3_vnode3.sim +./test.sh -f tsim/sync/3Replica1VgElect.sim +./test.sh -f tsim/sync/3Replica5VgElect.sim +./test.sh -f tsim/sync/oneReplica1VgElect.sim +./test.sh -f tsim/sync/oneReplica5VgElect.sim +./test.sh -f tsim/catalog/alterInCurrent.sim +./test.sh -f tsim/scalar/in.sim +./test.sh -f tsim/scalar/scalar.sim +./test.sh -f tsim/scalar/filter.sim +./test.sh -f tsim/scalar/caseWhen.sim +./test.sh -f tsim/scalar/tsConvert.sim +./test.sh -f tsim/alter/cached_schema_after_alter.sim +./test.sh -f tsim/alter/dnode.sim +./test.sh -f tsim/alter/table.sim +./test.sh -f tsim/cache/new_metrics.sim +./test.sh -f tsim/cache/restart_table.sim +./test.sh -f tsim/cache/restart_metrics.sim +./test.sh -f tsim/column/commit.sim +./test.sh -f tsim/column/metrics.sim +./test.sh -f tsim/column/table.sim +./test.sh -f tsim/compress/commitlog.sim +./test.sh -f tsim/compress/compress2.sim +./test.sh -f tsim/compress/compress.sim +./test.sh -f tsim/compress/uncompress.sim +./test.sh -f tsim/compute/avg.sim +./test.sh -f tsim/compute/block_dist.sim +./test.sh -f tsim/compute/bottom.sim +./test.sh -f tsim/compute/count.sim +./test.sh -f tsim/compute/diff.sim +./test.sh -f tsim/compute/diff2.sim +./test.sh -f tsim/compute/first.sim +./test.sh -f tsim/compute/interval.sim +./test.sh -f tsim/compute/last_row.sim +./test.sh -f tsim/compute/last.sim +./test.sh -f tsim/compute/leastsquare.sim +./test.sh -f tsim/compute/max.sim +./test.sh -f tsim/compute/min.sim +./test.sh -f tsim/compute/null.sim +./test.sh -f tsim/compute/percentile.sim +./test.sh -f tsim/compute/stddev.sim +./test.sh -f tsim/compute/sum.sim +./test.sh -f tsim/compute/top.sim +./test.sh -f tsim/field/2.sim +./test.sh -f tsim/field/3.sim +./test.sh -f tsim/field/4.sim +./test.sh -f tsim/field/5.sim +./test.sh -f tsim/field/6.sim +./test.sh -f tsim/field/binary.sim +./test.sh -f tsim/field/bigint.sim +./test.sh -f tsim/field/bool.sim +./test.sh -f tsim/field/double.sim +./test.sh -f tsim/field/float.sim +./test.sh -f tsim/field/int.sim +./test.sh -f tsim/field/single.sim +./test.sh -f tsim/field/smallint.sim +./test.sh -f tsim/field/tinyint.sim +./test.sh -f tsim/field/unsigined_bigint.sim +./test.sh -f tsim/vector/metrics_field.sim +./test.sh -f tsim/vector/metrics_mix.sim +./test.sh -f tsim/vector/metrics_query.sim +./test.sh -f tsim/vector/metrics_tag.sim +./test.sh -f tsim/vector/metrics_time.sim +./test.sh -f tsim/vector/multi.sim +./test.sh -f tsim/vector/single.sim +./test.sh -f tsim/vector/table_field.sim +./test.sh -f tsim/vector/table_mix.sim +./test.sh -f tsim/vector/table_query.sim +./test.sh -f tsim/vector/table_time.sim +./test.sh -f tsim/wal/kill.sim +./test.sh -f tsim/tag/3.sim +./test.sh -f tsim/tag/4.sim +./test.sh -f tsim/tag/5.sim +./test.sh -f tsim/tag/6.sim +./test.sh -f tsim/tag/add.sim +./test.sh -f tsim/tag/bigint.sim +./test.sh -f tsim/tag/binary_binary.sim +./test.sh -f tsim/tag/binary.sim +./test.sh -f tsim/tag/bool_binary.sim +./test.sh -f tsim/tag/bool_int.sim +./test.sh -f tsim/tag/bool.sim +./test.sh -f tsim/tag/change.sim +./test.sh -f tsim/tag/column.sim +./test.sh -f tsim/tag/commit.sim +./test.sh -f tsim/tag/create.sim +./test.sh -f tsim/tag/delete.sim +./test.sh -f tsim/tag/double.sim +./test.sh -f tsim/tag/filter.sim +./test.sh -f tsim/tag/float.sim +./test.sh -f tsim/tag/int_binary.sim +./test.sh -f tsim/tag/int_float.sim +./test.sh -f tsim/tag/int.sim +./test.sh -f tsim/tag/set.sim +./test.sh -f tsim/tag/smallint.sim +./test.sh -f tsim/tag/tinyint.sim +./test.sh -f tsim/tag/drop_tag.sim +./test.sh -f tsim/tag/tbNameIn.sim +./test.sh -f tmp/monitor.sim diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index f674277df9455d0abac76bc4f214b9c53a62ce91..b642bad285864fd2fd423d0e28c739bc4680bcd0 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -33,7 +33,7 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -rem set "fqdn=" +set "fqdn=localhost" for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" diff --git a/tests/system-test/0-others/backquote_check.py b/tests/system-test/0-others/backquote_check.py index 33577232538074953457830934fa853ba81aec6a..be8590f913f110e0156d71fa81560fc99fa39132 100644 --- a/tests/system-test/0-others/backquote_check.py +++ b/tests/system-test/0-others/backquote_check.py @@ -29,7 +29,7 @@ class TDTestCase: self.streamname = 'stm' self.streamtb = 'stm_stb' def topic_name_check(self): - tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'create database if not exists {self.dbname} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') for name in [self.dbname,self.stbname]: @@ -56,12 +56,12 @@ class TDTestCase: tdSql.execute(f'drop topic `{name}`') def db_name_check(self): - tdSql.execute(f'create database if not exists `{self.dbname}`') + tdSql.execute(f'create database if not exists `{self.dbname}` wal_retention_period 3600') tdSql.execute(f'use `{self.dbname}`') tdSql.execute(f'drop database {self.dbname}') def stream_name_check(self): - tdSql.execute(f'create database if not exists {self.dbname}') + tdSql.execute(f'create database if not exists {self.dbname} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(f'create stable {self.stbname} (ts timestamp,c0 int) tags(t0 int)') tdSql.execute(f'create stream `{self.streamname}` into `{self.streamtb}` as select count(*) from {self.stbname} interval(10s);') @@ -84,4 +84,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/check_assert.py b/tests/system-test/0-others/check_assert.py new file mode 100644 index 0000000000000000000000000000000000000000..ff69b9eeec02caddfe989eacfe3a4a16a5e6d656 --- /dev/null +++ b/tests/system-test/0-others/check_assert.py @@ -0,0 +1,219 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +''' +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +''' + +import sys +import random +import os + +#define code +NO_FOUND = 0 # not found assert or ASSERT +FOUND_OK = 1 # found ASSERT and valid usage +FOUND_NOIF = 2 # found ASSERT but no if like ASSERT(...) +FOUND_LOWER = 3 # found assert write with system assert +FOUND_HAVENOT = 4 # found ASSERT have if but have not like if(!ASSERT) + +code_strs = ["not found", "valid", "found but no if", "system assert","found but have not"] + + +# +# check assert +# +class CheckAssert: + def __init__(self): + self.files = 0 + self.total = [0,0,0,0,0] + + def wholeMatched(self, line, pos, n): + before = False + after = False + if pos != -1: + if pos == 0 or line[pos-1] == ' ' or line[pos-1] == '!' or line[pos-1] == '(': + before = True + if n > pos + 6 + 1: + last = line[pos+6] + if last == ' ' or last == '(': + after = True + + if before and after: + return True + else: + return False + + + def parseLine(self, line): + + # see FOUND_xxx define below + code = 0 + + n = len(line) + if(n < 7): + return NO_FOUND + s1 = line.find("//") # commit line + + # assert + pos = line.find("assert") + if pos != -1: + if self.wholeMatched(line, pos, n): + # comment line check + if not (s1 != -1 and s1 < pos): + return FOUND_LOWER + + # ASSERT + pos = 0 + while pos < n: + pos = line.find("ASSERT", pos) + #print(f" pos={pos} {line}") + if pos == -1: + return NO_FOUND + if self.wholeMatched(line, pos, n): + break + # next + pos += 6 + + # + # found + # + + # comment line + if s1 != -1 and s1 < pos: + return NO_FOUND + # check if + s2 = line.find("if") + if s2 == -1 or s2 > pos or pos - s2 > 5: + return FOUND_NOIF + s3 = line.find("!") + # if(!ASSERT + if s3 > s2 and s3 < pos: + return FOUND_HAVENOT + + return FOUND_OK + + + def checkFile(self, pathfile): + # check .h .c + ext = pathfile[-2:].lower() + if ext != ".h" and ext != ".c": + return + + print(" check file %s"%pathfile) + self.files += 1 + err = 0 + ok = 0 + i = 0 + + # read file context + with open(pathfile, "r") as fp: + lines = fp.readlines() + for line in lines: + i += 1 + code = self.parseLine(line) + self.total[code] += 1 + if code == FOUND_OK: + ok += 1 + if code != NO_FOUND and code != FOUND_OK: + err += 1 + if code != NO_FOUND: + print(f" line: {i} code: {code} {line}") + + # parse end output total + if err > 0 or ok > 0: + print(f" found problem: {err} \n") + + + def scanPath(self, path): + #print(" check path %s"%path) + + ignores = ["/test/"] + for ignore in ignores: + if ignore in path: + print(f" ignore {path} keyword: {ignore}") + return + + with os.scandir(path) as childs: + for child in childs: + if child.is_file(): + self.checkFile(os.path.join(path, child.name)) + elif child.is_dir(): + self.scanPath(child.path) + + + def doCheck(self, path): + print(f" start check path:{path}") + self.scanPath(path) + + # print total + print("\n") + print(f" --------------- total ({self.files} files)--------------") + for i in range(5): + print(f" code : {i} num: {self.total[i]} ({code_strs[i]})") + print(" --------------- end ----------------") + print(f"\n") + +# +# main function +# + +if __name__ == "__main__": + print(" hello, welcome to use check assert tools 1.0.") + path = os.path.dirname(os.path.realpath(__file__)) + label = "TDengine" + pos = path.find(label) + if pos != -1: + pos += len(label) + 1 + src = path[0:pos] + "source" + else: + src = path + + checker = CheckAssert() + checker.doCheck(src) + print(" check assert finished") + + + +''' +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.checker = CheckAssert() + + # run + def run(self): + # calc + selfPath = os.path.dirname(os.path.realpath(__file__)) + projPath = "" + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + src = self.projPath + "src/" + self.checker.checkAssert(src) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + +''' \ No newline at end of file diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 21d307ce3795fcb402d3a061e6a725b092ac7d44..1922ebfb2f9d6c48c911214b3fe13d68a77e5d26 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -3,6 +3,7 @@ import taos import sys import os import time +import platform import inspect from taos.tmq import Consumer @@ -97,7 +98,7 @@ class TDTestCase: def buildTaosd(self,bPath): # os.system(f"mv {bPath}/build_bak {bPath}/build ") - os.system(f" cd {bPath} && make install ") + os.system(f" cd {bPath} ") def run(self): @@ -106,6 +107,9 @@ class TDTestCase: if distro_id == "alpine": tdLog.info(f"alpine skip compatibility test") return True + if platform.system().lower() == 'windows': + tdLog.info(f"Windows skip compatibility test") + return True bPath = self.getBuildPath() cPath = self.getCfgPath() dbname = "test" @@ -142,6 +146,10 @@ class TDTestCase: tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + cmd = f" LD_LIBRARY_PATH={bPath}/build/lib {bPath}/build/bin/taos -h localhost ;" + if os.system(cmd) == 0: + raise Exception("failed to execute system command. cmd: %s" % cmd) + os.system("pkill taosd") # make sure all the data are saved in disk. self.checkProcessPid("taosd") @@ -152,8 +160,10 @@ class TDTestCase: sleep(1) tdsql=tdCom.newTdSql() print(tdsql) - - + cmd = f" LD_LIBRARY_PATH=/usr/lib taos -h localhost ;" + if os.system(cmd) == 0: + raise Exception("failed to execute system command. cmd: %s" % cmd) + tdsql.query(f"SELECT SERVER_VERSION();") nowServerVersion=tdsql.queryResult[0][0] tdLog.info(f"New server version is {nowServerVersion}") @@ -163,7 +173,7 @@ class TDTestCase: tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers1) + tdsql.checkData(0,0,tableNumbers*recordNumbers1) # tdsql.query("show streams;") # os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") # tdsql.query("show streams;") @@ -177,6 +187,7 @@ class TDTestCase: tdsql.execute("drop database if exists db") tdsql.execute("create database db") tdsql.execute("use db") + tdsql.execute("alter database db wal_retention_period 3600") tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") @@ -192,15 +203,15 @@ class TDTestCase: tdsql.query("describe information_schema.ins_databases;") qRows=tdsql.queryRows comFlag=True - j=0 - while comFlag: + j=0 + while comFlag: for i in range(qRows) : if tdsql.queryResult[i][0] == "retentions" : print("parameters include retentions") comFlag=False break else : - comFlag=True + comFlag=True j=j+1 if j == qRows: print("parameters don't include retentions") diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 720eab74c42eba2dad40500c673d6a7427d3324b..3c4a71c3e4e47baf7c6503f77ea9b588eaebec83 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -93,7 +93,6 @@ class TDTestCase: tdSql.checkEqual(i[2],len(self.perf_list)) tdSql.execute('create table db1.ntb (ts timestamp,c0 int)') tdSql.query(f'select db_name, count(*) from information_schema.ins_tables group by db_name') - print(tdSql.queryResult) for i in tdSql.queryResult: if i[0].lower() == 'information_schema': tdSql.checkEqual(i[1],len(self.ins_list)) @@ -101,9 +100,77 @@ class TDTestCase: tdSql.checkEqual(i[1],len(self.perf_list)) elif i[0].lower() == self.dbname: tdSql.checkEqual(i[1],self.tbnum+1) + + + + def ins_col_check_4096(self): + tdSql.execute('create database db3 vgroups 2 replica 1') + col_str = tdCom.gen_tag_col_str("col", "int",4094) + tdSql.execute(f'create stable if not exists db3.stb (col_ts timestamp, {col_str}) tags (t1 int)') + for i in range(100): + tdLog.info(f"create table db3.ctb{i} using db3.stb tags({i})") + tdSql.execute(f"create table db3.ctb{i} using db3.stb tags({i})") + col_value_str = '1, ' * 4093 + '1' + tdSql.execute(f"insert into db3.ctb{i} values(now,{col_value_str})(now+1s,{col_value_str})(now+2s,{col_value_str})(now+3s,{col_value_str})") + tdSql.query("select * from information_schema.ins_columns") + + tdSql.execute('drop database db3') + def ins_stable_check(self): + tdSql.execute('create database db3 vgroups 2 replica 1') + tbnum = 10 + ctbnum = 10 + for i in range(tbnum): + tdSql.execute(f'create stable db3.stb_{i} (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute(f'create table db3.ntb_{i} (ts timestamp,c0 int)') + for j in range(ctbnum): + tdSql.execute(f"create table db3.ctb_{i}_{j} using db3.stb_{i} tags({j})") + tdSql.query("select stable_name,count(table_name) from information_schema.ins_tables where db_name = 'db3' group by stable_name order by stable_name") + result = tdSql.queryResult + for i in range(len(result)): + if result[i][0] == None: + tdSql.checkEqual(result[0][1],tbnum) + else: + tdSql.checkEqual(result[i][0],f'stb_{i-1}') + tdSql.checkEqual(result[i][1],ctbnum) + + + + def ins_columns_check(self): + tdSql.execute('drop database if exists db2') + tdSql.execute('create database if not exists db2 vgroups 1 replica 1') + for i in range (5): + self.stb4096 = 'create table db2.stb%d (ts timestamp' % (i) + for j in range (4094 - i): + self.stb4096 += ', c%d int' % (j) + self.stb4096 += ') tags (t1 int)' + tdSql.execute(self.stb4096) + for k in range(10): + tdSql.execute("create table db2.ctb_%d_%dc using db2.stb%d tags(%d)" %(i,k,i,k)) + for t in range (2): + tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="SUPER_TABLE"') + tdSql.checkEqual(20465,len(tdSql.queryResult)) + for t in range (2): + tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="CHILD_TABLE"') + tdSql.checkEqual(204650,len(tdSql.queryResult)) + + for i in range (5): + self.ntb4096 = 'create table db2.ntb%d (ts timestamp' % (i) + for j in range (4095 - i): + self.ntb4096 += ', c%d binary(10)' % (j) + self.ntb4096 += ')' + tdSql.execute(self.ntb4096) + for t in range (2): + tdSql.query(f'select * from information_schema.ins_columns where db_name="db2" and table_type=="NORMAL_TABLE"') + tdSql.checkEqual(20470,len(tdSql.queryResult)) + + def run(self): self.prepare_data() self.count_check() + self.ins_columns_check() + # self.ins_col_check_4096() + self.ins_stable_check() + def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/performance_schema.py b/tests/system-test/0-others/performance_schema.py index 70e86009a67fb0fdd3c04e0e98b89539c4ed8ad0..9d2a362254714a2daefc98787a3c1ec236c3e80c 100755 --- a/tests/system-test/0-others/performance_schema.py +++ b/tests/system-test/0-others/performance_schema.py @@ -75,7 +75,7 @@ class TDTestCase: def prepare_data(self): tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") #1 query tdSql.execute(f'use {self.dbname}') #1 query - + tdsql.execute(f"alter database {self.dbname} wal_retention_period 3600") tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) #1 query for i in range(self.tbnum): #self.tbnum query @@ -92,12 +92,12 @@ class TDTestCase: def run(self): tdSqlTran = TDSql() tdSqlTran.init(self.obj.conn.cursor()) - tdSqlTran.execute(f"create database if not exists %s vgroups 20"%(self.obj.transTestDBName)) + tdSqlTran.execute(f"create database if not exists %s vgroups 20 wal_retention_period 3600"%(self.obj.transTestDBName)) tdSqlTran.execute(f"DROP DATABASE %s"%(self.obj.transTestDBName)) def init_tmq_env(self, db, topic): self.conn.execute("drop topic if exists {}".format(topic)) - self.conn.execute("create database if not exists {}".format(db)) + self.conn.execute("create database if not exists {} wal_retention_period 3600".format(db)) self.conn.select_db(db) self.conn.execute( "create stable if not exists stb_sub (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -211,4 +211,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index 3e176fe25162b2053f5d86d83b97bc61e6a354d8..b284605a0e8dc619c5ab004d4a65d022b0eb8946 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -28,7 +28,7 @@ class TDTestCase: self.perf_param = ['apps','connections','consumers','queries','transactions'] self.perf_param_list = ['apps','connections','consumers','queries','trans'] self.dbname = "db" - self.vgroups = 10 + self.vgroups = 4 self.stbname = f'`{tdCom.getLongName(5)}`' self.tbname = f'`{tdCom.getLongName(3)}`' self.db_param = { diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index 9ffa6ca9e60e104cebb51fc1a94eacd7aa494773..43a0400f1880ded44bbbaadfa1a3467a8c3d9775 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -49,8 +49,6 @@ class TDTestCase: #!for bug tdDnodes.stoptaosd(1) sleep(self.delaytime * 5) - if platform.system().lower() == 'windows': - sleep(10) tdSql.error('select server_status()') def run(self): diff --git a/tests/system-test/0-others/tag_index_advance.py b/tests/system-test/0-others/tag_index_advance.py index a8d6cde85a7a67f1b1e23ef06d5b30d7908b02ea..cc942bbf03da12e2c74885577ed9418086795745 100644 --- a/tests/system-test/0-others/tag_index_advance.py +++ b/tests/system-test/0-others/tag_index_advance.py @@ -11,45 +11,42 @@ # -*- coding: utf-8 -*- -from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE -import taos -import sys -import time import os -import socket -import subprocess import random +import socket import string -import random - +import subprocess +import sys +import time +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE -from util.log import * -from util.sql import * +import taos from util.cases import * from util.common import * +from util.dnodes import * +from util.dnodes import TDDnode, TDDnodes +from util.log import * +from util.sql import * from util.sqlset import * -from util.dnodes import * -from util.dnodes import TDDnodes -from util.dnodes import TDDnode # # -------------- util -------------------------- # def pathSize(path): - + total_size = 0 for dirpath, dirnames, filenames in os.walk(path): for i in filenames: - #use join to concatenate all the components of path + # use join to concatenate all the components of path f = os.path.join(dirpath, i) - #use getsize to generate size in bytes and add it to the total size + # use getsize to generate size in bytes and add it to the total size total_size += os.path.getsize(f) - #print(dirpath) + # print(dirpath) - print(" %s %.02f MB"%(path, total_size/1024/1024)) + print(" %s %.02f MB" % (path, total_size/1024/1024)) return total_size - + ''' total = 0 with os.scandir(path) as it: @@ -67,24 +64,27 @@ def pathSize(path): # --------------- cluster ------------------------ # + class MyDnodes(TDDnodes): - def __init__(self ,dnodes_lists): - super(MyDnodes,self).__init__() + def __init__(self, dnodes_lists): + super(MyDnodes, self).__init__() self.dnodes = dnodes_lists # dnode must be TDDnode instance self.simDeployed = False + class TagCluster: noConn = True + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None self.depoly_cluster(5) self.master_dnode = self.TDDnodes.dnodes[0] - self.host=self.master_dnode.cfgDict["fqdn"] - conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) + self.host = self.master_dnode.cfgDict["fqdn"] + conn1 = taos.connect( + self.master_dnode.cfgDict["fqdn"], config=self.master_dnode.cfgDir) tdSql.init(conn1.cursor()) - def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -101,8 +101,7 @@ class TagCluster: break return buildPath - - def depoly_cluster(self ,dnodes_nums): + def depoly_cluster(self, dnodes_nums): testCluster = False valgrind = 0 @@ -126,7 +125,7 @@ class TagCluster: self.TDDnodes.setAsan(tdDnodes.getAsan()) self.TDDnodes.stopAll() for dnode in self.TDDnodes.dnodes: - self.TDDnodes.deploy(dnode.index,{}) + self.TDDnodes.deploy(dnode.index, {}) for dnode in self.TDDnodes.dnodes: self.TDDnodes.starttaosd(dnode.index) @@ -136,7 +135,8 @@ class TagCluster: sql = "" for dnode in self.TDDnodes.dnodes[1:]: # print(dnode.cfgDict) - dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] + dnode_id = dnode.cfgDict["fqdn"] + \ + ":" + dnode.cfgDict["serverPort"] if dnode_first_host == "": dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] @@ -145,18 +145,17 @@ class TagCluster: cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s " cmd += f'"{sql}"' print(cmd) - os.system(cmd) + os.system(cmd) time.sleep(2) tdLog.info(" create cluster done! ") - def getConnection(self, dnode): host = dnode.cfgDict["fqdn"] port = dnode.cfgDict["serverPort"] config_dir = dnode.cfgDir return taos.connect(host=host, port=int(port), config=config_dir) - + def run(self): tdLog.info(" create cluster ok.") @@ -168,22 +167,22 @@ class TagCluster: class PerfDB: def __init__(self): self.sqls = [] - self.spends= [] - + self.spends = [] + # execute def execute(self, sql): - print(f" perfdb execute {sql}") + print(f" perfdb execute {sql}") stime = time.time() ret = tdSql.execute(sql, 1) spend = time.time() - stime - + self.sqls.append(sql) self.spends.append(spend) return ret # query def query(self, sql): - print(f" perfdb query {sql}") + print(f" perfdb query {sql}") start = time.time() ret = tdSql.query(sql, None, 1) spend = time.time() - start @@ -203,9 +202,9 @@ class TDTestCase: self.tagCluster = TagCluster() self.tagCluster.init(conn, logSql, replicaVar) self.lenBinary = 64 - self.lenNchar = 32 - - # column + self.lenNchar = 32 + + # column self.column_dict = { 'ts': 'timestamp', 'col1': 'tinyint', @@ -252,14 +251,14 @@ class TDTestCase: # query def query(self, sql): - return self.dbs[self.cur].query(sql) - - def set_stb_sql(self,stbname,column_dict,tag_dict): + return self.dbs[self.cur].query(sql) + + def set_stb_sql(self, stbname, column_dict, tag_dict): column_sql = '' tag_sql = '' - for k,v in column_dict.items(): + for k, v in column_dict.items(): column_sql += f"{k} {v}, " - for k,v in tag_dict.items(): + for k, v in tag_dict.items(): tag_sql += f"{k} {v}, " create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' return create_stb_sql @@ -268,37 +267,41 @@ class TDTestCase: def create_database(self, dbname, vgroups, replica): sql = f'create database {dbname} vgroups {vgroups} replica {replica}' tdSql.execute(sql) - #tdSql.execute(sql) + # tdSql.execute(sql) tdSql.execute(f'use {dbname}') - + # create stable and child tables def create_table(self, stbname, tbname, count): # create stable - create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) + create_table_sql = self.set_stb_sql( + stbname, self.column_dict, self.tag_dict) tdSql.execute(create_table_sql) # create child table tdLog.info(f" start create {count} child tables.") - for i in range(count): - ti = i % 128 - binTxt = self.random_string(self.lenBinary) - tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"{binTxt}","nch{i}",now' - sql = f'create table {tbname}{i} using {stbname} tags({tags})' - tdSql.execute(sql) - if i > 0 and i % 1000 == 0: - tdLog.info(f" child table count = {i}") + batchSql = "" + batchSize = 5000 + for i in range(int(count/batchSize)): + batchSql = "create table" + for j in range(batchSize): + ti = (i * batchSize + j) % 128 + binTxt = self.random_string(self.lenBinary) + idx = i * batchSize + j + tags = f'{ti},{ti},{idx},{idx},{ti},{ti},{idx},{idx},{idx}.000{idx},{idx}.000{idx},true,"{binTxt}","nch{idx}",now' + sql = f'{tbname}{idx} using {stbname} tags({tags})' + batchSql = batchSql + " " + sql + tdSql.execute(batchSql) + tdLog.info(f" child table count = {i * batchSize}") - tdLog.info(f" end create {count} child tables.") - - # create stable and child tables + def create_tagidx(self, stbname): cnt = -1 for key in self.tag_dict.keys(): # first tag have default index, so skip if cnt == -1: cnt = 0 - continue; + continue sql = f'create index idx_{key} on {stbname} ({key})' tdLog.info(f" sql={sql}") tdSql.execute(sql) @@ -309,11 +312,11 @@ class TDTestCase: def insert_data(self, tbname): # d1 insert 3 rows for i in range(3): - sql = f'insert into {tbname}1(ts,col1) values(now+{i}s,{i});' + sql = f'insert into {tbname}1(ts,col1) values(now+{i}s,{i});' tdSql.execute(sql) # d20 insert 4 for i in range(4): - sql = f'insert into {tbname}20(ts,col1) values(now+{i}s,{i});' + sql = f'insert into {tbname}20(ts,col1) values(now+{i}s,{i});' tdSql.execute(sql) # check show indexs @@ -376,17 +379,17 @@ class TDTestCase: self.query(sql) tdSql.checkRows(4) - # drop child table + def drop_tables(self, tbname, count): # table d1 and d20 have verify data , so can not drop start = random.randint(21, count/2) - end = random.randint(count/2 + 1, count - 1) + end = random.randint(count/2 + 1, count - 1) for i in range(start, end): sql = f'drop table {tbname}{i}' tdSql.execute(sql) - cnt = end - start + 1 - tdLog.info(f' drop table from {start} to {end} count={cnt}') + cnt = end - start + 1 + tdLog.info(f' drop table from {start} to {end} count={cnt}') # drop tag index def drop_tagidx(self, dbname, stbname): @@ -396,11 +399,11 @@ class TDTestCase: # first tag have default index, so skip if cnt == -1: cnt = 0 - continue; + continue sql = f'drop index idx_{key}' tdSql.execute(sql) cnt += 1 - + # check idx result is 0 sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="{dbname}"' tdSql.query(sql) @@ -408,17 +411,19 @@ class TDTestCase: tdLog.info(f' drop {cnt} tag indexs ok.') # show performance - def show_performance(self, count) : - db = self.dbs[0] + def show_performance(self, count): + db = self.dbs[0] db1 = self.dbs[1] cnt = len(db.sqls) cnt1 = len(db1.sqls) - if cnt != len(db1.sqls): - tdLog.info(f" datebase sql count not equal. cnt={cnt} cnt1={cnt1}\n") + if cnt != len(db1.sqls): + tdLog.info( + f" datebase sql count not equal. cnt={cnt} cnt1={cnt1}\n") return False tdLog.info(f" database sql cnt ={cnt}") - print(f" ----------------- performance (child tables = {count})--------------------") + print( + f" ----------------- performance (child tables = {count})--------------------") print(" No time(index) time(no-index) diff(col3-col2) rate(col2/col3) sql") for i in range(cnt): key = db.sqls[i] @@ -427,12 +432,13 @@ class TDTestCase: value1 = db1.spends[i] diff = value1 - value rate = value/value1*100 - print(" %d %.3fs %.3fs %.3fs %d%% %s"%(i+1, value, value1, diff, rate, key)) + print(" %d %.3fs %.3fs %.3fs %d%% %s" % ( + i+1, value, value1, diff, rate, key)) print(" --------------------- end ------------------------") - return True - + return True + def show_diskspace(self): - #calc + # calc selfPath = os.path.dirname(os.path.realpath(__file__)) projPath = "" if ("community" in selfPath): @@ -451,43 +457,41 @@ class TDTestCase: idx_size = vnode2_size + vnode3_size noidx_size = vnode4_size + vnode5_size - print(" index = %.02f M"%(idx_size/1024/1024)) - print(" no-index = %.02f M"%(noidx_size/1024/1024)) - print(" index/no-index = %.2f multiple"%(idx_size/noidx_size)) + print(" index = %.02f M" % (idx_size/1024/1024)) + print(" no-index = %.02f M" % (noidx_size/1024/1024)) + print(" index/no-index = %.2f multiple" % (idx_size/noidx_size)) print(" -------------------- end ------------------------") - - - # main + def testdb(self, dbname, stable, tbname, count, createidx): # cur if createidx: self.cur = 0 - else : + else: self.cur = 1 - # do + # do self.create_database(dbname, 2, 1) self.create_table(stable, tbname, count) - if(createidx): - self.create_tagidx(stable) + if (createidx): + self.create_tagidx(stable) self.insert_data(tbname) - if(createidx): - self.show_tagidx(dbname,stable) + if (createidx): + self.show_tagidx(dbname, stable) self.query_tagidx(stable) - #self.drop_tables(tbname, count) - #if(createidx): + # self.drop_tables(tbname, count) + # if(createidx): # self.drop_tagidx(dbname, stable) # query after delete , expect no crash - #self.query_tagidx(stable) + # self.query_tagidx(stable) tdSql.execute(f'flush database {dbname}') # run def run(self): self.tagCluster.run() - + # var dbname = "tagindex" dbname1 = dbname + "1" @@ -511,10 +515,10 @@ class TDTestCase: self.show_diskspace() - def stop(self): self.tagCluster.stop() tdLog.success("%s successfully executed" % __file__) + tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/tag_index_basic.py b/tests/system-test/0-others/tag_index_basic.py index 96c3dca016df5c9f7bc995951bff0d01cbadd23f..72ed559ffd94f808a906d0547c8b10f2ea6f1500 100644 --- a/tests/system-test/0-others/tag_index_basic.py +++ b/tests/system-test/0-others/tag_index_basic.py @@ -24,7 +24,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.setsql = TDSetSql() self.column_dict = { 'ts': 'timestamp', diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 944ff52d5bacf386369c80b8561229f48d554008..8094c4e0f555dcd7a5a7340c439a0a34948b212d 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -292,7 +292,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "4" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) sql = "create table db3.stb (ts timestamp, f int) tags (t int)" diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 2125b5eebe448b4ac044158f7157cd689eaec29e..3b9eb66705407e80a9e130680675a618283679e0 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -4,6 +4,7 @@ import sys import time from datetime import datetime import socket +import psutil import os import platform if platform.system().lower() == 'windows': @@ -67,19 +68,25 @@ class TDTestCase: return buildPath def get_process_pid(self,processname): - #origin artical link：https://blog.csdn.net/weixin_45623536/article/details/122099062 - process_info_list = [] - process = os.popen('ps -A | grep %s'% processname) - process_info = process.read() - for i in process_info.split(' '): - if i != "": - process_info_list.append(i) - print(process_info_list) - if len(process_info_list) != 0 : - pid = int(process_info_list[0]) - else : - pid = 0 - return pid + if platform.system().lower() == 'windows': + pids = psutil.process_iter() + for pid in pids: + if(pid.name() == processname): + return pid.pid + return 0 + else: + process_info_list = [] + process = os.popen('ps -A | grep %s'% processname) + process_info = process.read() + for i in process_info.split(' '): + if i != "": + process_info_list.append(i) + print(process_info_list) + if len(process_info_list) != 0 : + pid = int(process_info_list[0]) + else : + pid = 0 + return pid def checkAndstopPro(self,processName,startAction): i = 1 @@ -88,23 +95,29 @@ class TDTestCase: taosdPid=self.get_process_pid(processName) if taosdPid != 0 and taosdPid != "" : tdLog.info("stop taosd %s ,kill pid :%s "%(startAction,taosdPid)) - os.system("kill -9 %d"%taosdPid) + os.system("kill -9 %d"%taosdPid) break else: tdLog.info( "wait start taosd ,times: %d "%i) time.sleep(1) i+= 1 else : - tdLog.exit("taosd %s is not running "%startAction) + tdLog.exit("taosd %s is not running "%startAction) def taosdCommandStop(self,startAction,taosdCmdRun): processName="taosd" + if platform.system().lower() == 'windows': + processName="taosd.exe" taosdCmd = taosdCmdRun + startAction tdLog.printNoPrefix("%s"%taosdCmd) - logTime=datetime.now().strftime('%Y%m%d_%H%M%S_%f') - os.system(f"nohup {taosdCmd} > {logTime}.log 2>&1 & ") - self.checkAndstopPro(processName,startAction) - os.system(f"rm -rf {logTime}.log") + if platform.system().lower() == 'windows': + cmd = f"mintty -h never {taosdCmd}" + os.system(cmd) + else: + logTime=datetime.now().strftime('%Y%m%d_%H%M%S_%f') + os.system(f"nohup {taosdCmd} > {logTime}.log 2>&1 & ") + self.checkAndstopPro(processName,startAction) + os.system(f"rm -rf {logTime}.log") def taosdCommandExe(self,startAction,taosdCmdRun): @@ -116,7 +129,7 @@ class TDTestCase: # database\stb\tb\chiild-tb\rows\topics tdSql.execute("create user testpy pass 'testpy'") tdSql.execute("drop database if exists db0;") - tdSql.execute("create database db0;") + tdSql.execute("create database db0 wal_retention_period 3600;") tdSql.execute("use db0;") tdSql.execute("create table if not exists db0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned);") tdSql.execute("create table db0.ct1 using db0.stb tags(1000);") @@ -132,14 +145,14 @@ class TDTestCase: #stream tdSql.execute("drop database if exists source_db;") - tdSql.query("create database source_db vgroups 3;") + tdSql.query("create database source_db vgroups 3 wal_retention_period 3600;") tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS startts, min(k), max(k), sum(k) from source_db.stb interval(10m);") - #TD-19944 -Q=3 + #TD-19944 -Q=3 tdsqlN=tdCom.newTdSql() tdsqlN.query("select * from source_db.stb") @@ -186,7 +199,11 @@ class TDTestCase: startAction=" -a jsonFile:./taosdCaseTmp.json" tdLog.printNoPrefix("================================ parameter: %s"%startAction) - os.system("echo \'{\"queryPolicy\":\"3\"}\' > taosdCaseTmp.json") + + if platform.system().lower() == 'windows': + os.system("echo {\"queryPolicy\":\"3\"} > taosdCaseTmp.json") + else: + os.system("echo \'{\"queryPolicy\":\"3\"}\' > taosdCaseTmp.json") self.taosdCommandStop(startAction,taosdCmdRun) startAction = " -a jsonFile:./taosdCaseTmp.json -C " @@ -206,12 +223,12 @@ class TDTestCase: self.taosdCommandStop(startAction,taosdCmdRun) - startAction=" -E taosdCaseTmp/.env" + startAction=f" -E taosdCaseTmp{os.sep}.env" tdLog.printNoPrefix("================================ parameter: %s"%startAction) - os.system(" mkdir -p taosdCaseTmp ") - os.system("echo \'TAOS_QUERY_POLICY=3\' > taosdCaseTmp/.env ") + os.system(" mkdir -p taosdCaseTmp ") + os.system("echo TAOS_QUERY_POLICY=3 > taosdCaseTmp/.env ") self.taosdCommandStop(startAction,taosdCmdRun) - os.system(" rm -rf taosdCaseTmp ") + os.system(" rm -rf taosdCaseTmp ") startAction = " -V" tdLog.printNoPrefix("================================ parameter: %s"%startAction) diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index bc5d276faa92e2ec6226a7b36c4ac5b0b3d46420..3b6cb105099dfacebc705d347b320005e4b3d109 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -181,7 +181,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "4" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) diff --git a/tests/system-test/0-others/testRoll.py b/tests/system-test/0-others/testRoll.py index 56e5b3630afe9c875be74dcbbdce52d024c9a22e..c5489146dc4edff7c77a6e2888925bb284dd3fc3 100644 --- a/tests/system-test/0-others/testRoll.py +++ b/tests/system-test/0-others/testRoll.py @@ -13,7 +13,7 @@ def init_tmq_env(db, topic): conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} replica 1 ".format(db)) + conn.execute("create database if not exists {} replica 1 wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -37,7 +37,7 @@ def init_tmq_rest_env(db, topic): conn.execute("drop topic if exists {}".format(topic)) conn.execute("drop database if exists {}".format(db)) - conn.execute("create database if not exists {} replica 3 ".format(db)) + conn.execute("create database if not exists {} replica 3 wal_retention_period 3600".format(db)) conn.select_db(db) conn.execute( "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") @@ -80,4 +80,4 @@ if __name__ == '__main__': val = res.value() for block in val: - print(block.fetchall()) \ No newline at end of file + print(block.fetchall()) diff --git a/tests/system-test/0-others/tmqBasic.json b/tests/system-test/0-others/tmqBasic.json index 24e815708aeefcb18452cfd20670aba2e185e589..d716bff3ac254c3d1106c25bd0a45ce8f43603d4 100644 --- a/tests/system-test/0-others/tmqBasic.json +++ b/tests/system-test/0-others/tmqBasic.json @@ -14,7 +14,6 @@ "auto.offset.reset": "earliest", "enable.auto.commit": "true", "auto.commit.interval.ms": 1000, - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "false", "topic_list": [ diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 15253df0c4eda40378ed2c875f01a8448f73003d..78020cb9586e6c59f7c0f84e5aeacdbb596b421c 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -146,7 +146,7 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -171,7 +171,7 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -191,20 +191,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -226,13 +226,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -375,14 +375,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -401,9 +401,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -412,20 +412,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -562,7 +562,7 @@ class TDTestCase: # create function without buffer tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) - tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.execute("create function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -570,7 +570,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create function db as '%s' outputtype int "%self.libudf1) tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") @@ -619,9 +619,9 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int "%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int "%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) diff --git a/tests/system-test/0-others/udf_cfg1.py b/tests/system-test/0-others/udf_cfg1.py index 35d43ea9b73eedc7b9432cf0590d90927ad2932c..913e5fcca14283579f6d2cc72d9674b448fc0c2b 100644 --- a/tests/system-test/0-others/udf_cfg1.py +++ b/tests/system-test/0-others/udf_cfg1.py @@ -148,7 +148,7 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -173,7 +173,7 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index 869cb098e2e0415acf931179dc4e6887aa781f4a..b535b4f626cdf92de1a406185d9c1af807ea6c7c 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -148,7 +148,7 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -173,7 +173,7 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -564,7 +564,7 @@ class TDTestCase: # create function without buffer tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) - tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.execute("create function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -572,7 +572,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create function db as '%s' outputtype int "%self.libudf1) tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") @@ -621,9 +621,9 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 65dad6400041e8c52d39d0ff4e484bdc29099894..f467e802ac5d5f8adb15ef6b705d273e744270a3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -148,7 +148,7 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -173,7 +173,7 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -574,7 +574,7 @@ class TDTestCase: # create function without buffer tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) - tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.execute("create function udf2 as '%s' outputtype double "%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -582,7 +582,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create function db as '%s' outputtype int "%self.libudf1) tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") @@ -631,9 +631,9 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index dae707520f55e2e2c6b02c60e5d9015f8c613d3d..61b6a4ea684f6150834cfc1e7c6ce1dc03a8b91b 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -145,7 +145,7 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -170,7 +170,7 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) # create aggregate functions @@ -190,20 +190,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -212,7 +212,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -225,13 +225,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -374,14 +374,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -400,9 +400,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -411,20 +411,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -468,12 +468,12 @@ class TDTestCase: tdSql.checkData(1,0,1) tdSql.checkData(1,1,1) tdSql.checkData(1,2,1.110000000) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") tdSql.checkData(1,0,8) tdSql.checkData(1,1,88.880000000) - tdSql.checkData(1,2,88) + tdSql.checkData(1,2,1) tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") tdSql.checkRows(22) @@ -561,7 +561,7 @@ class TDTestCase: # create function without buffer tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) - tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.execute("create function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -569,7 +569,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create function db as '%s' outputtype int "%self.libudf1) tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") @@ -618,9 +618,9 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) - tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create function max as '%s' outputtype int"%self.libudf1) tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) diff --git a/tests/system-test/0-others/udfpy/af_count.py b/tests/system-test/0-others/udfpy/af_count.py new file mode 100644 index 0000000000000000000000000000000000000000..ce29abca13866f9f311d1e868a5df68c71c5f735 --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_count.py @@ -0,0 +1,23 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(0) + +def finish(buf): + count = pickle.loads(buf) + return count + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + count = pickle.loads(buf) + for i in range(rows): + val = datablock.data(i, 0) + if val is not None: + count += 1 + return pickle.dumps(count) \ No newline at end of file diff --git a/tests/system-test/0-others/udfpy/af_min.py b/tests/system-test/0-others/udfpy/af_min.py new file mode 100644 index 0000000000000000000000000000000000000000..9f1aadf4145c7bb585040d1f05640c913aab2aee --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_min.py @@ -0,0 +1,30 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps([]) + +def finish(buf): + mins = pickle.loads(buf) + min_val = None + for min in mins: + if min_val is None or (min is not None and min < min_val): + min_val = min + return min_val + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + mins = pickle.loads(buf) + min = None + for i in range(rows): + val = datablock.data(i, 0) + if min is None or (val is not None and val < min) : + min = val + if min is not None: + mins.append(min) + return pickle.dumps(mins) diff --git a/tests/system-test/0-others/udfpy/af_null.py b/tests/system-test/0-others/udfpy/af_null.py new file mode 100644 index 0000000000000000000000000000000000000000..230eac6888245444a4f902274a83bb5671626515 --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_null.py @@ -0,0 +1,19 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps([]) + +def finish(buf): + return None + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + mins = pickle.loads(buf) + mins.append(None) + return pickle.dumps(mins) diff --git a/tests/system-test/0-others/udfpy/af_sum.py b/tests/system-test/0-others/udfpy/af_sum.py new file mode 100644 index 0000000000000000000000000000000000000000..8b88aba56ca08f64249f85a4df596d9fd204b05a --- /dev/null +++ b/tests/system-test/0-others/udfpy/af_sum.py @@ -0,0 +1,26 @@ +import pickle + +def init(): + pass + +def destroy(): + pass + +def start(): + return pickle.dumps(None) + +def finish(buf): + sum = pickle.loads(buf) + return sum + +def reduce(datablock, buf): + (rows, cols) = datablock.shape() + sum = pickle.loads(buf) + for i in range(rows): + val = datablock.data(i, 0) + if val is not None: + if sum is None: + sum = val + else: + sum += val + return pickle.dumps(sum) diff --git a/tests/system-test/0-others/udfpy/sf_concat_nch.py b/tests/system-test/0-others/udfpy/sf_concat_nch.py new file mode 100644 index 0000000000000000000000000000000000000000..84d8eb2c961c11282784f81b6206a36fba0f2edc --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_concat_nch.py @@ -0,0 +1,27 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + row = [] + for j in range(ncols): + val = block.data(i, j) + if val is None: + row = None + break + row.append(val.decode('utf_32_le')) + if row is None: + results.append(None) + else: + row_str = ''.join(row) + results.append(row_str.encode('utf_32_le')) + return results + + diff --git a/tests/system-test/0-others/udfpy/sf_concat_var.py b/tests/system-test/0-others/udfpy/sf_concat_var.py new file mode 100644 index 0000000000000000000000000000000000000000..fc8292c718d5c8def7cebc87ee7938b2c7bd3849 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_concat_var.py @@ -0,0 +1,26 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + row = [] + for j in range(ncols): + val = block.data(i, j) + if val is None: + row = None + break + row.append(val.decode('utf-8')) + if row is None: + results.append(None) + else: + results.append(''.join(row)) + return results + + diff --git a/tests/system-test/0-others/udfpy/sf_multi_args.py b/tests/system-test/0-others/udfpy/sf_multi_args.py new file mode 100644 index 0000000000000000000000000000000000000000..1026661d8d127eeb52b6b79f9b81dd29ccc24ba5 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_multi_args.py @@ -0,0 +1,23 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (nrows, ncols) = block.shape() + results = [] + for i in range(nrows): + rows = [] + for j in range(ncols): + val = block.data(i, j) + if type(val) is bytes: + rows.append(val.decode('utf-8')) + else: + rows.append(repr(val)) + results.append(','.join(rows)) + return results + diff --git a/tests/system-test/0-others/udfpy/sf_null.py b/tests/system-test/0-others/udfpy/sf_null.py new file mode 100644 index 0000000000000000000000000000000000000000..c22ca95b19f93d2d494a4890154923626a6a445a --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_null.py @@ -0,0 +1,16 @@ + +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (rows, cols) = block.shape() + results = [] + for i in range(rows): + results.append(None) + return results \ No newline at end of file diff --git a/tests/system-test/0-others/udfpy/sf_origin.py b/tests/system-test/0-others/udfpy/sf_origin.py new file mode 100644 index 0000000000000000000000000000000000000000..9158e044d2defb118683f56b889bf95bb7d04ee6 --- /dev/null +++ b/tests/system-test/0-others/udfpy/sf_origin.py @@ -0,0 +1,15 @@ +# init +def init(): + pass + +# destroy +def destroy(): + pass + +# return origin column one value +def process(block): + (rows, cols) = block.shape() + results = [] + for i in range(rows): + results.append(block.data(i,0)) + return results diff --git a/tests/system-test/0-others/udfpy_main.py b/tests/system-test/0-others/udfpy_main.py new file mode 100644 index 0000000000000000000000000000000000000000..916b032edbde72a4ddc011608b924f95bb035c58 --- /dev/null +++ b/tests/system-test/0-others/udfpy_main.py @@ -0,0 +1,464 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +import random +import os +import subprocess + + +class PerfDB: + def __init__(self): + self.sqls = [] + self.spends = [] + + # execute + def execute(self, sql): + print(f" perfdb execute {sql}") + stime = time.time() + ret = tdSql.execute(sql, 1) + spend = time.time() - stime + + self.sqls.append(sql) + self.spends.append(spend) + return ret + + # query + def query(self, sql): + print(f" perfdb query {sql}") + start = time.time() + ret = tdSql.query(sql, None, 1) + spend = time.time() - start + self.sqls.append(sql) + self.spends.append(spend) + return ret + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + # udf path + self.udf_path = os.path.dirname(os.path.realpath(__file__)) + "/udfpy" + + + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(120)', + 'col13': 'nchar(100)', + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(120)', + 't13': 'nchar(100)', + } + + def set_stb_sql(self,stbname,column_dict,tag_dict): + column_sql = '' + tag_sql = '' + for k,v in column_dict.items(): + column_sql += f"{k} {v}, " + for k,v in tag_dict.items(): + tag_sql += f"{k} {v}, " + create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' + return create_stb_sql + + # create stable and child tables + def create_table(self, stbname, tbname, count): + tdSql.execute("create database db wal_retention_period 4") + tdSql.execute('use db') + self.child_count = count + self.stbname = stbname + self.tbname = tbname + + # create stable + create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) + tdSql.execute(create_table_sql) + + batch_size = 1000 + # create child table + for i in range(count): + ti = i % 128 + tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"var{i}","nch{i}"' + sql = f'create table {tbname}{i} using {stbname} tags({tags});' + tdSql.execute(sql) + if i % batch_size == 0: + tdLog.info(f" create child table {i} ...") + + tdLog.info(f" create {count} child tables ok.") + + # create with dicts + def create_sf_dicts(self, dicts, filename): + for fun_name, out_type in dicts.items(): + sql = f' create function {fun_name} as "{self.udf_path}/{filename}" outputtype {out_type} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + # create_udfpy_function + def create_scalar_udfpy(self): + # scalar funciton + self.scalar_funs = { + 'sf0': 'timestamp', + 'sf1': 'tinyint', + 'sf2': 'smallint', + 'sf3': 'int', + 'sf4': 'bigint', + 'sf5': 'tinyint unsigned', + 'sf6': 'smallint unsigned', + 'sf7': 'int unsigned', + 'sf8': 'bigint unsigned', + 'sf9': 'float', + 'sf10': 'double', + 'sf11': 'bool', + 'sf12': 'varchar(120)', + 'sf13': 'nchar(100)' + } + # agg function + self.agg_funs = { + 'af1': 'tinyint', + 'af2': 'smallint', + 'af3': 'int', + 'af4': 'bigint', + 'af5': 'tinyint unsigned', + 'af6': 'smallint unsigned', + 'af7': 'int unsigned', + 'af8': 'bigint unsigned', + 'af9': 'float', + 'af10': 'double', + 'af11': 'bool', + 'af12': 'varchar(120)', + 'af13': 'nchar(100)', + 'af14': 'timestamp' + } + + # multi_args + self.create_sf_dicts(self.scalar_funs, "sf_origin.py") + fun_name = "sf_multi_args" + self.create_udf_sf(fun_name, f'{fun_name}.py', "binary(1024)") + + # all type check null + for col_name, col_type in self.column_dict.items(): + self.create_udf_sf(f"sf_null_{col_name}", "sf_null.py", col_type) + + # concat + fun_name = "sf_concat_var" + self.create_udf_sf(fun_name, f'{fun_name}.py', "varchar(1024)") + fun_name = "sf_concat_nch" + self.create_udf_sf(fun_name, f'{fun_name}.py', "nchar(1024)") + + + # fun_name == fun_name.py + def create_udf_sf(self, fun_name, file_name, out_type): + sql = f'create function {fun_name} as "{self.udf_path}/{file_name}" outputtype {out_type} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + def create_udf_af(self, fun_name, file_name, out_type, bufsize): + sql = f'create aggregate function {fun_name} as "{self.udf_path}/{file_name}" outputtype {out_type} bufsize {bufsize} language "Python" ' + tdSql.execute(sql) + tdLog.info(sql) + + + # sql1 query result eual with sql2 + def verify_same_result(self, sql1, sql2): + # query + result1 = tdSql.getResult(sql1) + tdSql.query(sql2) + + for i, row in enumerate(result1): + for j , val in enumerate(row): + tdSql.checkData(i, j, result1[i][j]) + + # same value like select col1, udf_fun1(col1) from st + def verify_same_value(self, sql, col=0): + tdSql.query(sql) + nrows = tdSql.getRows() + for i in range(nrows): + val = tdSql.getData(i, col) + tdSql.checkData(i, col + 1, val) + + # verify multi values + def verify_same_multi_values(self, sql): + tdSql.query(sql) + nrows = tdSql.getRows() + for i in range(nrows): + udf_val = tdSql.getData(i, 0) + vals = udf_val.split(',') + for j,val in enumerate(vals, 1): + tdSql.checkData(i, j, val) + + # query multi-args + def query_multi_args(self): + cols = list(self.column_dict.keys()) + list(self.tag_dict.keys()) + cols.remove("col13") + cols.remove("t13") + cols.remove("ts") + ncols = len(cols) + print(cols) + for i in range(2, ncols): + sample = random.sample(cols, i) + print(sample) + cols_name = ','.join(sample) + sql = f'select sf_multi_args({cols_name}),{cols_name} from {self.stbname} limit 10' + self.verify_same_multi_values(sql) + tdLog.info(sql) + + + # query_udfpy + def query_scalar_udfpy(self): + # col + for col_name, col_type in self.column_dict.items(): + for fun_name, out_type in self.scalar_funs.items(): + if col_type == out_type : + sql = f'select {col_name}, {fun_name}({col_name}) from {self.stbname} limit 10' + tdLog.info(sql) + self.verify_same_value(sql) + sql = f'select * from (select {col_name} as a, {fun_name}({col_name}) as b from {self.stbname} limit 100) order by b,a desc' + tdLog.info(sql) + self.verify_same_value(sql) + + # multi-args + self.query_multi_args() + + # all type check null + for col_name, col_type in self.column_dict.items(): + fun_name = f"sf_null_{col_name}" + sql = f'select {fun_name}({col_name}) from {self.stbname}' + tdSql.query(sql) + if col_type != "timestamp": + tdSql.checkData(0, 0, "None") + else: + val = tdSql.getData(0, 0) + if val is not None: + tdLog.exit(f" check {sql} not expect None.") + + # concat + sql = f'select sf_concat_var(col12, t12), concat(col12, t12) from {self.stbname} limit 1000' + self.verify_same_value(sql) + sql = f'select sf_concat_nch(col13, t13), concat(col13, t13) from {self.stbname} limit 1000' + self.verify_same_value(sql) + + # create aggregate + def create_aggr_udfpy(self): + + bufsize = 200 * 1024 + # all type check null + for col_name, col_type in self.column_dict.items(): + self.create_udf_af(f"af_null_{col_name}", "af_null.py", col_type, bufsize) + + # min + file_name = "af_min.py" + fun_name = "af_min_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_min_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + + # sum + file_name = "af_sum.py" + fun_name = "af_sum_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_sum_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + fun_name = "af_sum_bigint" + self.create_udf_af(fun_name, file_name, f"bigint", bufsize) + + # count + file_name = "af_count.py" + fun_name = "af_count_float" + self.create_udf_af(fun_name, file_name, f"float", bufsize) + fun_name = "af_count_int" + self.create_udf_af(fun_name, file_name, f"int", bufsize) + fun_name = "af_count_bigint" + self.create_udf_af(fun_name, file_name, f"bigint", bufsize) + + + # query aggregate + def query_aggr_udfpy(self) : + # all type check null + for col_name, col_type in self.column_dict.items(): + fun_name = f"af_null_{col_name}" + sql = f'select {fun_name}({col_name}) from {self.stbname}' + tdSql.query(sql) + if col_type != "timestamp": + tdSql.checkData(0, 0, "None") + else: + val = tdSql.getData(0, 0) + if val is not None: + tdLog.exit(f" check {sql} not expect None.") + + # min + sql = f'select min(col3), af_min_int(col3) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select min(col7), af_min_int(col7) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select min(col9), af_min_float(col9) from {self.stbname}' + self.verify_same_value(sql) + + # sum + sql = f'select sum(col1), af_sum_int(col1) from d0' + self.verify_same_value(sql) + sql = f'select sum(col3), af_sum_bigint(col3) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select sum(col9), af_sum_float(col9) from {self.stbname}' + self.verify_same_value(sql) + + # count + sql = f'select count(col1), af_count_int(col1) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select count(col7), af_count_bigint(col7) from {self.stbname}' + self.verify_same_value(sql) + sql = f'select count(col8), af_count_float(col8) from {self.stbname}' + self.verify_same_value(sql) + + # nest + sql = f'select a+1000,b+1000 from (select count(col8) as a, af_count_float(col8) as b from {self.stbname})' + self.verify_same_value(sql) + # group by + sql = f'select a+1000,b+1000 from (select count(col8) as a, af_count_float(col8) as b from {self.stbname} group by tbname)' + self.verify_same_value(sql) + # two filed expr + sql = f'select sum(col1+col2),af_sum_float(col1+col2) from {self.stbname};' + self.verify_same_value(sql) + # interval + sql = f'select af_sum_float(col2+col3),sum(col3+col2) from {self.stbname} interval(1s)' + self.verify_same_value(sql) + + + # insert to child table d1 data + def insert_data(self, tbname, rows): + ts = 1670000000000 + values = "" + batch_size = 500 + child_name = "" + for i in range(self.child_count): + for j in range(rows): + tj = j % 128 + cols = f'{tj},{tj},{j},{j},{tj},{tj},{j},{j},{j}.000{j},{j}.000{j},true,"var{j}","nch{j}涛思数据codepage is utf_32_le"' + value = f'({ts+j},{cols})' + if values == "": + values = value + else: + values += f",{value}" + if j % batch_size == 0 or j + 1 == rows: + sql = f'insert into {tbname}{i} values {values};' + tdSql.execute(sql) + tdLog.info(f" child table={i} rows={j} insert data.") + values = "" + + # partial columns upate + sql = f'insert into {tbname}0(ts, col1, col9, col11) values(now, 100, 200, 0)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts, col2, col5, col8) values(now, 100, 200, 300)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts, col3, col7, col13) values(now, null, null, null)' + tdSql.execute(sql) + sql = f'insert into {tbname}0(ts) values(now)' + tdSql.execute(sql) + tdLog.info(f" insert {rows} to child table {self.child_count} .") + + + # create stream + def create_stream(self): + sql = f"create stream ma into sta subtable(concat('sta_',tbname)) \ + as select _wstart,count(col1),af_count_bigint(col1) from {self.stbname} partition by tbname interval(1s);" + tdSql.execute(sql) + tdLog.info(sql) + + # query stream + def verify_stream(self): + sql = f"select * from sta limit 10" + self.verify_same_value(sql, 1) + + # create tmq + def create_tmq(self): + sql = f"create topic topa as select concat(col12,t12),sf_concat_var(col12,t12) from {self.stbname};" + tdSql.execute(sql) + tdLog.info(sql) + + def install_taospy(self): + tdLog.info("install taospyudf...") + packs = ["taospyudf"] + for pack in packs: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-i', 'https://pypi.org/simple', '-U', pack]) + tdLog.info("call ldconfig...") + os.system("ldconfig") + tdLog.info("install taospyudf successfully.") + + # run + def run(self): + self.install_taospy() + + # var + stable = "meters" + tbname = "d" + count = 10 + rows = 5000 + # do + self.create_table(stable, tbname, count) + + # create + self.create_scalar_udfpy() + self.create_aggr_udfpy() + + # create stream + self.create_stream() + + # create tmq + self.create_tmq() + + # insert data + self.insert_data(tbname, rows) + + # query + self.query_scalar_udfpy() + self.query_aggr_udfpy() + + # show performance + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index d27892b588a832ff1edeb581b1975e9771510383..7b5578ba45f66bd3a993f0ce2ee666dcf802d3b3 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -444,7 +444,7 @@ class TDTestCase: self.__grant_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__grant_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"GRANT {self.__privilege[0]} ON * TO {self.__user_list[0]}" , - f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" , + # f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" , ] def __revoke_err(self): @@ -456,7 +456,7 @@ class TDTestCase: self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" , - f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" , + # f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" , ] def test_grant_err(self): diff --git a/tests/system-test/0-others/user_manage.py b/tests/system-test/0-others/user_manage.py index 6f90a2873afa4e08c1ec4b72457b6cba9f232d48..1e33d4bb1c628bd43569b68d8f7fc07e68f2bec8 100644 --- a/tests/system-test/0-others/user_manage.py +++ b/tests/system-test/0-others/user_manage.py @@ -115,6 +115,7 @@ class TDTestCase: jiacy0_read_conn = taos.connect(user='jiacy0_read', password='123') jiacy0_write_conn = taos.connect(user='jiacy0_write', password='123') jiacy0_none_conn = taos.connect(user='jiacy0_none', password='123') + tdSql.execute('alter database db wal_retention_period 3600') tdSql.execute('create topic root_db as select * from db.stb') for user in [jiacy1_all_conn, jiacy1_read_conn, jiacy0_all_conn, jiacy0_read_conn]: user.execute(f'create topic db_jiacy as select * from db.stb') diff --git a/tests/system-test/0-others/walFileIdex.py b/tests/system-test/0-others/walFileIdex.py index cd34c7e5e30315df92b0f0aa4c0dda7c460b450a..f8309519cd77a63d863105a8a1cd2227021a4ff4 100644 --- a/tests/system-test/0-others/walFileIdex.py +++ b/tests/system-test/0-others/walFileIdex.py @@ -58,7 +58,7 @@ class TDTestCase: #stream tdSql.execute("drop database if exists source_db;") - tdSql.query("create database source_db vgroups 3;") + tdSql.query("create database source_db vgroups 3 wal_retention_period 3600;") tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") diff --git a/tests/system-test/0-others/walRetention.py b/tests/system-test/0-others/walRetention.py new file mode 100644 index 0000000000000000000000000000000000000000..2b340b79697f874a1e300970fac74f90529f0cc9 --- /dev/null +++ b/tests/system-test/0-others/walRetention.py @@ -0,0 +1,472 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# The option for wal_retetion_period and wal_retention_size is work well +# + +import taos +from taos.tmq import Consumer + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + + +import os +import threading +import json +import time +from datetime import date +from datetime import datetime +from datetime import timedelta +from os import path + + +# +# -------------- util -------------------------- +# +def pathSize(path): + + total_size = 0 + for dirpath, dirnames, filenames in os.walk(path): + for i in filenames: + # use join to concatenate all the components of path + f = os.path.join(dirpath, i) + # use getsize to generate size in bytes and add it to the total size + total_size += os.path.getsize(f) + # print(dirpath) + + print(" %s %.02f MB" % (path, total_size/1024/1024)) + return total_size + + +# load json from file +def jsonFromFile(jsonFile): + fp = open(jsonFile) + return json.load(fp) + + +# +# ----------------- class ------------------ +# + +# wal file object +class WalFile: + def __init__(self, pathFile, fileName): + self.mtime = os.path.getmtime(pathFile) + self.startVer = int(fileName) + self.fsize = os.path.getsize(pathFile) + self.endVer = -1 + self.pathFile = pathFile + + def needDelete(self, delTsLine): + return True + +# VNode object +class VNode : + # init + def __init__(self, dnodeId, path, walPeriod, walSize, walStayRange): + self.path = path + self.dnodeId = dnodeId + self.vgId = 0 + self.snapVer = 0 + self.firstVer = 0 + self.lastVer = -1 + self.walPeriod = walPeriod + self.walSize = walSize + self.walStayRange = walStayRange + self.walFiles = [] + self.load(path) + + # load + def load(self, path): + # load wal + walPath = os.path.join(path, "wal") + metaFile = "" + with os.scandir(walPath) as items: + for item in items: + if item.is_file(): + fileName, fileExt = os.path.splitext(item.name) + pathFile = os.path.join(walPath, item) + if fileExt == ".log": + self.walFiles.append(WalFile(pathFile, fileName)) + elif fileExt == "": + if fileName[:8] == "meta-ver": + metaFile = pathFile + # load config + tdLog.info(f' meta-ver file={metaFile}') + if metaFile != "": + jsonVer = jsonFromFile(metaFile) + metaNode = jsonVer["meta"] + self.snapVer = int(metaNode["snapshotVer"]) + self.firstVer = int(metaNode["firstVer"]) + self.lastVer = int(metaNode["lastVer"]) + + # sort with startVer + self.walFiles = sorted(self.walFiles, key=lambda x : x.startVer, reverse=True) + # set endVer + startVer = -1 + for walFile in self.walFiles: + if startVer == -1: + startVer = walFile.startVer + continue + walFile.endVer = startVer - 1 + startVer = walFile.startVer + + # print total + tdLog.info(f" ---- dnode{self.dnodeId} snapVer={self.snapVer} firstVer={self.firstVer} lastVer={self.lastVer} {self.path} --------") + for walFile in self.walFiles: + mt = datetime.fromtimestamp(walFile.mtime) + tdLog.info(f" {walFile.pathFile} {mt} startVer={walFile.startVer} endVer={walFile.endVer}") + + # snapVer compare + def canDelete(self, walFile): + if walFile.endVer == -1: + # end file + return False + + # check snapVer + ret = False + if self.snapVer > walFile.endVer: + ret = True + + # check stayRange + if self.lastVer != -1 and ret: + # first wal file ignore + if walFile.startVer == self.firstVer: + tdLog.info(f" {walFile.pathFile} can del, but is first. snapVer={self.snapVer} firstVer={self.firstVer}") + return False + + # ver in stay range + smallVer = self.snapVer - self.walStayRange -1 + if walFile.startVer >= smallVer: + tdLog.info(f" {walFile.pathFile} can del, but range not arrived. snapVer={self.snapVer} smallVer={smallVer}") + return False + + return ret + + # get log size + def getWalsSize(self): + size = 0 + for walFile in self.walFiles: + size += walFile.fsize + + return size + + # vnode + def check_retention(self): + # + # check period + # + delta = self.walPeriod + if self.walPeriod == 0: + delta += 1 * 60 # delete after 1 minutes + elif self.walPeriod < 3600: + delta += 3 * 60 # 5 minutes + else: + delta += 5 * 60 # 10 minutes + + delTsLine = datetime.now() - timedelta(seconds = delta) + delTs = delTsLine.timestamp() + for walFile in self.walFiles: + mt = datetime.fromtimestamp(walFile.mtime) + info = f" {walFile.pathFile} mt={mt} line={delTsLine} start={walFile.startVer} snap={self.snapVer} end= {walFile.endVer}" + tdLog.info(info) + if walFile.mtime < delTs and self.canDelete(walFile): + # wait a moment then check file exist + time.sleep(1) + if os.path.exists(walFile.pathFile): + #report error + tdLog.exit(f" wal file expired need delete. \n {walFile.pathFile} \n modify time={mt} \n delTsLine={delTsLine}\n start={walFile.startVer} snap={self.snapVer} end= {walFile.endVer}") + return False + + # + # check size + # + if self.walSize == 0: + return True + + vnodeSize = self.getWalsSize() + if vnodeSize < self.walSize: + tdLog.info(f" wal size valid. {self.path} real = {vnodeSize} set = {self.walSize} ") + return True + + # check valid + tdLog.info(f" wal size over set. {self.path} real = {vnodeSize} set = {self.walSize} ") + for walFile in self.walFiles: + if self.canDelete(walFile): + # wait a moment then check file exist + time.sleep(1) + if os.path.exists(walFile.pathFile): + tdLog.exit(f" wal file size over .\ + \n wal file = {walFile.pathFile}\ + \n snapVer = {self.snapVer}\ + \n real = {vnodeSize} bytes\ + \n set = {self.walSize} bytes") + return False + return True + + +# insert by async +def thread_insert(testCase, tbname, rows): + print(f"start thread... {tbname} - {rows} \n") + new_conn = testCase.new_connect() + testCase.insert_data(tbname, rows, new_conn) + new_conn.close() + print("end thread\n") + +# case +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.ts = 1670000000000 + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.conn = conn + + # init cluster path + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + self.projDir = f"{projPath}sim/" + tdLog.info(f" init projPath={self.projDir}") + + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(120)', + 'col13': 'nchar(100)', + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(120)', + 't13': 'nchar(100)', + } + + # malloc new connect + def new_connect(self): + return taos.connect(host = self.conn._host, + user = self.conn._user, + password = self.conn._password, + database = self.dbname, + port = self.conn._port, + config = self.conn._config) + + def set_stb_sql(self,stbname,column_dict,tag_dict): + column_sql = '' + tag_sql = '' + for k,v in column_dict.items(): + column_sql += f"{k} {v}, " + for k,v in tag_dict.items(): + tag_sql += f"{k} {v}, " + create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' + return create_stb_sql + + def create_database(self, dbname, wal_period, wal_size_kb, vgroups): + self.wal_period = wal_period + self.wal_size = wal_size_kb * 1024 + self.vgroups = vgroups + self.dbname = dbname + tdSql.execute(f"create database {dbname} wal_retention_period {wal_period} wal_retention_size {wal_size_kb} vgroups {vgroups} replica 3") + tdSql.execute(f'use {dbname}') + + # create stable and child tables + def create_table(self, stbname, tbname, count): + self.child_count = count + self.stbname = stbname + self.tbname = tbname + + # create stable + create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) + tdSql.execute(create_table_sql) + + batch_size = 1000 + # create child table + for i in range(count): + ti = i % 128 + tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"var{i}","nch{i}"' + sql = f'create table {tbname}{i} using {stbname} tags({tags});' + tdSql.execute(sql) + if i % batch_size == 0: + tdLog.info(f" create child table {i} ...") + + tdLog.info(f" create {count} child tables ok.") + + + # insert to child table d1 data + def insert_data(self, tbname, insertTime): + start = time.time() + values = "" + child_name = "" + cnt = 0 + rows = 10000000000 + for j in range(rows): + for i in range(self.child_count): + tj = j % 128 + cols = f'{tj},{tj},{j},{j},{tj},{tj},{j},{j},{j}.000{j},{j}.000{j},true,"var{j}","nch{j}涛思数据codepage is utf_32_le"' + sql = f'insert into {tbname}{i} values ({self.ts},{cols});' + tdSql.execute(sql) + self.ts += 1 + #tdLog.info(f" child table={i} rows={j} insert data.") + cost = time.time() - start + if j % 100 == 0: + tdSql.execute(f"flush database {self.dbname}") + tdLog.info(" insert row cost time = %ds rows = %d"%(cost, j)) + self.consume_topic("topic1", 5) + + if cost > insertTime and j > 100: + tdLog.info(" insert finished. cost time = %ds rows = %d"%(cost, j)) + return + + # create tmq + def create_tmq(self): + sql = f"create topic topic1 as select ts, col1, concat(col12,t12) from {self.stbname};" + tdSql.execute(sql) + sql = f"create topic topic2 as select * from {self.stbname};" + tdSql.execute(sql) + #tdLog.info(sql) + + def check_retention(self, walStayRange): + # flash database + tdSql.execute(f"flush database {self.dbname}") + time.sleep(0.5) + + vnodes = [] + # put all vnode to list + for dnode in os.listdir(self.projDir): + vnodeDir = self.projDir + f"{dnode}/data/vnode/" + print(f"vnodeDir={vnodeDir}") + if os.path.isdir(vnodeDir) == False or dnode[:5] != "dnode": + continue + # enum all vnode + for entry in os.listdir(vnodeDir): + entryPath = path.join(vnodeDir, entry) + + if os.path.isdir(entryPath): + if path.exists(path.join(entryPath, "vnode.json")): + vnode = VNode(int(dnode[5:]), entryPath, self.wal_period, self.wal_size, walStayRange) + vnodes.append(vnode) + + # do check + for vnode in vnodes: + vnode.check_retention() + + # consume topic + def consume_topic(self, topic_name, consume_cnt): + print("start consume...") + consumer = Consumer( + { + "group.id": "tg2", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + } + ) + print("start subscrite...") + consumer.subscribe([topic_name]) + + cnt = 0 + try: + while True and cnt < consume_cnt: + res = consumer.poll(1) + if not res: + break + err = res.error() + if err is not None: + raise err + val = res.value() + cnt += 1 + print(f" consume {cnt} ") + for block in val: + print(block.fetchall()) + finally: + consumer.unsubscribe() + consumer.close() + + + # test db1 + def test_db(self, dbname, checkTime ,wal_period, wal_size_kb): + # var + stable = "meters" + tbname = "d" + vgroups = 6 + count = 10 + + # do + self.create_database(dbname, wal_period, wal_size_kb, vgroups) + self.create_table(stable, tbname, count) + + # create tmq + self.create_tmq() + + # insert data + self.insert_data(tbname, checkTime) + + #stopInsert = False + #tobj = threading.Thread(target = thread_insert, args=(self, tbname, rows)) + #tobj.start() + + # check retention + tdLog.info(f" -------------- do check retention ---------------") + self.check_retention(walStayRange = 256) + + + # stop insert and wait exit + tdLog.info(f" {dbname} stop insert ...") + tdLog.info(f" {dbname} test_db end.") + + + # run + def run(self): + # period + #self.test_db("db1", 10, 60, 0) + # size + #self.test_db("db2", 5, 10*24*3600, 2*1024) # 2M size + + # period + size + self.test_db("db", checkTime = 5*60, wal_period = 60, wal_size_kb=10) + #self.test_db("db", checkTime = 3*60, wal_period = 0, wal_size_kb=0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py index e3144edb4594521bb849b579b9c264e44b156de5..a12f8849812bcf5d72bf24c654e92cc8c8da2cb8 100644 --- a/tests/system-test/1-insert/delete_childtable.py +++ b/tests/system-test/1-insert/delete_childtable.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index a8bfea27415a2b89d230e14d95d2c8a2871ade29..9954b3557e23df193a4c4f84c819fc508c39f5bb 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -54,7 +54,7 @@ class TDTestCase: insert_list = [] self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) def drop_ntb_check(self): - tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'create database if not exists {self.dbname} replica {self.replicaVar} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) self.insert_data(self.column_dict,self.ntbname,self.rowNum) @@ -80,7 +80,7 @@ class TDTestCase: tag_values = [ f'1' ] - tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar}") + tdSql.execute(f"create database if not exists {self.dbname} replica {self.replicaVar} wal_retention_period 3600") tdSql.execute(f'use {self.dbname}') tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) for i in range(self.tbnum): @@ -116,7 +116,7 @@ class TDTestCase: tdSql.checkRows(self.tbnum) tdSql.execute(f'drop database {self.dbname}') def drop_topic_check(self): - tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'create database {self.dbname} replica {self.replicaVar} wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') stbname = tdCom.getLongName(5,"letters") topic_name = tdCom.getLongName(5,"letters") @@ -132,7 +132,7 @@ class TDTestCase: tdSql.execute(f'drop database {self.dbname}') def drop_stream_check(self): - tdSql.execute(f'create database {self.dbname} replica 1') + tdSql.execute(f'create database {self.dbname} replica 1 wal_retention_period 3600') tdSql.execute(f'use {self.dbname}') stbname = tdCom.getLongName(5,"letters") stream_name = tdCom.getLongName(5,"letters") diff --git a/tests/system-test/1-insert/insert_wide_column.py b/tests/system-test/1-insert/insert_wide_column.py new file mode 100644 index 0000000000000000000000000000000000000000..e383675a41dbc63c320f58fe9fe60938f81c5a84 --- /dev/null +++ b/tests/system-test/1-insert/insert_wide_column.py @@ -0,0 +1,87 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import random +import string +import time + +from util.log import * +from util.cases import * +from util.sql import * +from util.autogen import * + + +# +# Test Main class +# + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.autoGen = AutoGen() + + def query_test(self, stbname): + sql = f"select count(*) from {stbname}" + tdSql.execute(sql) + sql = f"select * from {stbname} order by ts desc;" + tdSql.execute(sql) + sql = f"select * from (select * from {stbname} where c1=c2 or c3=c4 or c5=c6) order by ts desc;" + tdSql.execute(sql) + + tdLog.info(" test query ok!") + + + def test_db(self, dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len): + self.autoGen.create_db(dbname) + self.autoGen.create_stable(stbname, tag_cnt, column_cnt, binary_len, nchar_len) + self.autoGen.create_child(stbname, childname, child_cnt) + self.autoGen.insert_data(insert_rows) + self.autoGen.insert_samets(insert_rows) + self.query_test(stbname) + + def run(self): + dbname = "test" + stbname = "st" + childname = "d" + child_cnt = 2 + insert_rows = 10 + tag_cnt = 15 + column_cnt = 20 + binary_len = 10240 + nchar_len = 1025 + self.autoGen.set_batch_size(1) + + # normal + self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len) + + # max + dbname = "test_max_col" + child_cnt = 3 + insert_rows = 50 + tag_cnt = 128 + binary_len = 3 + nchar_len = 4 + column_cnt = 4096 - tag_cnt + self.autoGen.set_batch_size(1) + self.test_db(dbname, stbname, childname, tag_cnt, column_cnt, child_cnt, insert_rows, binary_len, nchar_len) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 8780ecc477f6c6a9ec1076967aa27da63eb09f97..970554c6c17f8b50eb9261dfe1da2f307740cbd0 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -75,7 +75,7 @@ class TDTestCase: tdLog.exit(cmd) def cfg_str(self, filename, update_str): - cmd = f'echo "{update_str}" >> {filename}' + cmd = f'echo {update_str} >> {filename}' if os.system(cmd) != 0: tdLog.exit(cmd) @@ -94,41 +94,41 @@ class TDTestCase: def __err_cfg(self): cfg_list = [] err_case1 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] err_case2 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L2} {PRIMARY_DIR}" ] err_case3 = [ - f"dataDir {self.taos_data_dir}/data33 3 {NON_PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}data33 3 {NON_PRIMARY_DIR}" ] err_case4 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L1} {NON_PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L1} {NON_PRIMARY_DIR}" ] - err_case5 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + err_case5 = [f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] for i in range(16): - err_case5.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") + err_case5.append(f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") err_case6 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}1 {L0} {PRIMARY_DIR}", ] err_case7 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L2} {PRIMARY_DIR}", ] err_case8 = [ - f"dataDir {self.taos_data_dir}/data33 3 {PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}data33 3 {PRIMARY_DIR}" ] err_case9 = [ - f"dataDir {self.taos_data_dir}/data33 -1 {NON_PRIMARY_DIR}" + f"dataDir {self.taos_data_dir}{os.sep}data33 -1 {NON_PRIMARY_DIR}" ] cfg_list.append(err_case1) @@ -147,23 +147,23 @@ class TDTestCase: def __current_cfg(self): cfg_list = [] current_case1 = [ - #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" + #f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] - #current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + #current_case2 = [f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] current_case2 = [] for i in range(9): - current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") + current_case2.append(f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") # TD-17773bug current_case3 = [ - #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}", + #f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}0 ", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}{os.sep}{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}", ] cfg_list.append(current_case1) cfg_list.append(current_case3) diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 8ee2007450328b2edbb02822a68fa4ac9d9ca9a7..424ebff6c5764dec9a2395eaa2254acfb47056c5 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -26,7 +26,7 @@ class TDTestCase: 'c1':'int', 'c2':'float', 'c3':'binary(20)' - + } # structure of tag self.tag_dict = { @@ -60,7 +60,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(self.values_list)*tb_num) + tdSql.checkRows(len(self.values_list)*tb_num) for time in ['2020-01-32T08:00:00','2020-13-32T08:00:00','acd']: tdSql.query(f"select to_unixtimestamp('{time}') from {tbname}") if tb_type == 'ntb' or tb_type == 'ctb': @@ -74,7 +74,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(values_list)*tb_num) + tdSql.checkRows(len(values_list)*tb_num) for time in self.error_param: tdSql.error(f"select to_unixtimestamp({time}) from {tbname}") def timestamp_change_check_ntb(self): @@ -95,9 +95,20 @@ class TDTestCase: self.data_check(f'{self.stbname}_{i}',self.values_list,'ctb') self.data_check(self.stbname,self.values_list,'stb',self.tbnum) tdSql.execute(f'drop database {self.dbname}') + def timestamp_change_return_type(self): + tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);") + tdSql.checkEqual(tdSql.queryResult[0][0], 0) + tdSql.query(f"select to_unixtimestamp('1970-01-01 00:00:00', 1);") + tdSql.checkData(0, 0, '1970-01-01 00:00:00') + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', true);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1, 3);") def run(self): # sourcery skip: extract-duplicate-method self.timestamp_change_check_ntb() self.timestamp_change_check_stb() + self.timestamp_change_return_type() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 08f6ba6bafa8029a20ef3b5e80fc6e9e90ea68b7..77e6bd8cb6dfe94eaaeeb6c455a3a8e6784784bc 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -16,6 +16,7 @@ class TDTestCase: tdSql.init(conn.cursor()) self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") self.today_ts = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d").timestamp() + self.today_ts_ns = 0 self.time_unit = ['b','u','a','s','m','h','d','w'] self.error_param = ['1.5','abc','!@#','"abc"','today()'] self.arithmetic_operators = ['+','-','*','/'] @@ -74,8 +75,10 @@ class TDTestCase: num_same = 0 if v.lower() == 'timestamp': tdSql.query(f'select {k} from {tbname}') + for i in tdSql.queryResult: if precision == 'ms': + self.today_ts_trans = int(self.today_ts)*1000 if int(i[0].timestamp())*1000 > int(self.today_ts)*1000: num_up += 1 elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000: @@ -83,6 +86,7 @@ class TDTestCase: elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000: num_down += 1 elif precision == 'us': + self.today_ts_trans = int(self.today_ts)*1000000 if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000: num_up += 1 elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000: @@ -90,6 +94,7 @@ class TDTestCase: elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000: num_down += 1 elif precision == 'ns': + self.today_ts_trans = int(self.today_ts)*1000000000 if i[0] > int(self.today_ts)*1000000000: num_up += 1 elif i[0] == int(self.today_ts)*1000000000: @@ -97,8 +102,9 @@ class TDTestCase: elif i[0] < int(self.today_ts)*1000000000: num_down += 1 tdSql.query(f"select today() from {tbname}") - tdSql.checkRows(len(values_list)*tb_num) - tdSql.checkData(0, 0, str(self.today_date)) + tdSql.checkRows(len(values_list)*tb_num) + print(self.today_ts_trans,self.today_ts,precision,num_up,num_down,i[0]) + tdSql.checkData(0, 0, self.today_ts_trans) tdSql.query(f"select * from {tbname} where {k}=today()") if tb == 'tb': tdSql.checkRows(num_same*tb_num) @@ -149,11 +155,12 @@ class TDTestCase: if tb == 'tb': tdSql.checkRows(num_same*tb_num) for i in range(num_same*tb_num): - tdSql.checkData(i, 0, str(self.today_date)) + print(self.today_ts_trans,precision,num_up,num_down) + tdSql.checkData(i, 0, self.today_ts_trans) elif tb == 'stb': tdSql.checkRows(num_same) for i in range(num_same): - tdSql.checkData(i, 0, str(self.today_date)) + tdSql.checkData(i, 0, self.today_ts_trans) def today_check_ntb(self): for time_unit in self.db_percision: diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index e6542c7c2bc6b91e501098baf45562cadb07b599..ede1f28324dd768a40d7ae98d140c40570e7bb88 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -38,11 +38,8 @@ class TDTestCase: if data_tb_col[i] is None: tdSql.checkData( i, 0 , None ) if col_name not in ["c2", "double"] or tbname != f"{self.dbname}.t1" or i != 10: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = datetime.datetime.fromtimestamp(data_tb_col[i]/1000) + tdSql.checkData( i, 0, date_init_stamp) def __range_to_timestamp(self, cols, tables): for col in cols: @@ -60,12 +57,11 @@ class TDTestCase: self.__range_to_timestamp(cols=__col_list, tables=__table_list) def all_test(self): - + _datetime_epoch = datetime.datetime.fromtimestamp(0) tdSql.query(f"select c1 from {self.dbname}.ct4") data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] tdSql.query(f"select c1 from {self.dbname}.t1") data_t1_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - tdLog.printNoPrefix("==========step2: cast int to bigint, expect no changes") tdSql.query(f"select cast(c1 as bigint) as b from {self.dbname}.ct4") @@ -100,22 +96,16 @@ class TDTestCase: if data_ct4_c1[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c1[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c1 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c1)): - if data_ct4_c1[i] is None: + if data_t1_c1[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c1[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c1[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step8: cast bigint to bigint, expect no changes") @@ -157,11 +147,8 @@ class TDTestCase: if data_ct4_c2[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c2[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c2[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c2 as timestamp) as b from {self.dbname}.t1") @@ -171,11 +158,8 @@ class TDTestCase: elif i == 10: continue else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_t1_c2[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c2[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step12: cast smallint to bigint, expect no changes") @@ -217,22 +201,16 @@ class TDTestCase: if data_ct4_c3[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c3[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c3 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c3)): - if data_ct4_c3[i] is None: + if data_t1_c3[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c3[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c3[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes") @@ -274,22 +252,16 @@ class TDTestCase: if data_ct4_c4[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c4[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c4 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c4)): - if data_ct4_c4[i] is None: + if data_t1_c4[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(data_ct4_c4[i]/1000) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c4[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step20: cast float to bigint, expect no changes") @@ -327,21 +299,15 @@ class TDTestCase: if data_ct4_c5[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c5[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c5[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c5 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c5)): if data_t1_c5[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c5[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c5[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step24: cast double to bigint, expect no changes") tdSql.query(f"select c6 from {self.dbname}.ct4") @@ -383,11 +349,8 @@ class TDTestCase: if data_ct4_c6[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c6[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c6[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c6 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c6)): @@ -396,11 +359,8 @@ class TDTestCase: elif i == 10: continue else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c6[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c6[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdLog.printNoPrefix("==========step28: cast bool to bigint, expect no changes") tdSql.query(f"select c7 from {self.dbname}.ct4") @@ -437,21 +397,15 @@ class TDTestCase: if data_ct4_c7[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_ct4_c7[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_ct4_c7[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select cast(c7 as timestamp) as b from {self.dbname}.t1") for i in range(len(data_t1_c7)): if data_t1_c7[i] is None: tdSql.checkData( i, 0 , None ) else: - utc_zone = datetime.timezone.utc - utc_8 = datetime.timezone(datetime.timedelta(hours=8)) - date_init_stamp = datetime.datetime.utcfromtimestamp(int(data_t1_c7[i]/1000)) - date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") - tdSql.checkData( i, 0, date_data) + date_init_stamp = _datetime_epoch+datetime.timedelta(seconds=int(data_t1_c7[i]) / 1000.0) + tdSql.checkData( i, 0, date_init_stamp) tdSql.query(f"select c8 from {self.dbname}.ct4") @@ -567,7 +521,7 @@ class TDTestCase: if data_ct4_c10[i] is None: tdSql.checkData( i, 0, None ) else: - time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) + time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0,data_ct4_c10[i].tzinfo)).total_seconds())*1000+int(data_ct4_c10[i].microsecond / 1000)) tdSql.checkData( i, 0, time2str ) tdSql.query(f"select cast(c10 as nchar(32)) as b from {self.dbname}.t1") for i in range(len(data_t1_c10)): @@ -576,7 +530,7 @@ class TDTestCase: elif i == 10: continue else: - time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) + time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0,data_t1_c10[i].tzinfo)).total_seconds())*1000+int(data_t1_c10[i].microsecond / 1000)) tdSql.checkData( i, 0, time2str ) tdLog.printNoPrefix("==========step38: cast timestamp to binary, expect no changes ") @@ -585,7 +539,7 @@ class TDTestCase: if data_ct4_c10[i] is None: tdSql.checkData( i, 0, None ) else: - time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) + time2str = str(int((data_ct4_c10[i]-datetime.datetime.fromtimestamp(0,data_ct4_c10[i].tzinfo)).total_seconds())*1000+int(data_ct4_c10[i].microsecond / 1000)) tdSql.checkData( i, 0, time2str ) tdSql.query(f"select cast(c10 as binary(32)) as b from {self.dbname}.t1") for i in range(len(data_t1_c10)): @@ -594,7 +548,7 @@ class TDTestCase: elif i == 10: continue else: - time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0)).total_seconds()*1000)) + time2str = str(int((data_t1_c10[i]-datetime.datetime.fromtimestamp(0,data_t1_c10[i].tzinfo)).total_seconds())*1000+int(data_t1_c10[i].microsecond / 1000)) tdSql.checkData( i, 0, time2str ) tdLog.printNoPrefix("==========step39: cast constant operation to bigint, expect change to int ") @@ -695,7 +649,7 @@ class TDTestCase: tdSql.query(f"select cast('123' as tinyint unsigned) as b from {self.dbname}.stb1 partition by tbname") # uion with cast and common cols - + tdSql.query(f"select cast(c2 as int) as b from {self.dbname}.stb1 union all select c1 from {self.dbname}.stb1 ") tdSql.query(f"select cast(c3 as bool) as b from {self.dbname}.stb1 union all select c7 from {self.dbname}.ct1 ") tdSql.query(f"select cast(c4 as tinyint) as b from {self.dbname}.stb1 union all select c4 from {self.dbname}.stb1") diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py new file mode 100644 index 0000000000000000000000000000000000000000..e43b32a716017702382ce0384bb377d692e64684 --- /dev/null +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -0,0 +1,215 @@ + +import taos +import sys +import time +import socket +import os +import platform +if platform.system().lower() == 'windows': + import wexpect as taosExpect +else: + import pexpect as taosExpect + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (buildPath, key, value, expectString, sqlString=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + if platform.system().lower() == 'windows': + taosCmd = buildPath + '\\build\\bin\\taos.exe ' + taosCmd = taosCmd.replace('\\','\\\\') + else: + taosCmd = buildPath + '/build/bin/taos ' + + cfgPath = buildPath + "/../sim/psim/cfg" + taosCmd = taosCmd + ' -c' + cfgPath + ' -' + key + if len(value) != 0: + taosCmd = taosCmd + ' ' + value + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = taosExpect.spawn(taosCmd, timeout=20) + #output = child.readline() + #print (output.decode()) + if len(expectString) != 0: + i = child.expect([expectString, taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) + else: + i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=20) + + if platform.system().lower() == 'windows': + retResult = child.before + else: + retResult = child.before.decode() + print(retResult) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + return "TAOS_OK" + else: + return "TAOS_FAIL" + +class TDTestCase: + #updatecfgDict = {'clientCfg': {'serverPort': 7080, 'firstEp': 'trd02:7080', 'secondEp':'trd02:7080'},\ + # 'serverPort': 7080, 'firstEp': 'trd02:7080'} + hostname = socket.gethostname() + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP + serverPort = '7080' + rpcDebugFlagVal = '143' + clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + clientCfgDict["serverPort"] = serverPort + clientCfgDict["firstEp"] = hostname + ':' + serverPort + clientCfgDict["secondEp"] = hostname + ':' + serverPort + clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + clientCfgDict["fqdn"] = hostname + + updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + updatecfgDict["clientCfg"] = clientCfgDict + updatecfgDict["serverPort"] = serverPort + updatecfgDict["firstEp"] = hostname + ':' + serverPort + updatecfgDict["secondEp"] = hostname + ':' + serverPort + updatecfgDict["fqdn"] = hostname + + print ("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = self.hostname + keyDict['c'] = cfgPath + keyDict['P'] = self.serverPort + + tdSql.query("drop database if exists db1") + tdSql.query("create database if not exists db1 vgroups 1") + tdSql.query("use db1") + tdSql.query("create table tba (ts timestamp, f1 binary(2))") + tdSql.query("insert into tba values (now, '22')") + tdSql.query("select * from tba") + tdSql.checkData(0, 1, '22') + + keyDict['s'] = "\"alter table db1.tba modify column f1 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tba values (now, '55555')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select * from tba order by ts") + tdSql.checkData(0, 1, '22') + tdSql.checkData(1, 1, '55555') + + + + keyDict['s'] = "\"alter table db1.tba add column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select * from tba order by ts") + tdSql.query("select * from tba order by ts") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + + + + + keyDict['s'] = "\"alter table db1.tba add column f3 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select f3 from tba order by ts") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + + + tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))") + tdSql.query("create table tb1 using stb tags('bb')") + tdSql.query("insert into tb1 values (now, 2,'22')") + tdSql.query("select count(*) from stb group by tg1") + tdSql.checkData(0, 0, 1) + + keyDict['s'] = "\"alter table db1.stb modify tag tg1 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"create table db1.tb2 using db1.stb tags('bbbbb')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tb2 values (now, 2,'22')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"alter table db1.stb modify column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tb2 values (now, 3,'55555')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select count(*) from stb group by tg1") + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 0, 1) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index d7344c631f5a150d3b9bac907b263bb2d41ec91e..ddf3f2534dd42d80b4b9a7da647748a8193902f0 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -19,6 +19,7 @@ class TDTestCase: dbname = "db" tbname = "tb" tbname1 = "tb1" + tbname2 = "tb2" stbname = "stb" ctbname1 = "ctb1" ctbname2 = "ctb2" @@ -1948,6 +1949,7 @@ class TDTestCase: tdSql.checkData(59, 1, 60) tdSql.checkData(60, 1, 60) # + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) @@ -2002,7 +2004,362 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step12:test error cases") + tdLog.printNoPrefix("==========step12:test interp with boolean type") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname2} + (ts timestamp, c0 bool) + ''' + ) + + + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:01', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:03', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:05', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:07', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:09', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:11', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:13', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:15', NULL)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:17', NULL)") + + # test fill null + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(NULL)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, None) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, None) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, None) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, None) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + # test fill prev + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(prev)") + tdSql.checkRows(18) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + + tdSql.checkData(17, 0, '2020-02-02 00:00:18.000') + + # test fill next + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(next)") + tdSql.checkRows(18) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + + tdSql.checkData(17, 0, '2020-02-02 00:00:17.000') + + # test fill value + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 0)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 1234)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, false)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, true)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, '0')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, '123')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 'abc')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, NULL)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + # test fill linear + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(17) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, False) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, False) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, None) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + + tdSql.checkData(16, 0, '2020-02-02 00:00:17.000') + + tdLog.printNoPrefix("==========step13:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") @@ -2013,7 +2370,7 @@ class TDTestCase: # input can only be numerical types tdSql.error(f"select interp(ts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") - tdSql.error(f"select interp(c6) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + #tdSql.error(f"select interp(c6) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c7) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c8) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") @@ -2025,6 +2382,14 @@ class TDTestCase: tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + # invalid pseudo column usage + tdSql.error(f"select interp(_irowts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(_isfilled) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _isfilled = true range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _irowts > 0 range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + + + tdLog.printNoPrefix("==========step13:stable cases") @@ -2040,6 +2405,8 @@ class TDTestCase: #tdSql.query(f"select _irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") #tdSql.query(f"select tbname,_irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") + tdLog.printNoPrefix("======step 14: test interp pseudo columns") + tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index df460df5c387acf863e1a7aab3b2de67ef040abe..ebd580efd4fbd7a4ab71e5c06da1f83c99e9da97 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -301,7 +301,7 @@ class TDTestCase: # where json value is not exist tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag1' is null") - tdSql.checkData(0, 0, 'jsons1_9') + # tdSql.checkData(0, 0, 'jsons1_9') tdSql.checkRows(2) tdSql.query(f"select * from {dbname}.jsons1 where jtag->'tag4' is null") tdSql.checkRows(9) diff --git a/tests/system-test/2-query/last_row_interval.py b/tests/system-test/2-query/last_row_interval.py new file mode 100644 index 0000000000000000000000000000000000000000..ca3946af8874c59f41b27313942b464c80ad29ae --- /dev/null +++ b/tests/system-test/2-query/last_row_interval.py @@ -0,0 +1,789 @@ +# author : wenzhouwww +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + def prepare_data(self): + + tdSql.execute(f'''drop database if exists table_lastrow_2_2;''') + tdSql.execute(f'''create database if not exists table_lastrow_2_2 keep 36500 ;''') + tdSql.execute(f'''use table_lastrow_2_2;''') + tdSql.execute(f'''create stable stable_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute(f'''create stable stable_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute(f'''create stable stable_null_data (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute(f'''create stable stable_null_childtable (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute(f'''create stable stable_null_data_insert_data (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);''') + tdSql.execute(f'''create table stable_1_1 using stable_1 tags('stable_1_1', '-1018336694' , '1595508332262302224', '28072' , '-35' , 0 , 'binary1.MWLRCtBHUaCUgFSTdtBk' , 'nchar1.dALWGgKyWCbOuxCUbzKr' , '-5.335609', '48773689935.563004' ,'0') ;''') + tdSql.execute(f'''create table stable_1_2 using stable_1 tags('stable_1_2', '785397064' , '-6316393479307224208', '-12059' , '-20' , 1 , 'binary1.laHjdQwAnHfxLDvYcBRX' , 'nchar1.xylcIZwLgJewVUUBDjkc' , '353778266298.739014', '-994383623.121693' , '1999-09-09 09:09:09.090') ;''') + tdSql.execute(f'''create table stable_1_3 using stable_1 tags('stable_1_3', '-2147483647' , '-9223372036854775807' , '-32767' , '-127' , false , 'binary3' , 'nchar3nchar3' , '-3.3' , '-33.33' , '2099-09-09 09:09:09.090') ;''') + tdSql.execute(f'''create table stable_1_4 using stable_1 tags('stable_1_4', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + tdSql.execute(f'''create table stable_1_5 using stable_1 tags('stable_1_5', '2129429608' , '2307167663134055915', '-30801' , '97' , 1 , 'binary1.xxBwoDsTKoIfHezkMCMK' , 'nchar1.IyVWFtMtpqxfvpJpbgQe' , '-0.988745', '804627.676066' ,'-811294702') ;''') + tdSql.execute(f'''create table stable_1_6 using stable_1 tags('stable_1_6', '-418554527' , '7121649991943388108', '-17673' , '-32' , 1 , 'binary1.nqevHWrcyyicEQwayBtK' , 'nchar1.UzlaWBQIwovvojtniExg' , '63206.472351', '66.580164' ,'-2011047779') ;''') + tdSql.execute(f'''create table stable_2_1 using stable_2 tags('stable_2_1', '-650310817' , '-6448202879686146556', '-4887' , '-49' , 1 , 'binary2.vxWkNcASTlgvrLxckVzo' , 'nchar2.IkwxYduQGOkmMYozJsGu' , '-5503502270.709900', '-193919.697976' ,'0') ;''') + tdSql.execute(f'''create table stable_2_2 using stable_2 tags('stable_2_2' , '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + tdSql.execute(f'''create table stable_2_3 using stable_2 tags('stable_2_3', '844729931' , '3743631577053724543', '-29079' , '-68' , 1 , 'binary2.zemsTeoWSPFIVFIJVgqg' , 'nchar2.zbPxIWIIvmmnSsdjdUgv' , '18949277.377942', '-160343395.575977' ,'2055943475') ;''') + tdSql.execute(f'''create table stable_2_4 using stable_2 tags('stable_2_4', '630274019' , '4517699459805299748', '-10047' , '-66' , 1 , 'binary2.QAecEuLSbznsyExjWhWQ' , 'nchar2.KiKzPupgeBsBQCFpGMjf' , '93521718623175.000000', '5981.851300' ,'-1323235531') ;''') + tdSql.execute(f'''create table stable_2_5 using stable_2 tags('stable_2_5', '-1888888729' , '-4964455964757350108', '21832' , '-102' , 1 , 'binary2.IYBWfiyETfYXlLmoHvCL' , 'nchar2.VNSfIvIrgAZQsYfrvGuW' , '-7938.194391', '8946996355046.539062' ,'-1075201653') ;''') + tdSql.execute(f'''create table stable_2_6 using stable_2 tags('stable_2_6', '221977162' , '-4074099363599643838', '-15384' , '93' , 1 , 'binary2.IixhuAsSrChdQTizyBRi' , 'nchar2.XAECwFPOVvqbfAQnXFll' , '112970.139491', '-86723.729566' ,'124866521') ;''') + tdSql.execute(f'''create table stable_null_data_1 using stable_null_data tags('stable_null_data_1', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + tdSql.execute(f'''create table stable_null_data_insert_data_1 using stable_null_data_insert_data tags('stable_null_data_insert_data_1', '0' , '0' , '0' , '0' , 0 , '0' , '0' , '0' , '0' ,'0') ;''') + tdSql.execute(f'''create table regular_table_1 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute(f'''create table regular_table_2 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute(f'''create table regular_table_3 (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute(f'''create table regular_table_null (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630000000000, -618707591, -8247775922836565826, -5478, 118, 5.414333, 75.304275, 0, 'binary.mjCkHlPgXnuCpAFOIRiI', 'nchar.陕西省长春县涪城赵街S座 993438', 1630000000000) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, -679041833, 6481912273446973866, 7785, 25, 253.538943, -12801.160677, 0, 'binary.OIaLSSdTkyeghECZRlly', 'nchar.四川省长春县萧山黄街m座 950751', 1630000000000) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1629999999999, 1000633087, 4246783346267002529, 32349, 56, -38524798468.228600, 677.696961, 1, 'binary.tfdGRAJloMWlejFCvXGH', 'nchar.新疆维吾尔自治区彬县浔阳深圳路k座 106151', 1630000000000) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, 1519095863, 3730763636688762973, 32328, 18, 13261.792805, -23310331001.143700, 1, 'binary.YuznyMmpNmIBujYOnfBj', 'nchar.湖南省银川市蓟州沈阳街f座 828894', 1630000000000) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000001, -1721664965, -3230247488973407828, -22534, -78, -4615440117998.759766, 128542619.774818, 1, 'binary.VkIjDmJdfYRXCGOcoaUw', 'nchar.湖北省拉萨县吉区银川街p座 684472', 1630000000001) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000001, -457200104, -5211291163024696467, -16773, -94, -6983919702730.200195, -67714706.233835, 1, 'binary.LTmCJTcqnzIfNQYoRTPC', 'nchar.黑龙江省西宁市江北王路P座 603299', 1630000000001) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630000000000, 913998061, -3881184424925998682, 29221, -118, -7492895.381435, -119177303.116243, 0, 'binary.oryFxMGKONJblKZWJxLF', 'nchar.广东省荣市上街钟路A座 708810', 1630000000000) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630015000000, -1004099631, -2329741298995361936, -17228, 92, -7786612568.749220, 63297.402773, 0, 'binary.oIlAieuDxUnJAznSxgDa', 'nchar.陕西省呼和浩特县华龙马鞍山路B座 509550', 1630000000001) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, 1984250107, -733051146694989614, 25996, 7, 32391348.640906, 30496.996911, 0, 'binary.SjSkhuLmeqJhgvToyskb', 'nchar.广西壮族自治区武汉县梁平兰州街z座 108560', 1630000000001) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630014999999, 1167978495, 168911323243208687, 25054, 105, 9.297430, -74362.658908, 1, 'binary.gjlULgjKtTQMRsgDIVbu', 'nchar.陕西省合山市友好哈尔滨路y座 906640', 1630000000001) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, 2124176615, 3367944756720448423, 31669, 74, -40.578216, 3714810365386.549805, 1, 'binary.xzMdipXOFEkZkaqAnulR', 'nchar.台湾省洁市门头沟惠州街T座 814575', 1630000000001) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000001, -800594954, -6220928593410477008, -6436, -107, -379445813485.830017, 1817959.681926, 1, 'binary.fgQqvhJhqEAyqYcYCzop', 'nchar.江西省六安县蓟州长沙路K座 686835', 1630000000002) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000001, -1398934219, -5088222972686572580, -11996, -93, 737.964702, -702597046.276900, 1, 'binary.WqJNWJTeOypOEILqFfFM', 'nchar.福建省哈尔滨市合川李街A座 990061', 1630000000002) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630015000000, -1491848052, -3232978461444804772, 28885, 110, 68028.289282, -0.612906, 0, 'binary.qfQwRgWfrgQLefodbKQE', 'nchar.海南省汕尾市普陀刘街x座 901815', 1630000000001) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630030000000, -554992634, -8694605272129245647, -19744, -34, 371492.822008, 97594901899.304092, 0, 'binary.vATatWJTwyzWNENoWGqm', 'nchar.黑龙江省兴安盟市友好李路X座 412206', 1630000000002) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, 230798367, 4276367516153106032, 23696, -34, -2149198212.254210, 722029.566814, 0, 'binary.EZDyNTgJgCCPGkPuTwyr', 'nchar.辽宁省石家庄市高港广州路V座 518042', 1630000000002) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630029999999, 2093449032, 3029533084633134124, 17333, 74, 179560.971309, 5.196993, 1, 'binary.JYigiQaVrKTPLzQHXeDp', 'nchar.湖北省南昌县金平合山街g座 418389', 1630000000002) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, 2099331457, 3609885272656270889, 31314, 60, -8489143.581025, -214298.108651, 1, 'binary.ltSDheslsSNwSxfYxIwt', 'nchar.辽宁省深圳县翔安石家庄街c座 827189', 1630000000002) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000001, -1245000234, -1838653281383479618, -13572, -101, -20.386729, 41392371.908604, 1, 'binary.nqjueffbFvZcGQexkMuc', 'nchar.重庆市淮安市上街福州街F座 236907', 1630000000003) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000001, -1000510248, -8459051266344969043, -8477, -86, -6923089039562.599609, 7.587889, 1, 'binary.egZiyotIUjvmyEeabqWD', 'nchar.宁夏回族自治区银川县淄川辽阳路F座 365611', 1630000000003) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630030000000, 1641358132, 5291198999362916313, 1219, 43, 13391898193591.400391, -253345.858194, 0, 'binary.zowdfNbqLIBOvTqQmqVq', 'nchar.辽宁省大冶县永川济南街Z座 410021', 1630000000002) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 1273633633, -7910396749266717868, -2883, -35, -88018.597889, 331.964131, 0, 'binary.HLOtPGQtEAfRTTNAVbnF', 'nchar.福建省桂珍县翔安周路s座 201916', 1630000000003) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, -292105648, -9216209950218616247, 17873, -40, 510.442902, 4914.654078, 0, 'binary.FZYRLYLrlaZdIBnbOvWI', 'nchar.安徽省北京县平山郑州路p座 162038', 1630000000003) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630044999999, 1985684265, 7812482934230300604, 10735, 2, 39668.546371, 3772288865301.229980, 1, 'binary.DQDjLSXXpYjXDnGzrDiB', 'nchar.福建省丽丽市大东石家庄路O座 638078', 1630000000003) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 273025537, 6201710872284300698, 24247, 122, 34.519692, 3678779922.134800, 1, 'binary.kbpPYOgvAQcOBLFcuuZj', 'nchar.浙江省莹县吉区岳街N座 997620', 1630000000003) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000001, -755554172, -5922209280465971373, -16931, -1, -738590.998999, -213684725.699882, 1, 'binary.gGZITsRKKtOkkAZpwqFk', 'nchar.河南省石家庄市西夏施街d座 887826', 1630000000004) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000001, -1614464307, -1951583711943603971, -6, -26, 46698502859495.796875, -66331370.539177, 1, 'binary.anFFtpnYqkllelsTzUXp', 'nchar.重庆市长沙县丰都郑街v座 632337', 1630000000004) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630045000000, 1066735810, -1572767557463382095, -24233, -99, -8059716.727903, -44065130.621251, 0, 'binary.EHRUdvBpXXdxSfkPbHPY', 'nchar.贵州省桂兰市高坪广州街K座 807922', 1630000000003) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630060000000, 436440993, 4582366906832569569, 19691, -104, -770862234.875283, 67327.464546, 0, 'binary.rGBrQBwSJziIwOmEMtJD', 'nchar.黑龙江省通辽市高明张路I座 703149', 1630000000004) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, -875710377, -5188572182276438044, -19951, 75, -2.627015, -71356793.343289, 0, 'binary.xoMabkxpBOQvAWZlTVMz', 'nchar.贵州省汕尾县静安黄街M座 165440', 1630000000004) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630059999999, 1129217878, 8833855448102017424, 3948, 86, 1462546878675.199951, -9256879605.926800, 1, 'binary.pCnFTHmykgTwspRMSMDk', 'nchar.云南省西宁县合川苏街h座 357731', 1630000000004) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, 2091037114, 6355755798403937517, 32527, 22, 2.418504, 74.778674, 1, 'binary.yoobwWUyBSZuRZmlcWTL', 'nchar.天津市萍市大东姚街w座 255499', 1630000000004) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000001, -647490375, -2905281875821917763, -16147, -57, -8479790.621940, 5694596048115.910156, 1, 'binary.gqtxAiHhIoGKdGJgNhat', 'nchar.云南省齐齐哈尔市新城翟路J座 336359', 1630000000005) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000001, -1939480852, -8924947224421059592, -29393, -89, 4457679028853.139648, 3390867.192607, 1, 'binary.biEBPzDOrLOGitUZfDIy', 'nchar.新疆维吾尔自治区彬市南溪黄街i座 106358', 1630000000005) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630060000000, 760109584, 2794056460189383180, -17287, -72, -99807343228.163803, 1373392779.173220, 0, 'binary.xABwARkXCmOteoVJInrZ', 'nchar.吉林省芳县安次荆门路K座 396431', 1630000000004) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630075000000, -1866923177, -2210826056783865816, -26481, 25, -60150089285.989502, 7628.542314, 0, 'binary.rTYYBufHDmHMtGoZQOTT', 'nchar.宁夏回族自治区兵市孝南陈路x座 334815', 1630000000005) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, 2038781133, -328945288212162680, -28897, 95, 5608233.583575, -69.542113, 0, 'binary.JwNUFBltIXQKAdjZpTIn', 'nchar.河南省娟县南长吴路f座 790249', 1630000000005) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630074999999, 1145437501, 7925745151626620680, 4822, 91, -59887158.500668, -47581275646365.203125, 1, 'binary.DrUaLRxoRNzyNzXYUlRE', 'nchar.甘肃省南京市大东大冶街D座 872025', 1630000000005) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, 1789577030, 3132818518285337526, 26070, 65, 8974955.481672, 25.499355, 1, 'binary.VRegnoPEWKPRFsoYVtYC', 'nchar.贵州省东市东丽上海路s座 272562', 1630000000005) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000001, -1373109267, -5826806756838814238, -16841, -28, 51.267264, 48701588356.369003, 1, 'binary.WqMufKnCjgUuLBrWIgRT', 'nchar.安徽省玲市西夏海口街o座 109427', 1630000000006) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000001, -813108784, -1617673275161393067, -1248, -69, -972415684.192217, -1.653864, 1, 'binary.kmjtLhPVdLESTsOjNOxB', 'nchar.台湾省潮州市孝南廖街t座 266427', 1630000000006) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630075000000, -1223568312, -8946838664732429923, -4198, -109, 62156997799.337097, 5905968167.585610, 0, 'binary.gPXEGOdtZVrGqPNRAInE', 'nchar.西藏自治区齐齐哈尔市清城郑州街w座 590784', 1630000000005) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630090000000, -884439870, 1248874482756813386, -11000, 61, 80395.998486, -719886040980.203003, 0, 'binary.MUrOdVZgtgPWPXbIcstW', 'nchar.江苏省天津县龙潭贵阳路s座 612000', 1630000000006) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, 22358632, 696083452197273333, -13257, -41, 56423240.636894, 67488.334223, 0, 'binary.HEsJVMnaOFobHNDaoRtn', 'nchar.贵州省杭州县沈河陆路s座 645540', 1630000000006) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630089999999, 1652733516, 2575789832408445556, 16457, 72, -9677288.455314, -634151591.251647, 1, 'binary.kcMLZfZtpBrgBBeZWbaO', 'nchar.青海省汕尾市长寿周街N座 774464', 1630000000006) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, 1670284035, 7846771393632678538, 24419, 81, 256.686378, -932250077361.723022, 1, 'binary.dcqObzxzMWdcGMorJEdo', 'nchar.浙江省荣县平山王路W座 892684', 1630000000006) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000001, -1239207401, -2017719648880935228, -9628, -15, -77811.715389, -347.862346, 1, 'binary.arpuZMBhyXPuWeNDqoQg', 'nchar.上海市武汉县朝阳佛山街r座 687990', 1630000000007) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000001, -429906324, -294441740565391101, -18570, -27, -91849065.920391, -9504277.384224, 1, 'binary.YzliXowrLkmoHdpasClN', 'nchar.广东省静县新城王街H座 307037', 1630000000007) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630090000000, 1647824018, -1290146600224131452, -26320, -17, -97472611955.200394, 623.599606, 0, 'binary.eIXSvTrGfgIWKUgEQIKK', 'nchar.山西省俊县新城许路L座 732295', 1630000000006) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630105000000, 897426731, -6157764764029970873, 13753, 53, -76610.177184, 784852116.102444, 0, 'binary.oThZwmCKADWhpHiBQMEB', 'nchar.安徽省雷县高明六盘水街s座 104130', 1630000000007) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, 1727787759, 3219114917075431149, -30010, 105, -29098130696901.199219, 2716.301364, 0, 'binary.KpchCpVHGKjREJMxRCYb', 'nchar.河北省辽阳市南长岑街l座 554159', 1630000000007) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630104999999, 964911890, 9104291220948875270, 2620, 93, 6348.912976, -5.436949, 1, 'binary.enOSjjsQzNbvDdFfEKhA', 'nchar.西藏自治区桂芝县山亭刘路N座 825972', 1630000000007) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, 1265975391, 1097514590897652008, 30907, 127, 6962290728.631910, 4234.969142, 1, 'binary.IDDEVFUXulGKPGUxgwoa', 'nchar.贵州省龙市友好哈尔滨街V座 614938', 1630000000007) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000001, -997687142, -3588569170127119022, -31895, -11, 681764.467420, -927.233192, 1, 'binary.qAYrbGModOeljwoNdOSK', 'nchar.内蒙古自治区桂英县清城何街e座 656693', 1630000000008) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000001, -1838171592, -2931035207249690385, -27947, -4, -0.864426, -31972996.517570, 1, 'binary.xmGbCIlRspBnDeHNjzGO', 'nchar.香港特别行政区辽阳县朝阳黄路J座 742465', 1630000000008) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630105000000, -463217897, -8041553223283381090, -9791, 67, -69685155628.862000, 251717731390.667999, 0, 'binary.RvKNPCRWhwpJzghpxMnY', 'nchar.山东省宁县大兴王街f座 954429', 1630000000007) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630120000000, -54069770, 3955990383518016848, 8494, 120, 51149.933011, 1349638022797.659912, 0, 'binary.ctqJUvXzcLsBTGXFePLO', 'nchar.辽宁省超市六枝特马鞍山街F座 798879', 1630000000008) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, -403363944, -2394897603233419155, 16427, 48, 74402.339292, -52.539883, 0, 'binary.kobpHICjpqBLcDIEHpts', 'nchar.海南省琳县高明沈阳路e座 854884', 1630000000008) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630119999999, 806349342, 8818007624619329018, 11587, 27, 24820988.463016, -1.378348, 1, 'binary.SPIvOBZxamwoTLUyeDec', 'nchar.上海市玉梅县六枝特王路F座 154214', 1630000000008) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, 1843801013, 8068282721613219948, 23265, 89, 20575797079906.000000, 7.906923, 1, 'binary.smCqwmyottgAmLVxlXfX', 'nchar.内蒙古自治区杭州县南湖上海街P座 763569', 1630000000008) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000001, -1431341222, -5831324648859520070, -8527, -66, 507444690740.578003, 943376.611667, 1, 'binary.nZHlAmwbcUYtOTsAxArC', 'nchar.广西壮族自治区武汉市海陵施路Z座 727105', 1630000000009) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000001, -781555629, -3682028760781093775, -1217, -88, 967.988362, -998747442791.562012, 1, 'binary.CMDWBFfYnlYEdTeiqtsW', 'nchar.云南省敏县白云柳州路Z座 839560', 1630000000009) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630120000000, -1229205179, 1269213658170093198, 15696, -110, 8232823752181.839844, 36569.580002, 0, 'binary.voGRWKSImlHklgUXdYMu', 'nchar.新疆维吾尔自治区红霞市南湖成都路L座 163545', 1630000000008) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630135000000, 801515319, -3418248827901909079, 26288, 51, -534837.671505, 15.631926, 0, 'binary.DnbpUExxknvANXEEsaWR', 'nchar.重庆市旭市清浦冯路W座 960766', 1630000000009) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, 1651702520, 7472881261358437437, 27039, 39, -901147931976.785034, -21.400440, 0, 'binary.PDCCWnKlBJjPqTBmUhYK', 'nchar.广西壮族自治区秀英县秀英惠州路c座 534464', 1630000000009) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630134999999, 1705176915, 3853689828781845374, 27488, 40, 8339786734.762860, 74741492928.878296, 1, 'binary.jUuMQjVuORGeBYLOtCvN', 'nchar.四川省飞县海陵林路s座 610723', 1630000000009) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, 1401375995, 4401644438738533904, 10408, 76, -65049698292.958000, 762130648498.933960, 1, 'binary.NgPdruMUwoLdCxyEbNqw', 'nchar.甘肃省长春市南溪张街Q座 680073', 1630000000009) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000001, -2067806949, -5140997933871627569, -3032, -3, -771419731758.264038, 30386355.304924, 1, 'binary.XlCFISTlcLIRsxjboWHn', 'nchar.台湾省六盘水县清浦成都街a座 796270', 1630000000010) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000001, -1774018458, -2530948670547381542, -23399, -94, -495376178.216937, 151170.247760, 1, 'binary.TnrFRvylqBaNDthXRBAm', 'nchar.山西省帆县清浦昆明街Q座 225961', 1630000000010) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630135000000, -1363931402, 5604209803537258593, -18300, -23, 790034130.791400, 9237371157.649290, 0, 'binary.gyQzvuCzIfMlTIPPFNsH', 'nchar.广西壮族自治区淑珍市徐汇包路e座 156670', 1630000000009) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630150000000, -1894195697, -5734755902035702132, -15870, 111, -2670.233773, -4237.280417, 0, 'binary.UorBnZYRMylozHQezSJY', 'nchar.北京市英市清河张街N座 217253', 1630000000010) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 745446575, -9002015868827109100, 7664, 112, -360706186.393129, 47605535552382.398438, 0, 'binary.FZNKeCfgQVTcZyqRNJuK', 'nchar.贵州省桂芳县怀柔天津街b座 347544', 1630000000010) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630149999999, 672434773, 9109650904011996741, 14686, 14, -85079.408071, 86.251927, 1, 'binary.XKWsToLeAsNVsIygACBb', 'nchar.贵州省广州市海港长沙街d座 904810', 1630000000010) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 1515726636, 8521257827234172880, 11241, 103, -10.526128, -45196916520756.703125, 1, 'binary.zcHOLLudfLMvmIiXrANq', 'nchar.青海省六安市新城哈尔滨路Z座 312342', 1630000000010) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000001, -1923997635, -1295412466812568520, -20900, -107, 86532721.258284, 65558167665.583397, 1, 'binary.zNkNRRNyFRDVubonqLSh', 'nchar.广东省艳市东丽东莞路X座 591920', 1630000000011) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000001, -1357505482, -7556497983653555349, -27964, -5, 254673.529731, -273710012885.143005, 1, 'binary.oDZRgFAkFNMiHLGdpWcM', 'nchar.内蒙古自治区太原县秀英吴街F座 707601', 1630000000011) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630150000000, 720531993, -6639843123954951373, 1904, 87, 90228483.253351, 671689.823567, 0, 'binary.pbWPxvdZNrUoUOjzRJiX', 'nchar.湖北省柳市合川台北街k座 351073', 1630000000010) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630165000000, -1222920932, 8032055023952251119, -10742, -10, -567677.518580, -365410143.799096, 0, 'binary.gSPyErqDINWrLLVhCvup', 'nchar.辽宁省慧县牧野沈阳路d座 158793', 1630000000011) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, 414746648, 5927295586580216537, 31761, -42, 29.776511, -773061167437.739990, 0, 'binary.clWyILBusmbzejzbBoFT', 'nchar.福建省雷县东城曾路t座 587170', 1630000000011) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630164999999, 824419604, 891106057873489263, 23278, 95, 407384.105632, -35575.873087, 1, 'binary.MNQmhqizvbnJgkkIQqJe', 'nchar.广东省郑州市江北王路P座 783592', 1630000000011) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, 1281221057, 7024877965980246423, 27117, 31, -310144.477215, 7651054.553459, 1, 'binary.glajyvvlfYaEeVkkGCkB', 'nchar.西藏自治区博县黄浦沈阳街J座 833215', 1630000000011) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000001, -1830990548, -4649870365451699133, -7796, -55, -7445.199016, 5969245.537249, 1, 'binary.acIrBxurCVUcZrlhKNWy', 'nchar.北京市雪市城北佛山街L座 344309', 1630000000012) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000001, -156703760, -5651707522314260232, -20357, -53, -48419.774656, -412073702153.369995, 1, 'binary.RalpwNQKJazBBUIsHJxe', 'nchar.辽宁省阜新市大东郑州街k座 364193', 1630000000012) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630165000000, 862504464, -5500398940730781742, -14444, -90, 83.855205, -42218010095700.703125, 0, 'binary.TDihZaaECjQhZSuXOJkw', 'nchar.台湾省香港县滨城丁路O座 978512', 1630000000011) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 667837391, -3891675097346789325, -27238, -120, 1.955652, 9.112019, 0, 'binary.JzwDaPxfqmOCGLPvyrjl', 'nchar.湖北省合肥市朝阳贵阳路w座 542472', 1630000000012) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, -1060826174, 4656485123040609725, -2306, -49, -31070344.608357, 32855.487468, 0, 'binary.HuQooyRdCtqfxpfenEKD', 'nchar.福建省沈阳县门头沟合山街M座 860092', 1630000000012) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630179999999, 95309922, 7708532961668715531, 4871, 87, -2493.696406, -305230557.440988, 1, 'binary.sDzjVbzXXlrItKczJmNd', 'nchar.四川省彬市高明潜江街t座 525852', 1630000000012) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, 1834883689, 9087266843320529269, 10482, 90, 7620830515.391810, -42.560665, 1, 'binary.UNyjzeJnXfRpfYwqnoiq', 'nchar.宁夏回族自治区天津县门头沟韩街T座 460087', 1630000000012) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000001, -426435327, -3062037029747393786, -9095, -58, 3445.234181, -558141651179.500000, 1, 'binary.PPFhdHAqUnaKoAEmHsdU', 'nchar.湖南省霞市南湖石家庄街t座 857415', 1630000000013) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000001, -1181075026, -3180887222641598399, -31207, -111, -1782467190.826900, 5.516558, 1, 'binary.TGHKOSbljBlliohlXOWY', 'nchar.北京市华县南溪兰州路S座 340198', 1630000000013) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630180000000, -1179373775, -2798016462115894947, 13261, 81, -6.728964, -772564358011.891968, 0, 'binary.YugzktgmcxvJhQUFlTJx', 'nchar.重庆市大冶县新城蔡街a座 475876', 1630000000012) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630195000000, -1753829875, 9153441825434214315, 27033, 122, -736.689121, -92.766550, 0, 'binary.pUBZlnzauIFGdWqodaie', 'nchar.新疆维吾尔自治区海燕市高港刘路a座 320202', 1630000000013) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, 209604377, -5203497748513382294, 11811, 104, 6261702331.471590, 912558484347.494019, 0, 'binary.wZJaPJrGTgLaarzVLUSN', 'nchar.香港特别行政区武汉市海港长沙路y座 885641', 1630000000013) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630194999999, 927761777, 2507975814406403776, 19213, 90, -7617794.842135, -454202300.875154, 1, 'binary.icqbMuuToUNxqWXUeRPk', 'nchar.广东省西宁市丰都李路q座 597739', 1630000000013) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, 25571599, 9030074847417015485, 31156, 118, -75.396818, 48452341.676426, 1, 'binary.NqBtvtMnkQndvuYapikY', 'nchar.香港特别行政区南宁市长寿甘路u座 453721', 1630000000013) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000001, -428129781, -5700535713003042271, -10481, -28, 23726711.628846, -28428638085.690300, 1, 'binary.pCEqFOIEkNkTyynifvvP', 'nchar.西藏自治区张家港市上街阜新街O座 871243', 1630000000014) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000001, -848598147, -5339257050356453975, -18995, -126, -97029.616600, -142.756397, 1, 'binary.fnhJdkzuvTlaKyHNNFax', 'nchar.陕西省桂芝市永川马鞍山街k座 606648', 1630000000014) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630195000000, -1235643681, 7084534800526629066, -24801, 79, -145779524595.869995, 94.711588, 0, 'binary.zGptPlBGCsdKPNcWBTWB', 'nchar.湖南省莉市璧山王街S座 444986', 1630000000013) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630210000000, -257925835, 4113510788745659274, -7451, 82, -73680370.344940, 3976133.807981, 0, 'binary.RMHnWOvHGDkBnmnhajqK', 'nchar.甘肃省昆明县锡山东莞路f座 143739', 1630000000014) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, -441472850, 6731053996474537256, 25225, 88, 40422214874.591599, 23965.803627, 0, 'binary.EHzCItPZeIiFrqnZlZFx', 'nchar.广西壮族自治区丹丹市蓟州李街A座 716733', 1630000000014) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630209999999, 2038647205, 1189537624585781694, 7402, 77, -25786637314.127998, -8273884501116.790039, 1, 'binary.lQDfBPFCwRhZWQDOueFe', 'nchar.四川省南京县大东济南路X座 802062', 1630000000014) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 479757528, 6749026723270202494, 13649, 38, -11785225102.777700, 474555902413.706970, 1, 'binary.gZeGdOiDSRQDrrutAJpP', 'nchar.吉林省桂英县门头沟巢湖路W座 373162', 1630000000014) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000001, -973859244, -7125967416228878252, -23978, -103, 9.174338, -9074.207791, 1, 'binary.spVrNBOtDUJKWtVfGkIs', 'nchar.云南省重庆县秀英张街w座 814757', 1630000000015) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000001, -1099659644, -5069485856549254463, -7663, -21, -77435917488084.906250, 6157980965629.839844, 1, 'binary.neMplXKNDoTZmfaWuQIv', 'nchar.海南省嘉禾县翔安上海街P座 777755', 1630000000015) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630210000000, 1262139840, -4971682912616802373, 32119, 52, -763639.969220, 4466.886880, 0, 'binary.BedkvagFaejmBsJjHkQk', 'nchar.上海市玉英县东丽辽阳街q座 311759', 1630000000014) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 722222291, 150763568244594756, -22749, -105, 69152723689.975998, -5072701672145.320312, 0, 'binary.ZIoGBLOEmmhXJZvIxuMC', 'nchar.陕西省博市永川西安街H座 219040', 1630000000015) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 591411352, 513820708206298423, -29944, 120, -7450006471858.410156, 28263898844.649300, 0, 'binary.OOYGFQjeyKSgnzgKjpfl', 'nchar.甘肃省南昌县龙潭张街q座 299631', 1630000000015) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630224999999, 1729527872, 858836292464061984, 26768, 61, 8045315.625506, 463874.574633, 1, 'binary.qPTkezKHzqZjenBLvfin', 'nchar.新疆维吾尔自治区汕尾市海陵齐齐哈尔街e座 364606', 1630000000015) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, 1643785412, 3613608845439721039, 2830, 6, 20141358.132227, -10515528.243744, 1, 'binary.OHifAxUhqgMCYnGFqNtd', 'nchar.福建省惠州县秀英胡路z座 617394', 1630000000015) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000001, -1458828697, -6767897245471753991, -5337, -110, 26.347447, -2088415765277.760010, 1, 'binary.nKdzFeMTCkibzMHIBYWp', 'nchar.北京市武汉县海陵沈阳街j座 275130', 1630000000016) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000001, -847285817, -4303648033804475519, -29035, -126, 46793.514585, -1507581042167.719971, 1, 'binary.xMmlUvSqOUKadNTjwVbR', 'nchar.湖南省琳市沙湾张路b座 512648', 1630000000016) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630225000000, -978219428, -2955103196936757566, 30178, 32, 1631732545944.110107, -393.171524, 0, 'binary.seXNZyWFCBIKtovhPcPH', 'nchar.江苏省慧县上街宜都路e座 880474', 1630000000015) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630240000000, -1972126026, -7574542606285760315, 28119, 80, -529670.435949, -24383.929895, 0, 'binary.xhubWGtXmHdWMLKNToaW', 'nchar.甘肃省桂英县萧山邱街M座 532318', 1630000000016) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, -1796898694, -764951623915204003, -15993, 27, 4657602445.591260, 4586648.134693, 0, 'binary.TEhGCTdxkZNmFnzFtObM', 'nchar.甘肃省东县徐汇昆明街P座 260669', 1630000000016) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630239999999, 357526998, 1276204587787008311, 29238, 42, -5156150330.293080, 978546.269907, 1, 'binary.cTaghgmdTbnmdzLMfprn', 'nchar.青海省齐齐哈尔市海港汕尾街S座 223338', 1630000000016) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, 719321885, 6211071132372294861, 26203, 4, 97973007280575.000000, 4917172790.726020, 1, 'binary.UrLJfXtkYMRsRzBKupLN', 'nchar.安徽省永安市城东六盘水街x座 616834', 1630000000016) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000001, -131148573, -7418806044708315561, -24075, -95, -939211006187.709961, -62673687782.380798, 1, 'binary.ioRcxRqlBPWWUslmpXot', 'nchar.河南省瑞市崇文北京路v座 915899', 1630000000017) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000001, -926921347, -7119582382885284043, -22204, -33, -843815.847483, -733624.538013, 1, 'binary.CDgPdtLeDlVKfKVxhxpv', 'nchar.浙江省磊县城东阜新路q座 382954', 1630000000017) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630240000000, -256678368, -5278293054518319714, -19250, 63, 76804691.667068, 56757245265557.898438, 0, 'binary.mWDyzgzWsYmwcclLnyKM', 'nchar.新疆维吾尔自治区宁德县平山武路C座 669860', 1630000000016) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 105950788, 478455145201382299, 21202, 49, -492280820.857835, -59846.285166, 0, 'binary.bHXHFuvyzSODEypLHuvb', 'nchar.天津市惠州县兴山贵阳路q座 575823', 1630000000017) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, -1188272652, 3059329495007474279, 15772, 52, 97.388100, -179.239522, 0, 'binary.NcbSJdYAICljVIDdqLDs', 'nchar.河北省凤英市东丽曹路x座 451405', 1630000000017) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630254999999, 1374178023, 3719932182860509152, 15558, 26, -48778273445.594002, -4313900.906134, 1, 'binary.xrqMfwkMMYQAjZjHywqq', 'nchar.江西省广州市南湖汕尾街p座 889246', 1630000000017) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1682040807, 8317842464181056096, 1900, 87, 415547443.281930, 6910478.101601, 1, 'binary.CgZXfDNNfsnrMNDLhaFk', 'nchar.河南省海口县沈河长沙街D座 269675', 1630000000017) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000001, -1655416898, -6254408814439814177, -32731, -80, -926982830.629323, -59.656488, 1, 'binary.oerLzcmyCiDByTBvtqJC', 'nchar.贵州省关岭县涪城侯路y座 962245', 1630000000018) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000001, -1635083194, -8974109706346763955, -2735, -50, 3293.363373, 18681627453476.000000, 1, 'binary.GSVbHmgpelzbMoHbLPcC', 'nchar.内蒙古自治区晶县清城拉萨路X座 358275', 1630000000018) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630255000000, 1313669852, 6945103232995327793, 30581, 16, 85564937456162.203125, -77542.918719, 0, 'binary.XFHqNNaMVizYnZQgNXPC', 'nchar.新疆维吾尔自治区贵阳市崇文宁德街i座 343604', 1630000000017) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630270000000, 917363192, -2865889321649137964, 23695, 91, 5280836.562854, -3783947630346.270020, 0, 'binary.MNMoJuMUQGuncloQVsno', 'nchar.陕西省福州市吉区苏路s座 468218', 1630000000018) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, -2122120908, -2872995800597882541, 31983, 71, 6875560.276574, 5.493633, 0, 'binary.EPzGVtLuyIzhTxgbjEWF', 'nchar.山西省哈尔滨县浔阳蒲街L座 163917', 1630000000018) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630269999999, 795882903, 3280694140326918125, 20811, 108, 49.168498, 7651.321312, 1, 'binary.UikJWqrxtOeMoYKjVtNm', 'nchar.广东省桂芳县安次嘉禾街r座 604207', 1630000000018) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, 2120566264, 4722887562205021135, 28462, 1, 129.403026, 81642332991.346405, 1, 'binary.rFxHYhBonyaZGENLSiup', 'nchar.新疆维吾尔自治区建军市龙潭陈路N座 391792', 1630000000018) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000001, -426986357, -7028311428629812651, -4827, -64, -7.591949, -8288.667569, 1, 'binary.MTcHiKSZLMGfDPYnynMD', 'nchar.山东省辛集县东城汕尾路Y座 327226', 1630000000019) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000001, -1319979040, -1611680110566277558, -12591, -102, -0.766067, -9.985580, 1, 'binary.aSzFqnlqfbvcqqslssAX', 'nchar.北京市关岭县南长王路s座 855187', 1630000000019) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630270000000, -655731150, 952225036593449539, -12275, 62, 988070261.618785, 110828.718042, 0, 'binary.pwXbqQpeGiCAxVQjBByd', 'nchar.陕西省长沙县江北汕尾路F座 416889', 1630000000018) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1662865387, 6549115606741837921, 31658, -40, 6050748.290946, 850563444268.306030, 0, 'binary.PqxloDzsqJMLqHZQWCjg', 'nchar.上海市兰英市南湖乌鲁木齐路K座 550844', 1630000000019) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1925250353, -2675198429577937161, -954, 26, 42621483170.815201, -2.407178, 0, 'binary.UQoLagBLIXklApVNNHjj', 'nchar.福建省超县淄川大冶街R座 827801', 1630000000019) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630284999999, 1995360198, 5210480004052888682, 18624, 3, 97472570479742.500000, 1260.797791, 1, 'binary.ItgagWrnvwaUBcGaMaeN', 'nchar.广西壮族自治区重庆县平山香港路V座 260105', 1630000000019) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1294452263, 5523637851428821304, 15478, 5, 9507059.611693, 71182.768111, 1, 'binary.zGOYZBGdKbprwpIBOAyf', 'nchar.吉林省静县房山太原街o座 903227', 1630000000019) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000001, -1089491236, -4124928499125874309, -26711, -91, 102.262755, -86354.456039, 1, 'binary.wIAlEZNaSWVRvmnECAvO', 'nchar.浙江省梅县孝南广州街r座 602833', 1630000000020) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000001, -1857132633, -3826380662652726191, -2637, -78, -5244683893811.500000, 28720333520.104401, 1, 'binary.aCSuXOyNceiyMGbrnKiN', 'nchar.辽宁省海口市城东谌街E座 102059', 1630000000020) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630285000000, 1378099044, 2585214152437420786, 15693, -37, 9876.538042, -735632586.279405, 0, 'binary.SafiqluCtQtykHfilDhK', 'nchar.北京市雪市黄浦王路x座 719370', 1630000000019) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630300000000, 1474375413, 5114018721557230309, 17508, -51, 19062832.997828, -2871763657.353070, 0, 'binary.tQXLaCZMUMopSlkQosvw', 'nchar.甘肃省齐齐哈尔市花溪合山路F座 429081', 1630000000020) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, -388377907, -3327065272167311555, -118, 78, 891.667968, -939976.640106, 0, 'binary.vDLERQspegLuLpwaUwzM', 'nchar.安徽省潜江市南溪王街H座 114897', 1630000000020) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630299999999, 1778777098, 1247965499115398806, 31082, 36, -34708.444580, -9074056.186352, 1, 'binary.sSRJTkOuBEsIDUBYnvLI', 'nchar.重庆市倩市沈北新拉萨街s座 784670', 1630000000020) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, 1445490460, 8880599911239061729, 24463, 25, -3.382105, -1781392.553877, 1, 'binary.aoTqpxHvRtaSZcskkkvh', 'nchar.天津市北镇市牧野徐路j座 335208', 1630000000020) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000001, -414780851, -3272216576746070612, -22655, -77, 60986281647372.203125, 8206.582673, 1, 'binary.BYPTbfwFwATchckdDRGY', 'nchar.澳门特别行政区瑞市白云沈阳街D座 317572', 1630000000021) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000001, -1100845103, -4359114771451299690, -19221, -72, -454991497.458531, 66488519526.609200, 1, 'binary.IAUPjYJLQVMOrfjNhEGW', 'nchar.贵州省柳市丰都合山路I座 879430', 1630000000021) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630300000000, -1344260637, -2327749124997496795, 8396, 9, 53600362.915844, 39096334.438932, 0, 'binary.MCcnFYqPmshweRolVKXy', 'nchar.福建省辽阳市黄浦武汉路Z座 588915', 1630000000020) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630315000000, -355362703, -1297695182610353342, -9770, -89, 952785.346586, -74060980932193.796875, 0, 'binary.oxSCvbpFwbexlztlNbov', 'nchar.江西省阳县徐汇孙街A座 417152', 1630000000021) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, -1228614638, -3900819433463392818, -17590, -11, 8486255712070.299805, 776309237079.099976, 0, 'binary.lCmzbzWrnzVGWTlKKcLz', 'nchar.江西省健市花溪齐齐哈尔路s座 401365', 1630000000021) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630314999999, 2119897062, 705747859773422308, 11407, 81, -7145565.865688, -262229423.217684, 1, 'binary.eEvZPvUuQslbYxeHtVwF', 'nchar.山西省马鞍山县兴山台北街K座 317962', 1630000000021) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, 1675027583, 7700778986059081977, 16174, 9, -665864335638.520020, 71582.521862, 1, 'binary.tKmhWfIgqpVzDnSlVNyX', 'nchar.澳门特别行政区银川县高港侯路B座 867370', 1630000000021) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000001, -63872001, -6930483055721144761, -55, -27, -647803039.367343, 2303820.355916, 1, 'binary.cMZeLPHbvchDdpONluWD', 'nchar.海南省长沙县浔阳俞路y座 710843', 1630000000022) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000001, -550949355, -2196304875670875984, -11885, -39, -6711213741.192790, 743976.503579, 1, 'binary.ZjvtDrDegMEemNdVPYWI', 'nchar.江苏省丽华市东丽林路c座 701423', 1630000000022) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630315000000, -638988531, -238160149511091433, 5704, -64, -8.456931, 122326752.221053, 0, 'binary.bOAzwJsCnDilLpivHhcE', 'nchar.福建省台北县新城佛山街W座 190141', 1630000000021) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630330000000, -626643923, -2019791703137966083, 32109, -84, 3416720.968494, 9357038.426241, 0, 'binary.BaFBcjklAnYcYwGxAqoq', 'nchar.吉林省武汉县朝阳宜都街u座 246031', 1630000000022) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 1744535925, -781319618567345160, 25699, 25, 5357.659882, -618.983367, 0, 'binary.VPobijbSHTufuAsIpKoJ', 'nchar.吉林省建华市城北刘路m座 189127', 1630000000022) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630329999999, 880112073, 3294754321072095965, 6900, 77, -89650901.379152, -40.193724, 1, 'binary.slgqehRyEWrrSqbAYVJh', 'nchar.甘肃省佛山县吉区吴路i座 371158', 1630000000022) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 185074156, 4989180079312042024, 29358, 109, 553959035.299975, -336.524434, 1, 'binary.gsSAVskcQYNhhgdMqRao', 'nchar.内蒙古自治区海门市新城许路M座 262464', 1630000000022) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000001, -1466263570, -2228588793836525866, -21221, -43, -87005519974.974106, -660.194761, 1, 'binary.bxpguIjMdJUkxwsOGQEq', 'nchar.新疆维吾尔自治区杨县徐汇福州街W座 871563', 1630000000023) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000001, -1645451433, -4401356942676794117, -28334, -111, 279.991987, 22516.349037, 1, 'binary.dfdHQJlJuaDksxoabJoB', 'nchar.西藏自治区太原县涪城哈尔滨街p座 883295', 1630000000023) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630330000000, 2032301580, 4408736721579229542, -1683, -10, -14742.318595, -7092296.586329, 0, 'binary.GExwHTtWGjhZjTEeFYiG', 'nchar.江苏省大冶市闵行高街t座 870548', 1630000000022) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630345000000, -2137144022, 4079790010886186881, 25710, 117, -4064381240984.290039, 7015370675970.000000, 0, 'binary.CVqxsKDzVLudZenizkMH', 'nchar.澳门特别行政区广州县牧野北京街x座 612318', 1630000000023) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, 933500773, -9037818060698530179, -29395, 110, -635450613.922866, -7403.402429, 0, 'binary.BAfDibZjCIxGGNhZPzdE', 'nchar.天津市玉梅市六枝特曹街z座 306379', 1630000000023) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630344999999, 1707978124, 7108213046724490283, 23442, 53, -1690092798852.899902, -77.732195, 1, 'binary.CjoXkwnzCpbREZGghsfa', 'nchar.天津市俊市南长成都街Z座 386069', 1630000000023) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, 638219332, 7562431303377412667, 16354, 108, -1426154.974943, -5203.623256, 1, 'binary.YWcLHHceXXwuYatdWqpB', 'nchar.海南省武汉市闵行佛山路i座 350356', 1630000000023) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000001, -694792584, -5451159991441438649, -27358, -63, -421814477.346397, 658.908206, 1, 'binary.RjeNtbCwKmVzuSEznMIP', 'nchar.青海省兴城县双滦嘉禾街J座 854125', 1630000000024) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000001, -1486705498, -4529672645278837682, -21086, -87, 0.969525, -88911627001.666107, 1, 'binary.lFWwkNVbMUovcuYFIddp', 'nchar.澳门特别行政区拉萨市和平合山街j座 616189', 1630000000024) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630345000000, 1470276506, -7928540339509515647, -16251, -106, -75636523821237.203125, 1951788289389.639893, 0, 'binary.gAaAcTqtqUaUtILxCARx', 'nchar.广东省嘉禾县清河六盘水街k座 243900', 1630000000023) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630360000000, 1152276268, -8156217200537467815, 19660, 12, 28355311.730820, 6713567748470.700195, 0, 'binary.bmGZNDQVyGtLdMWCUUPh', 'nchar.云南省杰县南长兴城街P座 252275', 1630000000024) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, -504752588, 1971807127747770396, 27445, 121, -8539663.125419, 54448077.614627, 0, 'binary.cuEEkfEDPsfuHYlKnDOH', 'nchar.西藏自治区坤县金平澳门路Y座 688701', 1630000000024) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630359999999, 855880887, 4257747801443019050, 4002, 59, -651877334084.171021, -40661.869299, 1, 'binary.ZOOiXFRmbNwocDEGjyPI', 'nchar.山西省香港县沈河汕尾街w座 530063', 1630000000024) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, 24228908, 7366685839096261214, 24706, 108, -2895.104902, -690789994021.112061, 1, 'binary.WoiDFgqNXTSOSSqvghbK', 'nchar.香港特别行政区哈尔滨县萧山重庆路S座 626140', 1630000000024) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000001, -1979347419, -8086150063002929949, -31773, -59, -6758.131288, 67.389494, 1, 'binary.BdPhqWnyifozXNBeGuOa', 'nchar.吉林省重庆县东丽齐齐哈尔路T座 928550', 1630000000025) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000001, -796374629, -6586675874501163952, -28989, -41, 64993999.806191, -8554109097.403460, 1, 'binary.AjiqARctYieemihXJRgw', 'nchar.广东省嘉禾县房山李街d座 910194', 1630000000025) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630360000000, -2078983709, -5723824485424673203, -23688, -93, 7495606593.360470, 264203417504.108002, 0, 'binary.LvbrLfgWtSAHPeWnNyif', 'nchar.重庆市峰县华龙廉街i座 702040', 1630000000024) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630375000000, -2135854035, 3495311946004641041, -8066, -32, 794122420146.477051, 54762.831473, 0, 'binary.iKNdSNvujYgDbKTFUoWj', 'nchar.湖北省秀珍市东丽佛山街a座 727430', 1630000000025) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 195905849, -4893460212808423867, 27472, -20, -60.648891, 9236.500458, 0, 'binary.fxBWbjZbbgBzyftfClnE', 'nchar.云南省春梅市普陀合山街d座 593581', 1630000000025) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630374999999, 1222537098, 2598300007761422980, 13555, 60, -21237649962112.101562, 56425481952996.296875, 1, 'binary.nljxYbIMonhZyPfyRWYU', 'nchar.河北省澳门市大兴贵阳街U座 357038', 1630000000025) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, 1255129224, 3767117331869115149, 20415, 43, -96039955.962346, -3228760442.535840, 1, 'binary.VjBuTRZJfGBAsllbmlCm', 'nchar.上海市潮州县长寿大冶街V座 568435', 1630000000025) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000001, -451008604, -9013008932944649264, -4123, -55, -858251929.793158, 54.556567, 1, 'binary.CGijFVktOvKYBbslrFnW', 'nchar.甘肃省丹市牧野刘街b座 221301', 1630000000026) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000001, -994075958, -3836907429942473253, -15968, -12, 4.809100, 97814318457.744095, 1, 'binary.ZkpIzssjZLBfwxySZcqJ', 'nchar.吉林省惠州市兴山辛路i座 132947', 1630000000026) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630375000000, -316707220, -6874259000415830359, -14656, -107, 9783647610530.900391, 6.913912, 0, 'binary.stonHOVphDrwzsFtAuEM', 'nchar.吉林省杭州县秀英任街B座 197635', 1630000000025) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630390000000, 1162924990, -5170396701176744771, 20522, -92, -96439892251.988800, 27285284869935.500000, 0, 'binary.VaAkDRQGuwRsRuHuWTBT', 'nchar.安徽省刚市璧山严路e座 453358', 1630000000026) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, -1787695146, -7200438941294278654, -1976, -22, -19864559903432.601562, -81662701579034.406250, 0, 'binary.cmeIZxizSzJowehZeTgi', 'nchar.甘肃省萍县吉区雷路Q座 278197', 1630000000026) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630389999999, 1569291471, 7500808657751046307, 17406, 11, 187598077622.457001, 9402508122.216591, 1, 'binary.rGPAUjewRUiNxuGlPQSI', 'nchar.湖北省宁德县永川邓街N座 873981', 1630000000026) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, 379511627, 959739624838461444, 32645, 40, 27073116046.845501, 9680357.747312, 1, 'binary.EnkonjEPYvAZMcbrIDhm', 'nchar.安徽省宇县南湖拉萨路l座 319850', 1630000000026) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000001, -367057952, -5935523220947872265, -31354, -124, 60936491.153154, -7459361.174603, 1, 'binary.ehReUikosMxxEsGOsivE', 'nchar.河北省宁德市六枝特武汉路j座 479101', 1630000000027) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000001, -1204076280, -9028857053490170794, -4305, -36, -4299.875188, 8440721199484.889648, 1, 'binary.LDFNunrLqdOUVBNKOEgL', 'nchar.湖北省西安县六枝特淮安街d座 633128', 1630000000027) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630390000000, -1734305566, 9014243322464991494, 2526, -91, 46980567127.508904, 2736.398653, 0, 'binary.kCqLuZWcVXncorOcRLOX', 'nchar.吉林省秀英市永川姜路P座 763227', 1630000000026) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 493810102, 750963322474159528, -23750, -17, -22892331.104465, 24868217245883.800781, 0, 'binary.CFELPsklozusjREurYzN', 'nchar.浙江省云市吉区丁路H座 432525', 1630000000027) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 1463044753, -1849311824110235905, 30811, -7, -17541415231.352200, 4359875.313503, 0, 'binary.XztqpyyklxsOpFMGQpdQ', 'nchar.内蒙古自治区兴城市和平李路f座 695390', 1630000000027) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630404999999, 1007184435, 7016082283155873873, 18680, 114, 921004.858357, -3749822553130.689941, 1, 'binary.nPsHiYGSrumTPlRhGlGo', 'nchar.内蒙古自治区长沙市海陵六盘水街Z座 569551', 1630000000027) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 2001976992, 6444735120490802040, 10757, 75, 3812894583.717430, -70452173102543.796875, 1, 'binary.MPllmpddghzaoIRJmAJs', 'nchar.河南省西安县崇文李街d座 986692', 1630000000027) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000001, -1748589793, -7997812859393713894, -9028, -46, -35.226372, -6653121231.857540, 1, 'binary.eVMhMPninMDMThzngiZC', 'nchar.江苏省鹏县长寿深圳街V座 475898', 1630000000028) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000001, -952046898, -1003514831999138377, -10475, -121, 583096872.778729, 750.380516, 1, 'binary.NhUrjkWQZoYMHNgZcAur', 'nchar.辽宁省畅县南溪陈街Q座 816736', 1630000000028) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630405000000, 1203696901, 5857368026922559698, 26958, 20, 7943395.425475, 3479394471444.229980, 0, 'binary.EWZLkQdNEMzlxMRcWPIF', 'nchar.河北省凤兰市秀英王街h座 943022', 1630000000027) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630420000000, -1140109939, -7140478781384267748, 23888, -102, -463251528.501850, 58.564400, 0, 'binary.wBOWZRMsrPHnjovxpVtM', 'nchar.福建省桂香市魏都张街s座 891983', 1630000000028) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1596211207, -6399610783775225615, -6948, 18, 5279544962.478530, -3254512966322.680176, 0, 'binary.pkLdHCSKTBJtvlkQZFel', 'nchar.宁夏回族自治区志强市沈北新哈尔滨街V座 984662', 1630000000028) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630419999999, 1987333617, 2406177688984023332, 5515, 81, 93334135.435757, 15200180248105.500000, 1, 'binary.zunblBzpjKjvAprKttjO', 'nchar.陕西省娜市六枝特长春路u座 734021', 1630000000028) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1816593353, 6144436182936056183, 11139, 111, -5111160.608439, 3134629841.427010, 1, 'binary.SvSSddOyqqrLVbTuqmWK', 'nchar.黑龙江省北京市沙湾天津路l座 864505', 1630000000028) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000001, -1092293083, -6497162148443342823, -12861, -33, 7333968.460815, -786239055.465100, 1, 'binary.abpCKdOYaAAPNEGSEKTl', 'nchar.重庆市通辽市白云席路d座 390244', 1630000000029) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000001, -672330807, -1416095251335444839, -31617, -20, -268.247197, -187856842551.350006, 1, 'binary.ayrYptkczwLwwHSlrotJ', 'nchar.安徽省莹市崇文卓路D座 742319', 1630000000029) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630420000000, 1541531440, -8276464445557829911, -32389, 41, -205002775.660640, 773381214983.128052, 0, 'binary.DfbOMiMBChkQyxVRkVrV', 'nchar.江西省重庆县东城嘉禾路z座 664431', 1630000000028) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630435000000, 511355972, -4607655896423993759, -17667, -115, -72721.674990, -0.210169, 0, 'binary.TzInKrTBzLfzhusnMyJt', 'nchar.西藏自治区大冶县黄浦夏路c座 672141', 1630000000029) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, -794567134, -2235695034213568259, -18310, -76, 80256151932.389999, -0.448521, 0, 'binary.rSBKSseTjokrDbbUwbdS', 'nchar.新疆维吾尔自治区南宁市璧山王街D座 205832', 1630000000029) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630434999999, 1590979273, 8094673699827428920, 1803, 48, -6923.890078, -97907136406253.203125, 1, 'binary.GnrqLzMLFlinSCSmzsTt', 'nchar.辽宁省荆门县六枝特邯郸街p座 507994', 1630000000029) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, 1132029984, 2532896687056073497, 20663, 78, 72584604.270350, -52558156996.426201, 1, 'binary.hUBVgUHTpsSwyjcCwqYn', 'nchar.四川省邯郸县南湖张家港路V座 697294', 1630000000029) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000001, -214047405, -8929808789293020890, -14561, -75, 57922.963900, 629338050.201999, 1, 'binary.rUFdjCoxExrGdHJRdNrS', 'nchar.香港特别行政区艳市魏都香港街w座 547058', 1630000000030) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000001, -1769159233, -2980306744069119372, -24737, -85, 95463.992658, -3290730049210.250000, 1, 'binary.bcUBXvglPoTsHoRWOtZB', 'nchar.青海省建国市蓟州罗街V座 702124', 1630000000030) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630435000000, -787830057, 3539391148256582010, 8967, -84, -3405.578845, 6448387382.497230, 0, 'binary.nXPgVFiqtkagYaeMiBqc', 'nchar.宁夏回族自治区兴安盟县梁平沈阳街J座 590779', 1630000000029) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630450000000, 1186123659, -7951052163704337820, -29743, -14, -83621685835717.906250, 73159143.385460, 0, 'binary.VKjqDeckRehqfqnstQEu', 'nchar.河北省上海市门头沟梧州路n座 288381', 1630000000030) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, 1878200309, 7041626075528835244, -32078, -31, 37394.924981, -492.884685, 0, 'binary.tSYeYbpjVnhDUVatIqqr', 'nchar.云南省齐齐哈尔市朝阳王街j座 235227', 1630000000030) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630449999999, 1216265979, 1946865098328593924, 13349, 71, -8448.993057, -74684363814152.500000, 1, 'binary.HIQrJxrPESpsHxBeQTJr', 'nchar.香港特别行政区关岭县魏都呼和浩特路i座 304697', 1630000000030) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, 1436995521, 3840076561934756705, 24645, 53, 3385565140.362020, -18217011452.326302, 1, 'binary.RBEtocJRHaAIuNaYJMhP', 'nchar.辽宁省重庆市江北张路g座 561300', 1630000000030) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000001, -704813545, -8641984224483414170, -32490, -72, 305492940.896881, -3375029817498.339844, 1, 'binary.MUkAAGQFoHZIrYzgZIkn', 'nchar.甘肃省佛山市六枝特梧州街p座 155461', 1630000000031) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000001, -222045952, -7186471281941680094, -4325, -12, -4.504288, -107.133595, 1, 'binary.rYHScGxtnPQZbrvqmGLs', 'nchar.浙江省斌县西夏宁街i座 817397', 1630000000031) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630450000000, 1430474057, 6099257513015763510, -25953, -124, -4692.193024, 4.787536, 0, 'binary.ECCSIBNtxKQLYBfOfsJg', 'nchar.天津市西宁县璧山曾街N座 665290', 1630000000030) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630465000000, -1933715256, 1674195434522463261, 18595, -62, -2.112704, -1.361571, 0, 'binary.WFmGDlZbQZbxQoPTeYYD', 'nchar.江西省帅市秀英罗路i座 713857', 1630000000031) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, -1427366603, 348136735706226150, 11786, 45, 998.343140, -9.796246, 0, 'binary.mfnVlbsKfzjjqorEhocA', 'nchar.西藏自治区玉市安次成都街E座 433538', 1630000000031) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630464999999, 1659717258, 7758886491614905150, 8304, 125, 6399945699.318400, 86010265260.434601, 1, 'binary.PFttwdRtnOphOvIvENNh', 'nchar.山东省欣市沙湾孙路R座 922051', 1630000000031) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, 2080213156, 8048809385397600433, 18758, 65, -544332786.800377, 2865457.223765, 1, 'binary.oElRbKMuPpzhWDlPDXtl', 'nchar.北京市淑华县江北李路F座 469619', 1630000000031) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000001, -1260757693, -7052648703184778467, -19691, -58, 263396039624.993988, 4278958.521790, 1, 'binary.CZUckqTDCqkrcZvnYVOg', 'nchar.广东省秀珍市清城谢街t座 351701', 1630000000032) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000001, -716674007, -2090666240981221473, -8986, -13, 40900.288977, -311075618204.403015, 1, 'binary.svGVeKCsLqnIefICQikd', 'nchar.宁夏回族自治区哈尔滨市南湖梁路v座 181451', 1630000000032) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630465000000, -1114362408, 5527740642432952035, -26428, 12, -21611.508885, -307.575004, 0, 'binary.UdnFsgeAgiPHPPgFGuCS', 'nchar.天津市成市朝阳沈阳街c座 871777', 1630000000031) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630480000000, 572636870, -1348714871953192727, 11880, -9, 78191326.534929, 5.863674, 0, 'binary.ELYllxOgSmXkAHYWCKeC', 'nchar.河南省六安县和平惠州路b座 192567', 1630000000032) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, -1554586734, -5797603791431465981, 19152, -58, 376.263188, -9117680341029.800781, 0, 'binary.ItnjMhkccvDshKoJDFBL', 'nchar.广西壮族自治区上海县高明潜江街i座 632153', 1630000000032) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630479999999, 1889928969, 2650453196149359117, 1531, 101, 221512.929316, -89455817137.120804, 1, 'binary.JuOPYpivSOrAxirELGaK', 'nchar.云南省广州县清浦潜江路y座 321604', 1630000000032) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, 123268180, 7324096614566287829, 18716, 12, 727218062004.973999, 656857194.138679, 1, 'binary.LzrEjFPpRgwZHAABcyJl', 'nchar.宁夏回族自治区兴城县沈河济南路A座 434702', 1630000000032) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000001, -739224311, -3272068915371457037, -11723, -49, 952488.600157, -831167.359733, 1, 'binary.KMwlcGqAhCieyQLGVtBU', 'nchar.河北省玉梅县徐汇张街u座 155803', 1630000000033) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000001, -1363430935, -5467097084273074766, -2392, -77, -2426746190.920940, -72743043.214903, 1, 'binary.FYLJECmRlLXbWxOIZbkA', 'nchar.新疆维吾尔自治区荆门市沈河丁街Q座 909413', 1630000000033) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630480000000, -1790122655, -8076464905134511680, -30494, 39, -300117474923.549988, 64738074469674.398438, 0, 'binary.sleyeDsmwcqZzCKoASFB', 'nchar.江西省香港市西夏沈阳街A座 666990', 1630000000032) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 500469874, -345787852149044362, 31942, 66, 218.280717, 7741245996.308220, 0, 'binary.WbODLLMuYAWsCiVGykXg', 'nchar.甘肃省哈尔滨市萧山邯郸路M座 577140', 1630000000033) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 1790655569, -8921791989390937548, -7521, -10, 21190031.941426, 372852.384613, 0, 'binary.pvccoVhryCwBmdnzLuTc', 'nchar.云南省潮州市沈北新吴路C座 166459', 1630000000033) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630494999999, 175625025, 4866859483197899046, 22179, 83, -52019.335200, -315117.422010, 1, 'binary.EPoDgXsUUMvKeLUulmOd', 'nchar.上海市呼和浩特县孝南淮安街D座 688832', 1630000000033) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, 531699098, 9087043167036546370, 10882, 69, -343189.459391, 211280.823556, 1, 'binary.gZdLTctTatBiWQSJIJOv', 'nchar.黑龙江省鑫县友好石家庄路G座 894253', 1630000000033) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000001, -128692601, -3691685846078930525, -2891, -13, -473237.431473, 6.861402, 1, 'binary.PevuHflVwLyOeexwpRis', 'nchar.广西壮族自治区桂花县和平石家庄街b座 197395', 1630000000034) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000001, -1382111225, -8179715729898925736, -27170, -65, 94.689652, -9.323222, 1, 'binary.dQukgnOBymmhDJsngvdP', 'nchar.河北省西宁县六枝特谢街w座 636313', 1630000000034) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630495000000, -1067182718, 4525978413594477317, -15988, -12, 9592970.107203, -5857432670.557230, 0, 'binary.KmUaBLARuhpFnBlZhhjz', 'nchar.广西壮族自治区玲县丰都伍街N座 160120', 1630000000033) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 834635110, 7524163623717860346, -32317, -74, -15228783.766541, 189210.220590, 0, 'binary.EMKiAeOicxSQHtyaVgJp', 'nchar.广西壮族自治区洋县沙湾太原街b座 616020', 1630000000034) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 969850854, 946247056457664769, 21254, -118, -665960476591.563965, -26272792213245.500000, 0, 'binary.AmemAqKVRuTlgnqegQTU', 'nchar.四川省海门县牧野潜江路x座 318206', 1630000000034) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630509999999, 1292293650, 4182146421220552346, 9438, 26, 6515565137650.730469, 4326564.636935, 1, 'binary.sjAdlJbnpCLUoyTzXGLz', 'nchar.山东省杨市南湖刘街Y座 225115', 1630000000034) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 122648893, 3568976233348927576, 4722, 101, 3692.808659, 8.426917, 1, 'binary.idKTOjMQWLcoovoLZCRY', 'nchar.天津市关岭市锡山蔡街f座 426508', 1630000000034) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000001, -103653848, -6515029160766267522, -13842, -21, 461152.967231, -451711.406503, 1, 'binary.axaNePeGlnNRTVkjQgqj', 'nchar.黑龙江省芳县海陵贺路c座 606589', 1630000000035) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000001, -1047587411, -3104445727812638863, -8586, -20, 551018.785658, -69672859.800138, 1, 'binary.rrQrHCANqfAEkPaQtSxY', 'nchar.江西省辛集市平山南京街X座 619416', 1630000000035) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630510000000, 849850958, 4247759734843437140, 25856, 61, 216972633.179206, -688453.180700, 0, 'binary.HFGqhFMKQtiuVGsRfHzo', 'nchar.台湾省柳县清浦长春街g座 249559', 1630000000034) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630525000000, -1263472959, -6789381028960214442, -24794, 114, -380257099.745964, -4055421597.527550, 0, 'binary.vYeuboTOIvXZsvEDJLRp', 'nchar.云南省淮安县浔阳南宁街s座 962460', 1630000000035) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, -912546236, -763230547896477749, -22937, -100, -61226717.838399, 59403.615805, 0, 'binary.RfopmCaAJeNFUhoHdGfp', 'nchar.北京市嘉禾市海港梧州路Y座 185275', 1630000000035) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630524999999, 1491974080, 3628020701059333562, 19419, 61, 87024838.439295, 563459.869036, 1, 'binary.MGCglbwygRPypvScTmBl', 'nchar.山东省霞市吉区哈尔滨路U座 372048', 1630000000035) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, 1133422966, 4026166323599059583, 27738, 105, -67481518.148927, -5675366548881.980469, 1, 'binary.LFpWlYqzLoxtHDCeMOWl', 'nchar.黑龙江省北京市沈河王街n座 817738', 1630000000035) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000001, -104646696, -5475621281854878817, -8285, -77, -25646075.949196, -671.169300, 1, 'binary.nEdnhlzGKPxpfWnmznnT', 'nchar.辽宁省凤兰市上街李路t座 977250', 1630000000036) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000001, -62177409, -7664099839058566322, -13160, -111, 2741718151170.200195, -3.737565, 1, 'binary.kkhcrwzjebqpTkpaFkPz', 'nchar.山东省淑华县六枝特长春街L座 836041', 1630000000036) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630525000000, 1330077191, -57050396993719408, -17322, -114, -17573257017538.599609, 131480451841.740005, 0, 'binary.nBjEjjCxPpyJZZokLLYj', 'nchar.辽宁省拉萨县普陀香港路u座 175740', 1630000000035) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 1361888973, -4754878850289093930, -8386, -123, -73755105134.646896, 593521593.672730, 0, 'binary.aaJdLDhVgqXgbzOLtgEC', 'nchar.陕西省玉兰市城东东莞路y座 735950', 1630000000036) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, -2023047539, 207198366842950509, -16454, -51, -5064990.460660, -1880.742682, 0, 'binary.pgtomoahTdklLDwxLWlX', 'nchar.天津市哈尔滨市普陀赵路a座 285719', 1630000000036) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630539999999, 1161599505, 3137635940724704617, 16998, 16, 341287.343410, 3260540.642346, 1, 'binary.VpvoOstBpXhXgCFmTNbu', 'nchar.江西省坤县沙湾拉萨街f座 882807', 1630000000036) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 927677922, 1982533905502059580, 28136, 88, -554.710032, 860334626423.604004, 1, 'binary.xigdKxGBhwDummTjgWpP', 'nchar.甘肃省兴安盟市大兴李街R座 743679', 1630000000036) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000001, -1715472820, -1384061899373736161, -15979, -21, -144230.332607, 3166307136.273820, 1, 'binary.keFGWYyGPVWHcgVZNKld', 'nchar.江苏省莉县沈河王街j座 843427', 1630000000037) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000001, -1691042073, -7186599605570808771, -14609, -94, -20270760488095.101562, -479065667910.349976, 1, 'binary.qNCwJqmtvmXEEJXuqVcD', 'nchar.安徽省鑫县西峰香港街Q座 197792', 1630000000037) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630540000000, 1494052790, -8128813094838655682, 26938, 0, -5026361458.157280, 48738437612.827003, 0, 'binary.beRVwfLpgmeKWVAywXMR', 'nchar.重庆市西宁县沙市南京路l座 534992', 1630000000036) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 611413897, 4152857341280270300, -8759, -18, 2060373600.882190, 661904.107513, 0, 'binary.VNQYBgSikmUjTzZJlnVf', 'nchar.台湾省潮州县永川莫路h座 405659', 1630000000037) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, -2011948818, 7185570718886510818, -23571, 37, 224031795.273930, 529323.833439, 0, 'binary.gdqheapFdDFaeWrvvlFQ', 'nchar.黑龙江省丹市孝南白路Y座 681449', 1630000000037) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630554999999, 654809776, 1038278314990977803, 21715, 111, 9233.222241, 875.958899, 1, 'binary.zeHdYSkOUqnFvykwmfjx', 'nchar.山西省飞县淄川大冶路o座 582862', 1630000000037) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 1280839364, 1109786269551140763, 28107, 83, -7163.528250, -98949521955313.296875, 1, 'binary.GPZyYjBdWvtWXNqtBBAi', 'nchar.吉林省秀云市浔阳石路k座 906496', 1630000000037) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000001, -1722544909, -25495015738455645, -17889, -9, 8051193952.303000, -0.243396, 1, 'binary.NUGayWaiLgkIttYyWPen', 'nchar.江苏省帅县合川嘉禾路d座 756609', 1630000000038) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000001, -712202888, -3214543942801667949, -11436, -24, -4230794569.824540, 541.605598, 1, 'binary.NPvfKAwLNUBrUsbNQBjL', 'nchar.青海省晨市长寿深圳街y座 565486', 1630000000038) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630555000000, 238745662, 5355356224239767868, -31138, 41, 22.830003, 4601.537342, 0, 'binary.PeqIYBTzojcHPPpbASyS', 'nchar.台湾省邯郸市六枝特海口路s座 355514', 1630000000037) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 745693189, 4651138598564121948, -3614, -40, -41.868811, 96833493.796392, 0, 'binary.MRNQDZhMkAPOMwRDPLza', 'nchar.新疆维吾尔自治区坤市南湖蒙路z座 268958', 1630000000038) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, -2145996442, -3433600130799541776, -10809, -63, 855480.426782, 668870.374284, 0, 'binary.FlljuYVtUMddfJeXCePu', 'nchar.上海市林县永川深圳路h座 254543', 1630000000038) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630569999999, 1261953798, 314028281416970194, 7113, 92, 57276757.143326, 9980201888194.929688, 1, 'binary.CWKxzlTunbzsSgrpGTXp', 'nchar.河北省淑英县南湖武汉路Z座 875483', 1630000000038) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 1435842767, 6578024513554467170, 17462, 68, 3980927396.535860, -442434194.149287, 1, 'binary.VGfEsMVqNVUjawdSRhQz', 'nchar.重庆市深圳市西夏刘路N座 212798', 1630000000038) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000001, -1529141111, -2552308375560945509, -3088, -52, 65625.681921, -544.669991, 1, 'binary.bfdhAjhpOMjvNqKKrBns', 'nchar.西藏自治区杨县崇文永安街j座 354402', 1630000000039) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000001, -541899245, -5991226110563542766, -707, -107, 35892171731.112297, 385.252012, 1, 'binary.TbAKAMxnTbMsqbiNmVzL', 'nchar.香港特别行政区南京市兴山广州街t座 824478', 1630000000039) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630570000000, 101231908, -1793754924693109102, 28121, 5, -27297115.590120, 235254039.488185, 0, 'binary.viiOyqfWQeKCCeqjJSna', 'nchar.台湾省秀华县沙湾辛集路L座 217819', 1630000000038) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630585000000, -979159064, -4405552399360058755, 29025, -61, -213.670605, 43.565732, 0, 'binary.VTPMgOjAfDTIkXvCBRvm', 'nchar.北京市莹县崇文万街c座 320902', 1630000000039) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, -732603316, 3291957888362983413, -27560, -5, 19536.856481, 81.835797, 0, 'binary.XyZczHSyVrPUQdbZGVqI', 'nchar.河南省香港市蓟州蔡街Y座 824988', 1630000000039) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630584999999, 1148581502, 1746383234690030123, 6204, 56, -838550587.270660, -7435199862.146900, 1, 'binary.jcVlbATHJidNqVsLaIMo', 'nchar.河北省关岭市浔阳任街A座 776594', 1630000000039) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, 1999818388, 7210586798654528078, 1630, 18, 6208.528721, -335.584522, 1, 'binary.aoQUSVWoTxqKJPqRozVe', 'nchar.河南省平县滨城兰州路a座 637800', 1630000000039) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000001, -884290007, -4556726278944994370, -16568, -42, 3252403692312.240234, 6802621362.778860, 1, 'binary.ppFYTcNMeEFxKGYyuqxK', 'nchar.江西省淮安市城东潘路U座 501030', 1630000000040) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000001, -1116869032, -271141834781987082, -15740, -33, -9594782340313.500000, -19702686640.119999, 1, 'binary.ptSTibfgYgBqxXBcljXJ', 'nchar.上海市乌鲁木齐县上街吴路k座 975956', 1630000000040) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630585000000, -435132639, -8699906763762846441, -23184, -19, -4717893819233.429688, 8709722.643050, 0, 'binary.RcLPKAxeNyCrFVgZBWDc', 'nchar.贵州省婷婷市永川香港路M座 438076', 1630000000039) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 995611137, -3751963808881697081, -16228, -21, -96744.522458, -531.564886, 0, 'binary.uctNWRFZwpiODjjBdURc', 'nchar.江苏省琳县双滦乌鲁木齐街W座 315778', 1630000000040) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, -701305447, 7350533321420278894, 12461, -14, 637529643632.400024, -37006.524818, 0, 'binary.nBvNWvoxhJhaxlrXNwSq', 'nchar.河南省澳门市海港骆路d座 670505', 1630000000040) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630599999999, 2038087184, 3229395695102168112, 32164, 120, -60786248.451037, -66349595222.869400, 1, 'binary.jnEtTqkFChOMBYyRcHDQ', 'nchar.陕西省桂珍市秀英徐路N座 443126', 1630000000040) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 2133331669, 3831296733741228684, 640, 101, -22204789596077.000000, 12726162933.525200, 1, 'binary.uKmymrIXqCovPNkWAJLg', 'nchar.广西壮族自治区瑞县龙潭深圳路g座 407714', 1630000000040) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000001, -1455494423, -4099795228613872218, -24485, -97, -42580895412.277603, 438688.561578, 1, 'binary.GqYiJMfzdIKHFsKfxVpu', 'nchar.贵州省建国县海港陆街z座 539355', 1630000000041) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000001, -743807783, -2365949684422468696, -5832, -55, 152697699639.433990, -847.354407, 1, 'binary.TEyMfkklGoyQyuhoMfGa', 'nchar.江西省秀华市黄浦刘路o座 900428', 1630000000041) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630600000000, 1164985515, -1528321163891719683, -5017, 59, -4685058576109.339844, 573635.987915, 0, 'binary.YRSxugbLdtCzYDQuPHiO', 'nchar.青海省刚市吉区成都路D座 183949', 1630000000040) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630615000000, 1552612066, 2109390087439052468, -31070, -121, 8636.758774, 54571.528229, 0, 'binary.xlsMBheXDBRIbHzbgzdk', 'nchar.河北省兴城市东城西宁路M座 385581', 1630000000041) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, 686960603, -6424166539989529553, 24333, 44, 8214.897773, -47431121552021.000000, 0, 'binary.RWvwTVykxTTNOYllQrFG', 'nchar.湖南省鹏县清河徐路D座 553544', 1630000000041) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630614999999, 68548265, 4587156226014614794, 16331, 57, 154895.265334, -2787.111072, 1, 'binary.ixnKFyhNgVkQywiDtpSq', 'nchar.广东省海燕市白云兴城路u座 101958', 1630000000041) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, 2266587, 2581645298895273135, 22522, 42, 3276944.200912, 911196.685350, 1, 'binary.rAZwQmSTbVawENHakqoC', 'nchar.北京市玉英市西夏台北路Z座 806985', 1630000000041) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000001, -982806837, -8813143395330268157, -9811, -65, 4240824286.641320, -949974.279935, 1, 'binary.IhdGdIOduEtBfgveDAUU', 'nchar.广西壮族自治区佳县怀柔西安街o座 181079', 1630000000042) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000001, -680593150, -6347717442100280854, -12839, -105, -3011259806.223000, -99641241686.436996, 1, 'binary.BZPuOZNfEoXwZqhyVXdb', 'nchar.内蒙古自治区南宁县丰都支路Y座 726362', 1630000000042) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630615000000, -417426465, -3607718287738436914, 28714, 52, -972051082.524374, -9817388374.836500, 0, 'binary.cWDUQzDHybUekMyDXMXD', 'nchar.四川省帅市黄浦荆门路A座 476400', 1630000000041) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630630000000, 1094499153, -8105793501718430866, 3482, -67, -190.752084, -5267142.925286, 0, 'binary.uxspDdfsvojZAgyJTRsp', 'nchar.贵州省香港县徐汇合山街F座 623037', 1630000000042) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, -1816628643, -2390834298866111036, -25491, 93, 421896.632317, 21603.217576, 0, 'binary.hrCIGqyoKXyOENntzjcy', 'nchar.湖南省广州市怀柔六安街a座 810265', 1630000000042) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630629999999, 237543427, 5776314153706817860, 8994, 66, 3092.121681, 4306303.541656, 1, 'binary.utVrWGltqteXbNHvQIuO', 'nchar.香港特别行政区建军市长寿刘街N座 294264', 1630000000042) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, 726605466, 7765148407017524476, 9380, 66, -725.184771, 4.785127, 1, 'binary.eCvUodgIQEPGQAlVRXHW', 'nchar.湖南省石家庄市海港林路K座 736274', 1630000000042) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000001, -2034983874, -1663495204537070761, -3507, -95, -26436104.684611, 3096688941.682010, 1, 'binary.buRNGkuUJrPnERgeimCX', 'nchar.湖南省海口市门头沟贵阳街c座 812797', 1630000000043) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000001, -1349842048, -6086664054622894661, -26154, -20, 9.257350, 73467.254310, 1, 'binary.VLQrxUICLTloHhOcsChH', 'nchar.福建省超市崇文通辽路p座 610024', 1630000000043) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630630000000, -1077532579, -4438737277090132487, -4021, -89, -1603199944.809290, 972619729.980217, 0, 'binary.bmvtStTuvDxOdnRZezwW', 'nchar.海南省成县闵行康街q座 877974', 1630000000042) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 667040085, 5063347356651668746, -6033, 123, 372331463.295536, -1199.428365, 0, 'binary.yILHXSyMobxUzhCNgPvB', 'nchar.陕西省冬梅县高港谢路R座 540642', 1630000000043) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 1643497791, -7930526570990406176, 31172, 28, -53.309129, -52475027161.967003, 0, 'binary.lznIQgSqYtbfWrGxFBvU', 'nchar.山西省南昌市翔安李街Z座 864233', 1630000000043) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630644999999, 1757612759, 453614548545492325, 5562, 26, -42197958317985.296875, -395.764533, 1, 'binary.gZRMNooJsuQCqUkKgObf', 'nchar.浙江省呼和浩特县闵行张路Y座 696051', 1630000000043) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 716946034, 8100574991162286831, 8901, 92, 390.851634, -9995.448678, 1, 'binary.WChvjMXwNeLmicQzcpDK', 'nchar.湖南省丽华市江北张家港路O座 383877', 1630000000043) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000001, -1691922856, -2342118046897302678, -23270, -66, -172559.465035, 3039868.728966, 1, 'binary.RimPMjXlEtpLazOlwnuv', 'nchar.安徽省玉华市合川黄街F座 509787', 1630000000044) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000001, -1339923814, -5560716090143693393, -6671, -10, -84330915414.755005, -2113273814915.989990, 1, 'binary.YGgXnRFnSbGpfKSYyKgV', 'nchar.内蒙古自治区荣县永川朱路u座 853226', 1630000000044) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630645000000, 1122107930, -8861294204430315415, -1044, -23, -3031.346285, 18.218107, 0, 'binary.lUwNsZlGQGhpLpDYqvtB', 'nchar.广东省兴安盟市合川拉萨路K座 345819', 1630000000043) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630660000000, 815803254, -3671302785504904803, -3291, -47, 736.989253, -48462175883.852402, 0, 'binary.PcSjPekpFeCmxcWwgQcF', 'nchar.青海省深圳市黄浦淮安路P座 492194', 1630000000044) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, 484949428, -5416390991488308737, -14887, 67, 2398644.783561, 8171275.544491, 0, 'binary.BizAmDZxfvTsgphmgDgO', 'nchar.新疆维吾尔自治区齐齐哈尔市秀英杜路h座 144209', 1630000000044) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630659999999, 1440457969, 8606030504343208767, 24314, 96, 92496.379572, 0.813941, 1, 'binary.xAmVpiYdockKtVmdHSIc', 'nchar.河南省平市海港龚路P座 644038', 1630000000044) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, 1069631125, 7783641181026641655, 29140, 58, 69214.199993, 158732124515.929993, 1, 'binary.SBJQcfJBwhXLUJwatImS', 'nchar.广东省强市怀柔哈尔滨街L座 103945', 1630000000044) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000001, -899220871, -9196200436704202533, -27771, -90, -73.878944, -8912.628710, 1, 'binary.VQcAzgArqSQQZnbSiMNU', 'nchar.天津市沈阳市东丽西安街g座 810520', 1630000000045) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000001, -1450060823, -615484698684325866, -16629, -4, -8.793729, -23575246237039.601562, 1, 'binary.AESWGdfihvaPnTfmDemj', 'nchar.宁夏回族自治区帆县江北石家庄街M座 983792', 1630000000045) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630660000000, 1278398866, 5363278013106925425, 29093, -83, -70828.447306, -526391.541525, 0, 'binary.iFAFqKguHdrdQhGoCpuS', 'nchar.江苏省刚市江北东莞路b座 826956', 1630000000044) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630675000000, -1081748955, 720719842133998595, -22633, 24, -73662437394.636200, -2235207.112560, 0, 'binary.CdInxRzeRZrFBbTOVcoH', 'nchar.台湾省六盘水县兴山石家庄街l座 541218', 1630000000045) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, -1665303728, -6208374964158196325, -17938, -26, -12.743038, -6410996304252.719727, 0, 'binary.ESErZzoVFrwLvZLjvszF', 'nchar.宁夏回族自治区兵市南长沈阳路A座 196143', 1630000000045) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630674999999, 1830689374, 6847266081436213038, 27195, 56, -98.825225, -11.841657, 1, 'binary.bKBmkbDmVObIaINgcUkt', 'nchar.宁夏回族自治区六安县东丽黎路Z座 360011', 1630000000045) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, 905604744, 5992591234700515548, 30228, 29, -771431291465.423950, -75930131567683.500000, 1, 'binary.uJqjLxypwZAqCJHRMkKo', 'nchar.吉林省艳县城北贾街Y座 511120', 1630000000045) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000001, -1874157012, -4583327710928801759, -27522, -76, 53914287540.938499, -9385422.995425, 1, 'binary.wycnNJGrIpmtWWvFxuJY', 'nchar.宁夏回族自治区云市孝南江路A座 778572', 1630000000046) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000001, -1498560711, -9199781197179273356, -25882, -75, 4614968554.698110, -96421428706.153503, 1, 'binary.OOYjhYdCGZQViNShXiSg', 'nchar.吉林省惠州市江北关岭街W座 967516', 1630000000046) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630675000000, -596563585, -8961615493016741888, -10813, 102, 75.123805, 6555481001.569020, 0, 'binary.DVXQBGshLAcqMWMMeiDT', 'nchar.山西省桂兰市涪城佛山路C座 407229', 1630000000045) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630690000000, 81214187, -3182813081122938906, 4960, -50, 518940275802.109985, -7999305661.759080, 0, 'binary.fkEdEbNuIkBTKcrHnEjx', 'nchar.新疆维吾尔自治区佛山市怀柔呼和浩特街P座 745345', 1630000000046) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, -1132347210, -6643466065281753528, -27772, -3, 8162664758739.139648, -13662048540.698799, 0, 'binary.KmKfzpIUppEqzuPvExem', 'nchar.四川省莹市牧野哈尔滨路j座 521095', 1630000000046) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630689999999, 1019010549, 6975314636031351786, 28464, 43, -465.888140, 8910415.572030, 1, 'binary.rBiBGLvfQWmnGDquOTbn', 'nchar.广东省雷县南溪宋路F座 681268', 1630000000046) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, 378275737, 876425613508142928, 15367, 120, -49.379488, 7567.687919, 1, 'binary.BTwfiAxrnagWWuCCCSZQ', 'nchar.北京市南昌市南长香港街i座 314362', 1630000000046) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000001, -533673302, -4156210237594471836, -17141, -10, -931105.914523, 799382106.862106, 1, 'binary.NqTaorSXMNHGbcIiftLN', 'nchar.香港特别行政区淮安市牧野石家庄街R座 897882', 1630000000047) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000001, -1935282932, -2743254909741386818, -6325, -67, 25114142.334721, 78.857657, 1, 'binary.hLaIbMlaSZIaojoxNNAG', 'nchar.湖南省邯郸县新城重庆路W座 451361', 1630000000047) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630690000000, 693093205, 1731593666471158488, 32616, -34, 944511.517342, -3.585256, 0, 'binary.JGrrPFPHkdvblDPQLBzo', 'nchar.安徽省艳市龙潭陈街I座 824093', 1630000000046) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 1768911737, 5728476461209604096, -29190, 66, -6177089795.760590, -21430.400483, 0, 'binary.KANmXakVbwmKTQEcoIkv', 'nchar.吉林省帆市静安福州路e座 331035', 1630000000047) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, -248617365, -4341045528963124184, 31755, -98, 124670342.644870, -654637491670.374023, 0, 'binary.IzfBodxsdDxJktqQeDDU', 'nchar.浙江省秀芳县黄浦郑州路L座 115314', 1630000000047) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630704999999, 27001027, 339256900217870846, 11700, 1, -1455323856048.100098, 447841.805243, 1, 'binary.tMfHAqfBuEMKIckNFKLY', 'nchar.内蒙古自治区亮县清城陈街k座 990698', 1630000000047) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 1581778818, 8464425008394999321, 20061, 5, -5247104654.611190, -568125.266376, 1, 'binary.JAXQCWWoDrpJuxVSlCNu', 'nchar.宁夏回族自治区萍市安次六安街H座 295531', 1630000000047) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000001, -732869228, -2906753353691356694, -5441, -3, -109283.289234, 9159375.165842, 1, 'binary.VYdDCHOMFAhsntioQInn', 'nchar.澳门特别行政区璐县西夏海门街A座 607065', 1630000000048) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000001, -18474510, -5999800564410930013, -6311, -2, -398057879.913052, 8761.567919, 1, 'binary.fIJqDAvfpidoaJSJkpnq', 'nchar.安徽省宁德市浔阳长春街U座 799007', 1630000000048) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630705000000, 850296579, 8875126379280838717, 10524, 108, -2975287.612558, 0.445600, 0, 'binary.eoLmnRFaXvklftfvhFCU', 'nchar.四川省亮县东城南京路E座 405800', 1630000000047) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630720000000, -239418734, -7503726504441061411, -31831, 84, 69021572289.137802, -86475940923.449203, 0, 'binary.ApWGXJYAZWgkReWPsvDb', 'nchar.湖北省磊市璧山武汉路O座 323410', 1630000000048) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, -943706680, -214023538774872985, -12169, 127, 948155.120585, -65149.125324, 0, 'binary.sujGooIQzKvLdzqRekyb', 'nchar.河南省丽娟县华龙哈尔滨街l座 264772', 1630000000048) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630719999999, 1251825975, 5053354830656579627, 14919, 60, -7531018879634.370117, 310964263643.562988, 1, 'binary.RfrhrTWzkzBvUvupCUaE', 'nchar.海南省阳市平山马鞍山路I座 852814', 1630000000048) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, 1115544686, 4049894819935008122, 30814, 12, -75.765668, 1187804567.210350, 1, 'binary.hYtJJtMEzEnmlYixUssL', 'nchar.河南省哈尔滨县魏都杭州路z座 972095', 1630000000048) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000001, -1724765969, -736233583823352610, -6737, -57, 893.536927, -7421.472515, 1, 'binary.RsUbHbctLoVkgmtdCxjM', 'nchar.黑龙江省马鞍山市山亭孙街b座 213806', 1630000000049) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000001, -1986547513, -6324903826238608678, -14843, -46, 309302.506851, -614273917609.833008, 1, 'binary.vnLJpYckYTAcDOpDNuWh', 'nchar.天津市亮市南溪陈路q座 364119', 1630000000049) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630720000000, -1284939264, -9095077298976089415, -19239, -92, 31.280584, 7379.657977, 0, 'binary.knKChKaZsyhUfkgmgzph', 'nchar.贵州省兴安盟市淄川郭路G座 242776', 1630000000048) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630735000000, -762937350, 8729420312334257343, 27701, -108, -6.651477, -29.713736, 0, 'binary.fpjgqsvwAKHintMJaYER', 'nchar.黑龙江省淑华市花溪宁路W座 152088', 1630000000049) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, 1144380216, 5276665732684544642, -11205, -89, 8215.254115, -99811752553.775497, 0, 'binary.FflDToYYzhVTnyRGzKGA', 'nchar.江西省南京县金平孙街c座 254949', 1630000000049) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630734999999, 1075377727, 9020113518015848371, 11084, 28, 307417.308042, -87.561190, 1, 'binary.kEOyxbKVdHlFqhZZzYwq', 'nchar.青海省佛山市安次深圳街Z座 504751', 1630000000049) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, 582692563, 4638155561099114479, 23761, 48, -29824236570673.398438, -10200.908461, 1, 'binary.XziirLIHCuFKtpiXVrTy', 'nchar.广东省帅市西峰张街h座 174088', 1630000000049) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000001, -1716643044, -8326169264977759340, -4200, -30, -2.318414, -726141333.615880, 1, 'binary.FZBESQUiuERkOgEMEfQR', 'nchar.广西壮族自治区英市海港汪路W座 657875', 1630000000050) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000001, -686568708, -4371717027180867061, -6734, -88, -578830467.970145, 9456.996341, 1, 'binary.ybxyxSLSKWvhXIhIuTZo', 'nchar.西藏自治区鹏县华龙邯郸路K座 280464', 1630000000050) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630735000000, -463229385, 8322975415402188951, 7658, 81, 48067565948.430801, 1684.497042, 0, 'binary.VCPIPiYLkNeWVJRVYdAa', 'nchar.江苏省伟县新城杭州路U座 135067', 1630000000049) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 229305482, -8755922362006582494, 15607, 16, 433186324.150179, -9760091.526296, 0, 'binary.vzKjmFtaWUQJWUmLYsoF', 'nchar.广西壮族自治区潮州县璧山深圳街u座 513872', 1630000000050) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 887769742, 8828265476094538774, 15810, 6, -52009423024.988998, 457109451.488690, 0, 'binary.lFOSpWYYjCkwLqOGSclj', 'nchar.广西壮族自治区伟市大兴李街A座 575739', 1630000000050) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630749999999, 806743984, 7134986215003200921, 17680, 41, 80628998.429324, -398852949.893027, 1, 'binary.KNgNzEsWMMOErSdhZARB', 'nchar.台湾省大冶县西夏辛集路B座 294228', 1630000000050) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 1218616066, 4338651506702922568, 20337, 121, 607218.969786, 29540695532115.398438, 1, 'binary.hXxPnZwhSlMFInaNEPGO', 'nchar.上海市桂兰市金平刘路x座 286919', 1630000000050) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000001, -592467708, -3107486233936469903, -26640, -37, -77805664.682649, -1730.324950, 1, 'binary.fpSPyrjwsVtXzEHRGCaS', 'nchar.福建省沈阳市东城张路k座 869492', 1630000000051) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000001, -547550314, -5200749799296792327, -27549, -59, 2034147.803264, 96891027.798801, 1, 'binary.cyVZnAOokBmkMGIbacqs', 'nchar.江西省宇市蓟州唐路K座 406871', 1630000000051) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630750000000, 756019639, -653863967829096798, -30967, 59, 68379.967777, -1154096688678.510010, 0, 'binary.fHCaNBTuAEHvVNpuNcxn', 'nchar.安徽省宁县永川东莞路X座 798708', 1630000000050) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630765000000, -1406057540, 1446134498171766899, 30680, 68, -9307952080739.890625, -1153314466.810470, 0, 'binary.aDuEaSxjYaNCTgFqMHlC', 'nchar.西藏自治区红县静安黎路I座 596984', 1630000000051) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, 133613101, -1336954935529805727, 585, 116, 30.398745, -613552078.887368, 0, 'binary.NgMivPQObBgUbKnAwWEj', 'nchar.湖北省金凤县门头沟刘街f座 712446', 1630000000051) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630764999999, 656042024, 1646194287674616827, 26191, 10, 6678822.415847, -95361406615.342804, 1, 'binary.LZeGatPnXBHGNQggCUgW', 'nchar.澳门特别行政区西安市清城汤路p座 810444', 1630000000051) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, 1983919127, 7730061788745380233, 12586, 75, 10716267.549970, 7.199934, 1, 'binary.rnitdHGQBawBYXtOfvDJ', 'nchar.北京市玉兰市清河海口街u座 800278', 1630000000051) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000001, -1382937017, -109262277060062044, -24986, -53, -431535423232.208008, 34209788.409645, 1, 'binary.WSTgldQbIqYUaOeVUhSk', 'nchar.云南省秀英市涪城张路H座 199974', 1630000000052) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000001, -1301473352, -6756208355269860322, -17139, -45, -689118.755923, -3373577.841810, 1, 'binary.rcYvJxobOrraprgqcGvR', 'nchar.台湾省洋市花溪宜都路I座 962472', 1630000000052) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630765000000, -691470080, 1659254673769147544, -23790, -47, -429810768489.799988, 93436.707719, 0, 'binary.SxTTHFObymUpnuAfgYSQ', 'nchar.贵州省海门县西夏海门街a座 133401', 1630000000051) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 921298258, 8536844726148128680, 3660, 26, -34185874.434563, -26620.589344, 0, 'binary.xevnfxQByaOjIEnqHaqe', 'nchar.重庆市华县山亭赵路b座 801493', 1630000000052) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 1233028572, -8019335724423257023, 31663, -58, 14.142909, -50404.483362, 0, 'binary.MgKDUmZIKAlVhLpazpsJ', 'nchar.青海省永安市清浦彭路P座 393906', 1630000000052) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630779999999, 1876585459, 2853158241768185532, 24232, 50, -38916414122.860901, 6486812.485190, 1, 'binary.fafedlhFPmPAyJJGuMuZ', 'nchar.青海省潜江市秀英沈阳路X座 835387', 1630000000052) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 1194165890, 7697386066201959752, 12368, 15, -7467278746696.700195, -979006.127435, 1, 'binary.sEHpIDvtRWaxcUbqPHAn', 'nchar.新疆维吾尔自治区大冶县高明项街d座 680148', 1630000000052) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000001, -1899593535, -1379716049261187467, -26105, -97, -17.445226, -149.378109, 1, 'binary.sHyQLVjnSTQSXpcWxHjq', 'nchar.安徽省杭州市城东郑州街M座 499265', 1630000000053) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000001, -1856641529, -4102589906454836588, -24873, -61, 97710303188.778000, 9646.234878, 1, 'binary.fKSswvhHhMTcaIjfaUUk', 'nchar.安徽省建县牧野杨街U座 584763', 1630000000053) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630780000000, 542554488, -3045723568122762779, 18580, 33, 95242718247053.500000, -50170311.264161, 0, 'binary.PUzrCqSyZiemQmkCCdGz', 'nchar.重庆市玉兰县永川汪路N座 270623', 1630000000052) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630795000000, 1864777486, 9048501077169003317, 32497, 90, -21112693476.624199, 960.488977, 0, 'binary.EEwLsBOCVnXbQbGiHwjl', 'nchar.河北省天津市魏都徐街V座 183214', 1630000000053) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, -1607594419, 3244019062872113101, -3112, -38, 4371695.605750, -76333.232422, 0, 'binary.FoXqOarMdnRuIkzdWEfn', 'nchar.安徽省石家庄市上街李街j座 287811', 1630000000053) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630794999999, 173077592, 5900121444981917372, 31259, 47, -9.692719, -36537321184.241699, 1, 'binary.hAZkGTgZudTvNdAfGCrG', 'nchar.黑龙江省林县长寿长春街H座 568159', 1630000000053) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, 1357591704, 8166773293007138161, 11502, 15, 9824768605988.179688, 899.269307, 1, 'binary.ORKiLvVsgwUaMmmVzXYK', 'nchar.浙江省西安县海港关岭路w座 192982', 1630000000053) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000001, -1842197906, -1075636322848135854, -11343, -9, 9085057779.572281, -9427483311859.769531, 1, 'binary.urBDWyTekFLJVKfewubE', 'nchar.江西省静县高港辽阳街W座 980270', 1630000000054) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000001, -1534128171, -4579657021740144805, -28105, -23, -265482542779.970001, 74780.975545, 1, 'binary.sEpkrqnSpFfYKIgeSBvK', 'nchar.澳门特别行政区西宁市龙潭六安路X座 127241', 1630000000054) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630795000000, -760013920, -8462295357527459170, -31765, 11, -433.960695, 264.431691, 0, 'binary.DroLRaBIftSwkoEbBZUl', 'nchar.山东省丽丽县黄浦谢路f座 644732', 1630000000053) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630810000000, -1675325589, -360996985377003606, -2347, 24, -470.741524, -887230390.331345, 0, 'binary.udlpjXsecEvxMZCZyvrV', 'nchar.江苏省银川市山亭太原路J座 723211', 1630000000054) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 1632642973, 7850590523192382300, 13631, 45, -47687.239109, -81863875.566057, 0, 'binary.jmfRTYATfWhgLdSwABgZ', 'nchar.新疆维吾尔自治区长春市龙潭刘路m座 413049', 1630000000054) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630809999999, 1959334832, 4568407287596246175, 27961, 84, -665950847.396483, -29596243749715.199219, 1, 'binary.JOlrzeKxQTwVbozdztiV', 'nchar.山东省广州县高明张街C座 757018', 1630000000054) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 1375371462, 4011481089622391741, 11886, 126, 4.397144, 4202513.398747, 1, 'binary.EblbpxwjuFqWmvABRtpz', 'nchar.香港特别行政区坤县南长王路W座 576754', 1630000000054) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000001, -1132521267, -4710706752937901089, -27234, -102, -277590425.724995, 292916.997240, 1, 'binary.DaNIAqljzrMdiAdtjNrB', 'nchar.江苏省畅市梁平秦路K座 961783', 1630000000055) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000001, -394127423, -4195198128268963203, -5174, -112, -37786616.462910, -750832.942933, 1, 'binary.rDUuOHQMIpSTjzaBeDKl', 'nchar.青海省亮市城东惠州路R座 997920', 1630000000055) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630810000000, 415436252, 7840392447567623067, 18496, -2, -20101885975.533798, 789.450282, 0, 'binary.CIlEgPRwfDUmqTEAjBrU', 'nchar.山西省桂花市高港张家港路B座 439810', 1630000000054) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 1382842369, -3387166153829557613, 5113, 67, -14533655399834.400391, -47045088950.955902, 0, 'binary.fdwVbTqALVxEKqxPmklJ', 'nchar.香港特别行政区桂芝县沙市桂路K座 439607', 1630000000055) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 122830188, 5944964533814104404, -4275, -2, 503740.982450, -2713577716.766000, 0, 'binary.KnHTmENdKEWNLWdfvFxb', 'nchar.内蒙古自治区婷婷市大兴乔街a座 468770', 1630000000055) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630824999999, 609279903, 9143142539302518131, 18578, 58, 6.776714, 7355.869852, 1, 'binary.PsdCUwiiFfTfBdYWgMIf', 'nchar.广东省巢湖市高坪关岭街z座 165945', 1630000000055) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, 1229687800, 6229049320529684281, 13288, 61, 1136575.519568, -474.263794, 1, 'binary.UtlqumZjObldYJWTCHVF', 'nchar.上海市玉梅市城东陈街K座 187484', 1630000000055) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000001, -902381702, -6581003712798510587, -16241, -42, -136567151.288395, -81351450.825485, 1, 'binary.vZnItDAvBlQXFtrMrdAk', 'nchar.广东省合肥市闵行南昌路a座 421030', 1630000000056) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000001, -1633588040, -9147247193490618424, -16771, -115, -2.864546, -4.968001, 1, 'binary.OChQYNfkTrcSaZTJGfhO', 'nchar.云南省淑兰县牧野农街t座 752385', 1630000000056) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630825000000, -997989279, 7637562832426174518, -5, 72, 1838649266.575190, -32247151699394.300781, 0, 'binary.fLcdsxEDyCdsbzARmlzG', 'nchar.吉林省娟市秀英潘街u座 599699', 1630000000055) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630840000000, -1691466505, 3835361758424457104, -21465, 99, 17.423367, -927580467091.141968, 0, 'binary.ExYLmKXMaFjayqkBEWqC', 'nchar.浙江省乌鲁木齐市东丽康路s座 780818', 1630000000056) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, 990486101, -6476848417158885996, -6947, 34, -91088473.667455, 443265343.199751, 0, 'binary.dyTxrTzimrnLgwtLehEp', 'nchar.海南省帅市金平兴城街F座 303386', 1630000000056) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630839999999, 1355859529, 2260526383089822706, 30660, 63, -672086975563.209961, 9219879.518002, 1, 'binary.EEjtfcofhSPxOuFgMlug', 'nchar.山西省凤英市长寿胡路d座 722345', 1630000000056) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, 720225284, 8056479222648510406, 7870, 107, 3.205667, -15331943645.198200, 1, 'binary.qBFljuOsopeYwpczltun', 'nchar.海南省杭州市锡山杭州街Y座 563787', 1630000000056) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000001, -1666639404, -1502779151523040369, -32737, -27, -720231064.559860, 9483852418979.500000, 1, 'binary.sDGclxLTrGjIYGBWUzox', 'nchar.江苏省颖县华龙奚路M座 347083', 1630000000057) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000001, -513166008, -1798257110690905490, -17212, -65, -6875665232.854830, 1.501719, 1, 'binary.JioYSyYNMAmHQGKIwFpt', 'nchar.澳门特别行政区畅县永川淮安街i座 315960', 1630000000057) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630840000000, -784030741, -771760196117941650, 5269, -23, 488442494858.158997, 142.977034, 0, 'binary.RyfovtjrYYDbDQEVcHoA', 'nchar.福建省帅市兴山合山街z座 360892', 1630000000056) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630855000000, -759466985, 3260110813666913849, -21345, -118, 6001735483.766000, 811956391.434487, 0, 'binary.FgtcgDCGdWGhileXYeVC', 'nchar.辽宁省敏市城东潜江街t座 539658', 1630000000057) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, 1791483134, 1877345879709438108, 7122, 108, 649645.697610, -3.176571, 0, 'binary.VOAUXbpzoEXTfEombqFu', 'nchar.湖北省建国县双滦昆明街k座 555641', 1630000000057) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630854999999, 819862528, 8086075097163190658, 28478, 59, -8892423630966.660156, 927321477.432369, 1, 'binary.gEIBgRUfuMarfhivmcmN', 'nchar.海南省海门市沈河季街o座 938191', 1630000000057) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, 575972371, 8165684898549408541, 26532, 30, 2158.716179, 6332790959904.230469, 1, 'binary.RzgEyNIsdJwDbkhjuZhn', 'nchar.广西壮族自治区哈尔滨市双滦薛路M座 750545', 1630000000057) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000001, -568694685, -6156824576786534987, -5004, -80, -8796651.346065, 4.370643, 1, 'binary.vDKHYEXPpBzKIFbWrBHH', 'nchar.四川省春梅县永川董路R座 716271', 1630000000058) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000001, -202484706, -7242157303332761279, -5895, -39, 1369656077.152570, 45024.146483, 1, 'binary.UTIcQraFLhrzeKZQNGad', 'nchar.陕西省大冶市大东陈路V座 230063', 1630000000058) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630855000000, 968718547, -7083502452755942963, 30186, 34, -808.116870, -355374.445932, 0, 'binary.FIxenkYBuIDmoaBpvKJr', 'nchar.内蒙古自治区想市闵行合肥街m座 213729', 1630000000057) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630870000000, 259545745, 6793904417678601057, 29752, -91, 35083182.940608, 4579746.564176, 0, 'binary.wmnksDymOwGQownBXGRS', 'nchar.湖南省上海市淄川陈路f座 699980', 1630000000058) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, -798452451, -1421146164509922388, -30643, -44, -532.819299, 864903894.997066, 0, 'binary.FofIkZDYExLKhITHZjpy', 'nchar.北京市静县长寿刘路V座 929762', 1630000000058) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630869999999, 810375591, 6862940885997459592, 12888, 67, 1.743840, 42697.428148, 1, 'binary.OrGwClCMpfhfshmaeYAS', 'nchar.安徽省坤市长寿许街s座 864780', 1630000000058) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, 2141142518, 330171567484187630, 5038, 87, 73252.885684, 88624.734440, 1, 'binary.cdSnMdwrroFKtbPAJSbg', 'nchar.安徽省沈阳市清浦汕尾路C座 749970', 1630000000058) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000001, -702799898, -322816941471556942, -1394, -123, 9456805.646888, 2105.641243, 1, 'binary.rTaTBdXbBszZjTTwracU', 'nchar.四川省玉市海港大冶路h座 545972', 1630000000059) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000001, -706124307, -5958027025637340951, -30853, -27, -7966.705252, 2.270573, 1, 'binary.eOyuShSfdOAmaSNhUZUv', 'nchar.河南省长沙县萧山毛路H座 196587', 1630000000059) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630870000000, -486193469, 7258930411934958638, 23283, -84, -6.193098, 11.792424, 0, 'binary.KGiDWTZdgHETHLJePGIT', 'nchar.陕西省丽市白云兰州路x座 161175', 1630000000058) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630885000000, -282533404, 7684010141772871588, -23850, 17, -2997190086014.000000, 931579019.615816, 0, 'binary.kZsMznDmRkjSoUQCWmtw', 'nchar.宁夏回族自治区汕尾市清城北京街c座 432095', 1630000000059) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 1596401672, -5390915537011895583, -19578, 118, -9665.572275, -7154265.495520, 0, 'binary.JkHelKKoDnnJzPHiEROB', 'nchar.西藏自治区晶市梁平梧州路b座 369190', 1630000000059) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630884999999, 1840788362, 568759292490183791, 18205, 21, 16.931152, 17691.251149, 1, 'binary.EDicJkxTXgqUtLXKDjKy', 'nchar.新疆维吾尔自治区勇县蓟州沈阳路U座 172773', 1630000000059) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 365787442, 1666872794496374179, 13837, 37, 193390765070.401001, 546707.447097, 1, 'binary.lmEGkvWshsqnAktNzJbB', 'nchar.重庆市桂花县南湖巢湖街U座 689551', 1630000000059) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000001, -745755820, -8898782181578548041, -7682, -15, -443.639102, -255152262.128928, 1, 'binary.MWmdVCjrANIfsdHAAVmb', 'nchar.河北省淑珍县长寿谢街k座 131370', 1630000000060) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000001, -2029500216, -7380552851400848492, -3516, -8, 8.915821, -51.793696, 1, 'binary.vQnridOAWDVWkJYYdTtF', 'nchar.云南省六安县海陵宁德街t座 241913', 1630000000060) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630885000000, 1648193924, -5763807398275594930, 27798, -65, -3.657749, 9559111883869.490234, 0, 'binary.VLogQYbaHPpPpmgyuzos', 'nchar.台湾省志强县和平刘路a座 827192', 1630000000059) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630900000000, -997698344, -4360698110328218759, 31934, -79, 8770449833.941080, -71856107060.251297, 0, 'binary.trhWcwdEKeoGUcQdImgV', 'nchar.天津市西安市牧野天津街q座 880750', 1630000000060) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, -492616894, -7002758781471003065, 28164, -11, 4740658.868259, -5320539.530775, 0, 'binary.TzdNtGFTXZWMADJaayqx', 'nchar.西藏自治区西宁市秀英杨街A座 353774', 1630000000060) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630899999999, 1825013146, 1744104394556381743, 31778, 117, 908811698547.689941, 42.982948, 1, 'binary.FtOgWPVAvygSIYsTWnHX', 'nchar.澳门特别行政区梧州县海港六安路D座 341014', 1630000000060) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, 2076832076, 3306826763175476238, 29009, 65, 654125.128750, 23.786689, 1, 'binary.yRAQxkeArOAnSkFHvhjQ', 'nchar.青海省南昌县清河澳门街f座 357735', 1630000000060) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000001, -364752002, -5092648072975157908, -23087, -39, 9.504646, 93358807.508628, 1, 'binary.sFAvwLkiYcepPputybEK', 'nchar.四川省文县萧山惠州街i座 903240', 1630000000061) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000001, -1784446385, -2802855779017240513, -5552, -54, -860449735702.189941, -703215980204.104980, 1, 'binary.RtSNrxogNuhOoRhEPrMJ', 'nchar.青海省静市孝南李路H座 399925', 1630000000061) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630900000000, 1445349008, 992848000519415190, -17518, 28, 4014000610.625470, 592.602654, 0, 'binary.IFSsgUZILZuPRwXTuFEw', 'nchar.海南省沈阳市六枝特长春路n座 155408', 1630000000060) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630915000000, -134082975, 4026898983947505460, 20740, -64, 24146076.871632, -207518516.166626, 0, 'binary.irWEJqxWnIlrKDExOAvg', 'nchar.天津市太原县吉区拉萨路D座 176064', 1630000000061) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, 1299050318, 3065792171140876784, -12329, -3, 74005.658734, -418157994.246836, 0, 'binary.sqAYHZcIIihZMMauybjG', 'nchar.内蒙古自治区桂花市城北阜新街y座 784832', 1630000000061) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630914999999, 711052785, 2718796412708678637, 32243, 69, 4069895314.170900, 38406.385424, 1, 'binary.UgajoEPkcVXcWkUCGwkP', 'nchar.安徽省桂花县房山傅路O座 240748', 1630000000061) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, 2140137646, 1082661109056257532, 23250, 103, -44495004919076.296875, -7284.922434, 1, 'binary.ryfAsYaeEOkhesQMRitO', 'nchar.台湾省玉华县徐汇杨街C座 637917', 1630000000061) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000001, -1403755208, -3324715225642465377, -15193, -53, -3142.782618, 1794554.528658, 1, 'binary.aOAYKsmWzKDqjOpbOYrJ', 'nchar.河北省倩市清河黄路t座 205485', 1630000000062) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000001, -361144599, -8536611280568601202, -8124, -93, -13513340.923311, -20153.231767, 1, 'binary.XgncJmqFHazSpRWcMVeT', 'nchar.澳门特别行政区帅县清城深圳街j座 501390', 1630000000062) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630915000000, 929886382, 6830786048646808920, -26587, -84, 201249126.540132, -33020.844039, 0, 'binary.kUqBGPosKRCkaijDufGt', 'nchar.陕西省澳门市蓟州祁路C座 345922', 1630000000061) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630930000000, 1849726714, -3222927126454669927, 21701, 35, -764955573301.579956, -677588544809.604004, 0, 'binary.mXWIcnTunwqEKbKnJsRO', 'nchar.青海省云县南湖张路t座 577114', 1630000000062) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, 1948380077, 1829472867218108411, -2841, 62, -585408.996552, -8.398878, 0, 'binary.fZDJrrryXodeitdEZiDN', 'nchar.天津市辽阳市平山深圳街O座 892463', 1630000000062) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630929999999, 215833483, 6741487167704692542, 24982, 99, 7.720103, -4484984.118093, 1, 'binary.PUhvTciOFJERTXpNFDMf', 'nchar.海南省雪梅县萧山夏路s座 884246', 1630000000062) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, 283066224, 1560470331627072050, 2152, 14, 23735906455.603001, 977.642371, 1, 'binary.LQZzMIDAEVyjgirFHORu', 'nchar.河南省秀梅县房山黄街D座 578931', 1630000000062) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000001, -1383444081, -404967010555838935, -20478, -67, -29413948653.431301, 2289249875341.339844, 1, 'binary.gGaUfmagkabfKeDUnCCo', 'nchar.江苏省冬梅县六枝特张路g座 635526', 1630000000063) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000001, -821980909, -1357071288288006358, -23706, -57, 6152.175178, -26066560.757762, 1, 'binary.kmxekdKOIgvNnCSEZlwN', 'nchar.上海市宜都县双滦王路P座 262541', 1630000000063) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630930000000, 713911469, -6504806039884313820, -32386, 118, 431693877138.190002, 7127294.830732, 0, 'binary.FArXPzfLVQjsNuWiHLrQ', 'nchar.宁夏回族自治区秀华县长寿钟路t座 443656', 1630000000062) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 1000635145, -6949456453535507535, -32142, 99, -7613228535871.339844, 5.367833, 0, 'binary.yafjOZqJhdMZFCxrvkjO', 'nchar.天津市志强县上街陈路g座 745449', 1630000000063) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 2094975637, -590793247049504295, 92, 70, -846255.238019, 37.423177, 0, 'binary.IBHCEHNtLvmkpSCmVjQg', 'nchar.台湾省桂芝县华龙合肥路b座 733142', 1630000000063) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630944999999, 792673522, 2133996305758691365, 11545, 90, 8043187112236.719727, -328687222.743009, 1, 'binary.ZJIzomZBnRwLgnTtcylc', 'nchar.安徽省波市清河卢路e座 142743', 1630000000063) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, 1815853908, 1589355425036470987, 22184, 52, 8.753561, -0.772172, 1, 'binary.eqnhiDmrJHLxSTqywNzp', 'nchar.广东省合山市朝阳佛山街y座 254118', 1630000000063) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000001, -1908151355, -8449086371025617726, -31176, -107, -226488394.744318, 522688593.135097, 1, 'binary.mOQahkNsfbxPPiCysDOX', 'nchar.新疆维吾尔自治区潜江县魏都昆明路q座 187031', 1630000000064) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000001, -177205853, -5927588760169982955, -1616, -89, -3.764988, 22268450673521.398438, 1, 'binary.dIMAiZxTGtcVSpGTvrOy', 'nchar.重庆市南昌市房山上海街U座 493109', 1630000000064) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630945000000, -271977227, -4833640128107527469, -29661, -35, -25823270909397.800781, -215115.472476, 0, 'binary.FJHanbZZKKPTINTvFuJu', 'nchar.山西省太原县普陀海口路z座 522140', 1630000000063) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630960000000, -1061924199, 80315342405283554, -5983, -53, -35303358.173361, 823245211.259967, 0, 'binary.DHcLdCgRElelyoEjXxxo', 'nchar.云南省辽阳县兴山许街Q座 204291', 1630000000064) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, -2008448047, -2507465824745412203, 5703, 39, 3073425.192830, -4127246.560225, 0, 'binary.PGGakvXluJKSczwfkbFA', 'nchar.内蒙古自治区深圳市涪城韦路L座 184789', 1630000000064) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630959999999, 1731746435, 8656881002710317018, 24493, 42, -51366300790725.398438, 7057719908.426410, 1, 'binary.GcyDiiYlodcmmVYJcfsy', 'nchar.澳门特别行政区萍县长寿淮安路X座 216888', 1630000000064) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 1535163067, 199892343902462919, 8312, 26, -5886360.892067, 99168508385732.296875, 1, 'binary.NajqJanSydLerCaBsTFl', 'nchar.西藏自治区六安县魏都邹路E座 600769', 1630000000064) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000001, -1744206201, -5989201622885398375, -9394, -16, -22156768397.647999, -313965437.622286, 1, 'binary.ljgVmLXQbdPjEOYMKEmN', 'nchar.辽宁省玉兰市新城邵街T座 422511', 1630000000065) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000001, -416139810, -6604173252994763764, -6397, -53, 826553399123.199951, 28094588.168751, 1, 'binary.hcinwRrwHHsWiQqvAaPk', 'nchar.山西省兰州县永川张家港路l座 400205', 1630000000065) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630960000000, 1411363822, -5559490192650806316, -24296, -124, -84520160618.765900, 48233.620153, 0, 'binary.duMCmTwWDPUzYzHdZrXk', 'nchar.海南省建国市萧山朱街m座 337644', 1630000000064) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630975000000, 63717118, -1781107297269354794, 14103, -107, 4455585.961990, -65419.341974, 0, 'binary.BlHycIqcHPSyynpSsDHt', 'nchar.广西壮族自治区桂芝市西峰史路K座 709908', 1630000000065) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, -108473967, 5316060064111515170, 7075, -74, 673543603.332905, 8011713.391465, 0, 'binary.ZsNVSCEilaFozvIMOgVn', 'nchar.山西省秀兰县萧山惠州街s座 291738', 1630000000065) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630974999999, 1701097332, 6995506575159608552, 22867, 47, 4495217807.100080, 38.526985, 1, 'binary.VTDyicwtvKBaCCmjUaDu', 'nchar.吉林省天津县长寿宜都路k座 930822', 1630000000065) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, 257457420, 8835364401886927960, 21297, 21, 0.844779, 716050955.344671, 1, 'binary.TCUqNtJoXousOXFzOaFV', 'nchar.湖北省旭县花溪宋街n座 825626', 1630000000065) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000001, -622389272, -4556948567877249858, -30519, -93, 3.569778, -2781.981860, 1, 'binary.SSnWtedSkSpZxzvsXKty', 'nchar.西藏自治区兴安盟县静安哈尔滨街c座 120405', 1630000000066) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000001, -84782654, -8863584638236264237, -5954, -44, -220.796336, 81484582682754.296875, 1, 'binary.tQzwvVqEIsBAxnymBEia', 'nchar.甘肃省永安市江北南昌街u座 555029', 1630000000066) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630975000000, -7406744, 4193139347325385474, -9523, 51, 96.656028, -858851298.663605, 0, 'binary.QfqialqEnUJEPhCtbzcy', 'nchar.青海省敏县淄川卢路v座 513752', 1630000000065) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 1127802865, 3673503514622188410, -1852, -113, -34837378.663679, -77832209110.181000, 0, 'binary.rdtbEFZKYFDDJcbwZQUI', 'nchar.上海市潮州县友好辽阳街J座 716345', 1630000000066) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 222250382, 7102048992271642834, 30230, 0, -27475250902790.601562, -7.875430, 0, 'binary.okaTayFnJLFHVPbosPMw', 'nchar.湖北省娟县六枝特郑州街o座 923846', 1630000000066) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630989999999, 997138993, 291600547682899796, 8706, 49, 527231819.674759, -76.470030, 1, 'binary.gSCVCKyayTuQHeyozlnD', 'nchar.澳门特别行政区太原县江北六盘水路E座 813817', 1630000000066) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 1527492158, 8131177329340856446, 16932, 24, -485902.216583, 47420.341433, 1, 'binary.xIJLktMoyieUcwAitKAf', 'nchar.广西壮族自治区丽娟县永川钱路D座 710055', 1630000000066) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000001, -1186362057, -8101517704046664109, -8133, 0, 15457163.632735, 790549863.502675, 1, 'binary.domgcCyjkvkGvnfhJchV', 'nchar.安徽省台北县蓟州张路V座 853541', 1630000000067) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000001, -2038096305, -9196878310965066696, -9437, -7, -8043032690399.879883, -3.291758, 1, 'binary.OXzBvNftjfwTqreAuCzo', 'nchar.台湾省西安市西夏贵阳路b座 196727', 1630000000067) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1630990000000, 22826804, -2627900666391071715, -20499, 51, 21.403443, -394389822812.973999, 0, 'binary.abnUGmLKuAKIaCxUeyBU', 'nchar.海南省玉珍县房山罗街t座 776991', 1630000000066) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631005000000, 808597268, 2078929531741697152, -30212, 18, -94851063704.977005, -73674.167661, 0, 'binary.hsfTIdNWdxTPvUwMVrpW', 'nchar.江西省桂荣市清河通辽路U座 806067', 1630000000067) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, -915524872, -8857255033160626964, 266, 91, 11153.892476, -331797474.177709, 0, 'binary.KRBAAEghraBqMLVZKLxR', 'nchar.河南省成都县浔阳傅路o座 233168', 1630000000067) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631004999999, 2045109124, 8620061599081376707, 21842, 65, -577475988406.473022, -230.252609, 1, 'binary.BdfsaHZLBvLoViCnpcaf', 'nchar.重庆市敏市涪城张街Y座 339115', 1630000000067) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, 667151973, 2042134679839477524, 30807, 18, 410455086008.659973, -111467516118.406006, 1, 'binary.MRXRSCLnwHZQLMuEpxxc', 'nchar.上海市秀英县南长南京街E座 857787', 1630000000067) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000001, -1754458857, -2336346102444691323, -21518, -78, -35328084917.622902, 6799588.766270, 1, 'binary.XKvGUqmEfwrFunqrTSnZ', 'nchar.广东省广州县西夏梁路f座 812202', 1630000000068) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000001, -1198184531, -2273612721150875492, -27273, -110, -5590.967174, -36.872012, 1, 'binary.mVuGzysfOhHacpboXfya', 'nchar.澳门特别行政区合肥市怀柔淮安路Q座 830162', 1630000000068) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631005000000, -806268791, 6849292639236533681, -6170, -10, 67.282833, 189646569.949903, 0, 'binary.TDFoHEPsCyCFagPtryZJ', 'nchar.云南省彬县淄川潮州路h座 848046', 1630000000067) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631020000000, -1073614153, 298267925059640970, -26581, 48, -95632.677072, -4.716783, 0, 'binary.VHldMJRfKiPhYVUwVuHn', 'nchar.黑龙江省南宁县清城曾街M座 383184', 1630000000068) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 199565482, -5346899860399957284, -803, 119, 641654.245772, 6.968944, 0, 'binary.CdEpJsyAmonutUDWmrxV', 'nchar.山西省龙市新城任路n座 477007', 1630000000068) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631019999999, 117798603, 420439377677612553, 9253, 22, -1137.400660, -4635970947.218740, 1, 'binary.NJEFxITOCrCDbopXNjvy', 'nchar.广西壮族自治区通辽市华龙柳州路J座 634352', 1630000000068) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 1124688237, 3000258188050272185, 14502, 16, -74396554878085.203125, -9.906537, 1, 'binary.sIxumMWCehMDSrpZPLqj', 'nchar.上海市峰县双滦荆门街i座 639698', 1630000000068) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000001, -1012596446, -5485827047721236689, -13273, -79, -85711452679.407806, -9188.884106, 1, 'binary.vCyjrKXeBvkNVeElgyZi', 'nchar.湖北省明县沙市武汉街E座 620206', 1630000000069) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000001, -460195462, -2468912047516302638, -8474, -48, 1519432858382.800049, 17871.504434, 1, 'binary.dixQMmtXAeagNODPIEEv', 'nchar.上海市阳市平山龚街J座 946821', 1630000000069) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631020000000, 1852200026, -6007978387233236224, 8798, -115, 421576343321.414001, 19100383480.317001, 0, 'binary.iRZrrVUoaDQXPuJZnyDI', 'nchar.贵州省南昌县合川成都路a座 820810', 1630000000068) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631035000000, -1253564785, 9123919785095917881, -15701, 31, -186733280.924988, -61.177294, 0, 'binary.NgOZqYryGuLTQMouuNHu', 'nchar.天津市鹏市沙湾兰州街O座 549919', 1630000000069) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, -1657246812, -4709259982252400503, -17351, -53, 81659768.995530, -7671201354.697040, 0, 'binary.nzcJiagtuSvVewKFNcaK', 'nchar.上海市天津市城北福州路U座 681143', 1630000000069) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631034999999, 1413434601, 2190691240505370364, 11770, 93, 3703815955153.799805, 8268264767.279480, 1, 'binary.oIubCOjKHdxCpeWbqIpp', 'nchar.陕西省秀华市徐汇合肥街s座 902856', 1630000000069) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, 1217613348, 7975114090727787456, 13738, 43, 355726833911.968994, -438162359629.995972, 1, 'binary.uLWXXVcGhcuWJiRaByBZ', 'nchar.河南省福州市东丽徐路w座 747585', 1630000000069) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000001, -2146084891, -23804698624528061, -29407, -115, 3824609640014.350098, -67783404809047.703125, 1, 'binary.RnZVUuAxugAtOZSTEkkn', 'nchar.青海省北京市门头沟陈路Q座 257513', 1630000000070) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000001, -1123855083, -2174482244567720750, -12610, -101, -13889290720.493999, -44730168.376831, 1, 'binary.bKiOAOxQmTuUxvfbdpdl', 'nchar.宁夏回族自治区坤市锡山宁德路j座 838367', 1630000000070) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631035000000, -1406844909, 6947966913619883113, -6412, -80, -5.700965, -15608.263924, 0, 'binary.ShuwgacOGYsjNECdcUUt', 'nchar.河北省婷市浔阳南宁路D座 449083', 1630000000069) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631050000000, 1665571243, 5875622308303693056, 26189, -3, 9184334500122.300781, 692.411491, 0, 'binary.mkdoDznGUjHxCWsEReic', 'nchar.天津市明市翔安王街K座 149828', 1630000000070) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, -866699485, 5150552413312365721, -26931, -72, 304388231344.469971, 1887123.429534, 0, 'binary.gXoNDsUhSIxwTtChSVKC', 'nchar.广东省阜新市牧野何街b座 893235', 1630000000070) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631049999999, 1647088140, 6276524783005291573, 18506, 126, 520220.525305, -11656506505.388300, 1, 'binary.uFvJUrDBxvRNfycpsATy', 'nchar.广西壮族自治区春梅县新城梧州街p座 914349', 1630000000070) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, 110172144, 3880079542176303291, 7450, 60, -70101821.922084, -766086105.175900, 1, 'binary.sGupuerrJMrewABhIOQF', 'nchar.重庆市永安市孝南东莞街p座 524436', 1630000000070) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000001, -2334950, -9174808951498480949, -25376, -56, -706896093828.322021, -7054905528.909860, 1, 'binary.eAsXnsSVfXpwSeWxjIep', 'nchar.新疆维吾尔自治区婷县清城李路R座 863352', 1630000000071) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000001, -1709242287, -1024364150913211829, -18517, -95, 612563.744410, 71675616881.702698, 1, 'binary.xwUGLqzyDiIWFYjQChPb', 'nchar.澳门特别行政区建国市东城汕尾街H座 310414', 1630000000071) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631050000000, -1438324283, -2616966529186093777, -15740, 119, -7052.121172, 6.860693, 0, 'binary.gwSKZcnvepcSGvGhQIhU', 'nchar.江苏省娜县安次长春街b座 373327', 1630000000070) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631065000000, 1572844268, 7954082573907623254, 28743, -77, 526200414.754712, -610.377887, 0, 'binary.dhmqewaCwxuZgymXkJYg', 'nchar.甘肃省六盘水市高坪刘街K座 997983', 1630000000071) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, -1997711328, -8305458100196327173, 15855, 65, -1.591103, 978771.885779, 0, 'binary.ErzqbmUDFqGBseJATzNB', 'nchar.福建省邯郸市龙潭天津路S座 700145', 1630000000071) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631064999999, 118186835, 7105850083426581209, 23043, 123, -294186.598870, 631977325.777490, 1, 'binary.jZYzFZwYbBOMfCYxLkEu', 'nchar.北京市宜都县山亭福州街B座 229199', 1630000000071) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, 84760302, 4023057210065827734, 20658, 50, -374538737.396905, 2.546280, 1, 'binary.GrfdYDSeONBkUWnVxrIE', 'nchar.重庆市关岭县兴山荆门街i座 354010', 1630000000071) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000001, -31117145, -6368825817854208301, -4679, -13, -193.784819, 9941696226439.189453, 1, 'binary.DtkHUqQYbIbRkWfoerGJ', 'nchar.湖南省建国市六枝特广州路d座 104163', 1630000000072) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000001, -1214911170, -3868394291968674672, -20441, -79, 567.284011, -99982135106.938004, 1, 'binary.LThAPucPiLueKWedudxF', 'nchar.青海省海门市牧野淮安街D座 825707', 1630000000072) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631065000000, 1697187351, 530203760413095118, -17911, -116, -501.905581, 98.459334, 0, 'binary.mEMXftUdsHdFpIbnKRsd', 'nchar.宁夏回族自治区峰市平山王街M座 201716', 1630000000071) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1610525794, -5290794365132208573, -15731, 123, -335854906.304436, -24139226804197.199219, 0, 'binary.daKMhFVbQQZKGiTKOlls', 'nchar.新疆维吾尔自治区坤市牧野武汉路f座 977660', 1630000000072) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1087333161, 2572492011929726786, 25923, -25, -4763484.301768, -5226373.298693, 0, 'binary.mmMznhJKAQPTjucpnZSc', 'nchar.澳门特别行政区杭州市南湖大冶街g座 395605', 1630000000072) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631079999999, 579193173, 5793829126142110648, 4490, 80, 78951.390551, 19287929360.243801, 1, 'binary.MUCIKFgmyfodWgXzBwpA', 'nchar.台湾省淮安市上街海门街G座 209332', 1630000000072) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1885599296, 5754849838280444727, 16958, 107, -53.481198, -9.937711, 1, 'binary.BrUGLflxCuUJqYeSVVsN', 'nchar.吉林省刚市高明夏路B座 373737', 1630000000072) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000001, -718949368, -4253205043582583806, -12561, -68, -134568795933.429993, -71046052762709.500000, 1, 'binary.gOwViQPVHLSImYonXzkV', 'nchar.台湾省红市永川六盘水路m座 707904', 1630000000073) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000001, -1419546071, -5628013404742242442, -19653, -61, 4641210572.820400, -516.363302, 1, 'binary.OhALrYMiHXSjCpcdfoPs', 'nchar.甘肃省玉华县高坪长沙路K座 553129', 1630000000073) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631080000000, 1801658141, 3654105358778411870, 18529, -80, 8.652081, -1554626.681517, 0, 'binary.GyuqMCRbacNJHUbGmkXh', 'nchar.台湾省芳市安次薛街o座 207547', 1630000000072) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 2136843976, -1210226910088276755, 20600, 23, 539871237741.179993, 49233075000.644600, 0, 'binary.CoZYsjGjLcrAfxoNvULS', 'nchar.山东省嘉禾县平山呼和浩特街F座 641469', 1630000000073) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 486220117, 2599712669438010104, -6519, 47, -0.673515, 55.218328, 0, 'binary.hUsJbjOwNFjCpNXnLWPF', 'nchar.广东省六安县秀英上海路E座 148463', 1630000000073) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631094999999, 17707398, 8375178158416972213, 1322, 112, -62111552150188.000000, -3.488348, 1, 'binary.iibUAdMnGmuQRjnCboca', 'nchar.海南省澳门市萧山冯路k座 476673', 1630000000073) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 1540458244, 8608337136208602956, 13530, 122, -258.798333, -18390773070513.101562, 1, 'binary.dKifdiuRtzPzgDOgwskf', 'nchar.西藏自治区成市江北丁路k座 811706', 1630000000073) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000001, -295978274, -3611928104731611403, -928, -29, -349214171411.169983, 627603759.980824, 1, 'binary.tPAAsLZTOrrgDWhhbvPA', 'nchar.河北省深圳市和平李街k座 107359', 1630000000074) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000001, -1110370293, -2926428742790858456, -4685, -42, 594526965.911060, 979845489.815900, 1, 'binary.KCwadhHtPeNvxeOTurpC', 'nchar.四川省桂香市城北张路Z座 135595', 1630000000074) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631095000000, 1016372608, -248268842333445258, -24994, -40, -855742777.413557, 412898933.803171, 0, 'binary.oektpgoczIFkPkMZhePp', 'nchar.四川省楠市锡山王街T座 347259', 1630000000073) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 1618012586, -4877738856165917254, -14321, -36, -2.446783, -8386.899686, 0, 'binary.cGuZNtDbGSgYATZIfhUQ', 'nchar.湖南省军县闵行沈街e座 942969', 1630000000074) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 336517476, -2851549763879094055, 24260, 42, -284453185.760948, -82312598023267.500000, 0, 'binary.nNUcWFSnghHEXoJZyLSZ', 'nchar.台湾省玉英市海陵广州街D座 932017', 1630000000074) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631109999999, 325913454, 4965185676452141874, 1927, 45, 928.161230, 5002796.693905, 1, 'binary.KPSdukhCAQdFSCjoAXyb', 'nchar.天津市嘉禾市和平王街Y座 800951', 1630000000074) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 2115611339, 5603712632691054487, 2040, 92, -88631.744485, -33319292708726.398438, 1, 'binary.nDvmXubKLBkwjQvBrowE', 'nchar.广西壮族自治区合山市大兴邯郸街C座 120130', 1630000000074) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000001, -416517208, -3563087064047686173, -32590, -97, 804.877418, -8174392626838.679688, 1, 'binary.yJGXGtGzgxaoyzDFNKzn', 'nchar.新疆维吾尔自治区建平市龙潭曹街X座 409242', 1630000000075) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000001, -1502953470, -2238010217654677886, -11068, -38, 69129137.954435, 599892.899153, 1, 'binary.OdnrQcfrzxIUYGMYkhgP', 'nchar.上海市霞县南溪曾路b座 536348', 1630000000075) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631110000000, 451771496, -7517497607249821959, -32099, -92, -213627.556542, -8726497.619554, 0, 'binary.DhxlHgZRZtmSyyhkGtuW', 'nchar.云南省洁县滨城梁路p座 996135', 1630000000074) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631125000000, 32878642, 8801402285444036889, 21539, 48, -596417925905.134033, 373248896.126035, 0, 'binary.RjzhwvLAyYAYYCfkjFJj', 'nchar.重庆市超县梁平温街z座 283315', 1630000000075) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, 71147814, -5159443215460099961, -15082, 102, 5968659.321019, -26592.753571, 0, 'binary.CljlscxcGjjciERgOxKg', 'nchar.宁夏回族自治区玉梅县吉区海口路d座 230176', 1630000000075) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631124999999, 1015662815, 4795567142964951653, 7167, 1, 325791556.701457, -143633938642.838013, 1, 'binary.JHrHPuhooNwLgDYQlFil', 'nchar.上海市玉华市门头沟长春街L座 619720', 1630000000075) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, 785489232, 7788643811202703115, 31943, 105, -9652684.346045, -250.125950, 1, 'binary.FkrDvYiLeZOqRVvYQpJr', 'nchar.贵州省璐市永川长沙街K座 573245', 1630000000075) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000001, -370350844, -2328721985404462718, -8114, -97, -65.996745, -23054596683.701000, 1, 'binary.YpDtIgmdxGtGOlkpYQbb', 'nchar.吉林省潜江市普陀哈尔滨街G座 354284', 1630000000076) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000001, -1104506218, -4613678193581849079, -1965, -116, 67350915247071.203125, -124.355022, 1, 'binary.cGsEpboYJbYsTSnKYKUz', 'nchar.宁夏回族自治区贵阳市永川刘街l座 557372', 1630000000076) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631125000000, -1975758915, -7017697117625132855, -2347, -78, 13301869.493635, 734276825210.396973, 0, 'binary.JMTLLJHKsHvKhVKhxUdD', 'nchar.湖北省淑华县上街郑路d座 249081', 1630000000075) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631140000000, -638220498, 7787936362129921947, 2622, -93, 80359528.879084, -6439.177632, 0, 'binary.KYZqgXaxguXpZiDTipOe', 'nchar.云南省台北县翔安淮安路T座 548657', 1630000000076) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, 1695929408, 6613622196219250257, -19521, 126, -294182.533903, 872281666.129710, 0, 'binary.bxqGrgGKqftbTFsJVxse', 'nchar.陕西省长春县梁平重庆街f座 969435', 1630000000076) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631139999999, 910799127, 3748990393909505200, 19872, 110, -1.774037, 962.107067, 1, 'binary.SMVCQwJbrLKRzlPTMAle', 'nchar.广东省淮安县南长陈路r座 442094', 1630000000076) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, 82578099, 1083300683395237822, 14840, 49, -2897441254294.259766, 78241.170285, 1, 'binary.zRnJDsiOdVdrjyezgNqk', 'nchar.新疆维吾尔自治区欣县上街郑州路T座 334404', 1630000000076) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000001, -1933040168, -8697939131006546022, -21092, -35, -3404654038.996900, 64905.498178, 1, 'binary.VlVdAdmtswNJQrRzSAPQ', 'nchar.河南省辽阳县长寿兴安盟路C座 114674', 1630000000077) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000001, -472785311, -4446555925790326535, -23024, -3, 430107081.785272, -5.140820, 1, 'binary.NUAVJEKQvBvlnJToDnoJ', 'nchar.河北省金凤县双滦石家庄街D座 264667', 1630000000077) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631140000000, -1325721293, 1101829200999493900, 13912, 72, 151194.856760, 931990832.169087, 0, 'binary.siTpYJVsmIUfmSUiacBG', 'nchar.海南省六盘水市房山重庆街R座 786868', 1630000000076) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631155000000, -622133611, 4304164307993590613, -1028, -75, -794.965943, 7129885424.126380, 0, 'binary.PxLiilJCeaWfrPEPswvJ', 'nchar.辽宁省齐齐哈尔县秀英王街V座 753913', 1630000000077) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, 601975873, -8460674145788246343, 26537, -18, -117.582363, 4.278494, 0, 'binary.DIEUsVGrYCIYpOHOMbYp', 'nchar.青海省玉市高港六安路n座 697115', 1630000000077) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631154999999, 794346686, 2813747690977192290, 19623, 118, -592938779870.880981, -691339105509.250000, 1, 'binary.IkXhYGsGOuGyQXxMcENO', 'nchar.新疆维吾尔自治区兴城市白云海门街z座 410778', 1630000000077) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, 2077153378, 6007901518953365006, 32590, 25, 8269699092873.599609, -3523058497792.450195, 1, 'binary.zDNlRsoVabJhMNFJfTBy', 'nchar.江西省冬梅县永川王路R座 198556', 1630000000077) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000001, -969135765, -1764680015243416710, -27020, -59, -8.491632, -14661.965520, 1, 'binary.jnObVGyDvkIgXvnFpKMn', 'nchar.广东省南宁县涪城沈阳路O座 754414', 1630000000078) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000001, -1576113711, -3627767440156041231, -18534, -66, 495.121273, 53596032.859107, 1, 'binary.lOIexILoDTNYoRmKXkCf', 'nchar.西藏自治区瑜县海港史路q座 848639', 1630000000078) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631155000000, -1548677743, -6457865964462190618, 21103, -53, -421862729108.307983, -2360116167608.370117, 0, 'binary.ZmmTgtzFlywNsFIAjzDE', 'nchar.台湾省秀荣市高港高街E座 134599', 1630000000077) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 749993747, 3019243752157013695, -17619, 43, -1469831614725.439941, -6462342550.269600, 0, 'binary.PTOhqXMEYVQUAwAPztJg', 'nchar.广东省哈尔滨市花溪深圳街S座 621146', 1630000000078) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 256334370, -4650384778595555945, -7069, 22, 596577877.192931, -70129653418.844894, 0, 'binary.zWNroTXoIVAfFKMCHdqU', 'nchar.广西壮族自治区梧州县崇文张路j座 870677', 1630000000078) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631169999999, 366270121, 4191114768474723941, 28897, 88, 36764251.456900, -819995344305.519043, 1, 'binary.KMQKIVvLXKZtzgvGXoZV', 'nchar.上海市深圳市朝阳王路c座 308052', 1630000000078) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 904819741, 5165411582118084217, 22379, 49, 5.680573, -23448296491.688702, 1, 'binary.RksynZHDBofeCbnNeueF', 'nchar.西藏自治区秀兰县永川武汉街f座 720233', 1630000000078) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000001, -406248475, -3565233828007899422, -28694, -119, 6468114.299821, -9139.276334, 1, 'binary.yDkbSjlEhWxgIdEKjdyw', 'nchar.陕西省明市东丽郭街X座 797325', 1630000000079) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000001, -110865946, -7795005438146655234, -26791, -6, 614284367.384455, 50856740.580794, 1, 'binary.tpgIuOiJsKacrUZBOcHs', 'nchar.北京市太原县大兴高路N座 354025', 1630000000079) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631170000000, 1503400003, -80780332688017476, 31847, -84, -1188729980516.979980, 2115639995.697030, 0, 'binary.zMhdERtluQHYrYkarBsp', 'nchar.西藏自治区沈阳市沈河兰路O座 514687', 1630000000078) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631185000000, -703829541, -3015290584870545849, -26195, 25, 65051636.569539, -1355107949.868670, 0, 'binary.pFxWRymlGotRbPxBaKqU', 'nchar.重庆市拉萨县孝南刘路P座 393437', 1630000000079) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, 322375238, -6241662551377872210, 7578, -41, 9181.469818, 58823.373155, 0, 'binary.yeTYMVojTpBxHFVbiTVH', 'nchar.贵州省天津市吉区潜江街E座 745985', 1630000000079) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631184999999, 901680081, 2918745868131285426, 1954, 123, 799091.717635, 19529.323662, 1, 'binary.HriCdYQCgMqDJrsEPvId', 'nchar.香港特别行政区伟市蓟州崔路h座 182270', 1630000000079) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, 959830805, 8687660735074694942, 30908, 121, -56072935323.713600, -66360254180.759499, 1, 'binary.hnKLmxkdGfJPtsNffUnx', 'nchar.重庆市云市孝南深圳街D座 180484', 1630000000079) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000001, -88966985, -1936037048857392721, -18099, 0, -824204261009.363037, 1697.886509, 1, 'binary.pfXnsxKPgpyOLUjhDIlh', 'nchar.青海省辛集县东丽拉萨街Q座 914524', 1630000000080) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000001, -156903452, -7204919546158466118, -26775, -7, 3343624886104.990234, 25.895039, 1, 'binary.DVJPFxpqidEqIbsMkqvk', 'nchar.河南省桂香县普陀合山街y座 473206', 1630000000080) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631185000000, 242951644, -6425706509022302444, -115, -95, -7428172351326.129883, -243792.108001, 0, 'binary.XYfDpoamFuPazIzqqvyQ', 'nchar.江苏省六安县沙湾沈阳路D座 111215', 1630000000079) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631200000000, 917616213, -6105526769675738367, 16833, -4, -912283922.301867, -626.880806, 0, 'binary.UJhwYiRGgenyykMzvCIG', 'nchar.陕西省慧市沈北新台北街V座 622592', 1630000000080) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, -1010524677, -2564524247724824015, -31994, -48, 478258495629.161011, -2756525.506410, 0, 'binary.JqnispIFJYlFtxhJzBtt', 'nchar.贵州省梅市上街福州路U座 843893', 1630000000080) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631199999999, 1676885574, 5801667850572164138, 9268, 5, 81943569878020.000000, -2651878397664.700195, 1, 'binary.tJnMwFImycFDwadSKvfB', 'nchar.西藏自治区邯郸市东城刘路G座 620688', 1630000000080) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, 202560740, 8585057740824701967, 2754, 102, 4256312.669784, 2841641554.533030, 1, 'binary.iZuBXyHivNVGIECkZpgp', 'nchar.青海省北镇市永川广州路t座 650954', 1630000000080) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000001, -1469589614, -6328986345920555773, -13057, -114, 7124.472347, 1922792124.232900, 1, 'binary.IttYeshrxTWRQzrMpNEl', 'nchar.江苏省呼和浩特市大东钟路V座 373715', 1630000000081) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000001, -635536894, -7540406212992578302, -356, -42, 7506079.766892, -846620.925646, 1, 'binary.FBEKhNRXwVaIefMjxKLw', 'nchar.重庆市马鞍山市金平孔路A座 105425', 1630000000081) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631200000000, -1786041154, -5812150165598736743, 31691, -112, -99.932548, -78491263.832105, 0, 'binary.EeccUPRfABMAdtMqrUTP', 'nchar.海南省鑫市西夏兴城街R座 301651', 1630000000080) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -1625191731, 6167214474139728168, 11139, 121, -94335581842.567902, 3059478274.613470, 0, 'binary.ZxUIonzZMsBhCCWqYrmZ', 'nchar.天津市拉萨县萧山上海路d座 252099', 1630000000081) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -47135681, -6436433940308594642, -21351, 98, -8067393645072.580078, -55.209057, 0, 'binary.wklKlUzIzCbSAlTVhPtf', 'nchar.江西省通辽市西夏曾街D座 770856', 1630000000081) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631214999999, 1572527931, 6199432476029588286, 15047, 53, -329811.437951, -690962.952604, 1, 'binary.VQbrSGdoLASviEaZfVKb', 'nchar.河南省磊市城东曹路Y座 412471', 1630000000081) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, 1025618009, 8668297953739699167, 6895, 36, 809883.310963, -1830.234393, 1, 'binary.bSASugSRycgShDfZVaRz', 'nchar.天津市亮县江北潜江街S座 540483', 1630000000081) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000001, -256464376, -896102382541120115, -19194, -122, 4123590577424.399902, 2656422.305476, 1, 'binary.CyxinbAlvUlbbQCMCmuX', 'nchar.青海省东莞市蓟州叶路g座 257461', 1630000000082) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000001, -425746119, -8056851883105951074, -32462, -51, -590076.331164, 324718675328.812988, 1, 'binary.MqmXkQoSJpKeTWWfLSjQ', 'nchar.青海省宁市高坪何街L座 312122', 1630000000082) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631215000000, -1005159716, 5814777795009857976, -20922, -120, -5.121036, 72269204554.118393, 0, 'binary.krPFwdRphWiLhiLWffGj', 'nchar.云南省太原市翔安叶街s座 968090', 1630000000081) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631230000000, 300807334, -1393720031020538944, -10631, 114, -7497311.501447, -7843875.126062, 0, 'binary.gUfZXmtuaUyEMqvzCGVT', 'nchar.甘肃省潮州市海陵张路j座 111530', 1630000000082) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, -61728336, -6277168556471947655, -14631, 121, 108260.570588, -6225158.445263, 0, 'binary.wjIFBHVvPkzNnyfFqcez', 'nchar.内蒙古自治区小红县锡山张路q座 451673', 1630000000082) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631229999999, 456695922, 6065595906254092225, 13855, 95, -9.750988, 52263.948430, 1, 'binary.HpsqFduULuvFgUNWLaIS', 'nchar.上海市辽阳市友好宜都街J座 260640', 1630000000082) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, 1314422800, 2343245198878315836, 4530, 110, 5.523721, 52.963146, 1, 'binary.BqILcPWCGdFitnkIUITX', 'nchar.黑龙江省萍市兴山香港街w座 689416', 1630000000082) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000001, -1404438854, -6648294746644259055, -29920, -45, -4879.347387, -397204029.641602, 1, 'binary.hWYLNHYZihIwrBwiQKLu', 'nchar.内蒙古自治区大冶县上街谢街p座 354579', 1630000000083) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000001, -1560018283, -8582655292633538077, -7916, -9, -813743561.246321, 180605419479.515991, 1, 'binary.rBKhOAPgsFlVbktylkBZ', 'nchar.海南省贵阳县新城张家港路o座 184911', 1630000000083) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631230000000, -1185224620, 3781565114446947943, -32425, -124, -966176.836584, -6194.817285, 0, 'binary.MAHjOJrSnXwRnDjhjhgs', 'nchar.湖北省哈尔滨县璧山嘉禾路W座 107867', 1630000000082) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631245000000, -629005742, 2962477037931783881, 30753, -22, 97626628.877839, -55509075810587.898438, 0, 'binary.bHeXAGMGzghlOuWCGmyS', 'nchar.内蒙古自治区荆门市上街王路f座 313159', 1630000000083) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, 1289038189, 208643021208222754, 24329, -84, 528626.682214, 4631263449.782700, 0, 'binary.VBcGIQKGeBlgodeKkqIO', 'nchar.天津市梅县孝南许街M座 955579', 1630000000083) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631244999999, 1089857524, 8092714018391461421, 30637, 62, 822.840197, 86578529728.784698, 1, 'binary.ReHWrHhhULYKBWJbDHAl', 'nchar.安徽省合肥县大兴太原街z座 183173', 1630000000083) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, 1508646219, 5498801626248258629, 6153, 2, -6981.974013, -1.262354, 1, 'binary.rKJJrwLYFsmeLilRmdII', 'nchar.云南省秀英市静安罗路N座 799241', 1630000000083) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000001, -1072147772, -6285554104164008129, -13029, -38, 399653108.497808, 9.954981, 1, 'binary.CrzTcvxjDbWSNGHlnGgZ', 'nchar.福建省文市沙市聂路v座 919011', 1630000000084) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000001, -1744587122, -4289304118779607521, -3261, -49, 578908.973758, -3283275.161042, 1, 'binary.hnKdQBaXRtRTvdBMxxki', 'nchar.四川省秀荣市江北韩路N座 794727', 1630000000084) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631245000000, -1768419964, 6812389139494481536, -22315, 57, -3991350668.256720, 272847.980876, 0, 'binary.eaKoadYSnHxIAwGvMADW', 'nchar.内蒙古自治区颖市城北呼和浩特路M座 270782', 1630000000083) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631260000000, 62032825, 2419036897017949691, -18301, 34, -34067326787.909401, -4490033.331223, 0, 'binary.vcjOGKOcKjTDVUVgwDDP', 'nchar.福建省太原县上街宁街J座 989695', 1630000000084) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, -955905466, 8811462063140315136, -25829, 76, 4112554.855015, -353081090.814735, 0, 'binary.sjVkMmweDnQHqIahYBJg', 'nchar.宁夏回族自治区佛山市清浦陈街C座 507576', 1630000000084) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631259999999, 1740302532, 7618062051313344766, 24234, 108, 19.258022, -481986511.838220, 1, 'binary.FHckvgRsNysvnrkBnxPi', 'nchar.河北省丽华市朝阳龚路E座 249405', 1630000000084) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, 25207874, 8557074180207725958, 5121, 76, -27.837670, 7472.845942, 1, 'binary.yuETGYVvdZgoTrKTDVlM', 'nchar.甘肃省红霞市牧野关岭街C座 146380', 1630000000084) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000001, -368156241, -8290129222861749037, -32369, -87, 8778614.707332, -8.396911, 1, 'binary.oTlXrpmNqzuPQFtdJXAL', 'nchar.广东省淑珍市东丽周街e座 132961', 1630000000085) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000001, -1391854623, -5255734908673798063, -25898, -63, 6.380914, -981.272293, 1, 'binary.fRTcoeHlACSZuECHxLsF', 'nchar.西藏自治区兴城市东丽惠州街w座 939232', 1630000000085) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631260000000, -1478713358, -7256379885586266224, 7456, -127, 39.210475, -37429036261.203003, 0, 'binary.iLHDKCeEycRrHQjdctpP', 'nchar.贵州省琴市锡山嘉禾街V座 586837', 1630000000084) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 1637182628, -7252330432834078022, -13577, -34, -21982.383242, 851033.725636, 0, 'binary.qbwfbTcZcBXRYDFLPsYc', 'nchar.西藏自治区香港县门头沟袁路c座 659900', 1630000000085) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, -226012380, 714907917095146625, 26998, 45, -378863.223518, 63837840.367547, 0, 'binary.wLhqoOhYDhOLDWgTquSO', 'nchar.广东省淮安县城东惠州街H座 226497', 1630000000085) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631274999999, 1196830217, 7803469523299853932, 4057, 68, -72455041894.121094, 31008329.254161, 1, 'binary.tnTDmSbUwexajzyGQKbU', 'nchar.天津市桂香市海港邱路v座 930985', 1630000000085) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, 1896164509, 1992693218102163749, 20742, 127, 3279876658623.740234, 97971293613823.203125, 1, 'binary.EwwiwjOJeqmxqOmBSCvF', 'nchar.宁夏回族自治区秀英县高坪银川街T座 599205', 1630000000085) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000001, -238216455, -6595002088264755363, -17352, -35, 30291122943460.601562, -523766877.255375, 1, 'binary.JiuAgoqIJOWzLvLnmvBq', 'nchar.四川省六盘水市梁平佛山街Y座 160978', 1630000000086) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000001, -1959680714, -5838771977352828969, -30312, -80, -40302216017.319298, 65262645551.558701, 1, 'binary.kKcTvQecUQcVNHCxdbMq', 'nchar.吉林省亮市海陵汕尾街y座 455171', 1630000000086) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631275000000, -1454514443, 2104326254606882084, -3977, 31, -10.683775, 8146970744.319100, 0, 'binary.baHWrZaiEoVOFqPQVViT', 'nchar.上海市明市锡山台北街g座 112480', 1630000000085) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631290000000, 202676239, -997640316674086097, -22675, 17, -597819.667252, -989103790.253560, 0, 'binary.rdoqbRUQFOZbxDrDpEXv', 'nchar.福建省斌市清河刘路r座 438569', 1630000000086) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, 274493417, -5455741168804436813, -27247, -111, 9288775.263537, -3826.901732, 0, 'binary.eWJNzFpYfQFYqDTlVEuS', 'nchar.广西壮族自治区深圳市海陵邓路e座 598287', 1630000000086) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631289999999, 995318978, 44640273583296417, 14989, 72, 86411387362.516006, -21052069.696864, 1, 'binary.WhVBrissKQCBrAUpwRHT', 'nchar.陕西省桂香市清浦米街p座 269494', 1630000000086) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, 2136730540, 5315445691851854143, 14450, 103, -6335650603.174980, 501305134.199676, 1, 'binary.zljXuXtZduZhLQoCTEYI', 'nchar.辽宁省凯县西夏太原街R座 148010', 1630000000086) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000001, -1917806640, -4720258233332513149, -31214, -45, -8113.185976, -82050.442684, 1, 'binary.dwHmbtStCEgnJspmTqwD', 'nchar.新疆维吾尔自治区桂芝县长寿巢湖路A座 782531', 1630000000087) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000001, -981665942, -278699770296496646, -13050, -47, -41130.414162, 453641642.546780, 1, 'binary.BkkbbPyxvTkDiTzQpRiV', 'nchar.浙江省合肥市东城成都路c座 895347', 1630000000087) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631290000000, -834112430, -6635305571569691219, -19258, 73, 919646.148350, 321.289191, 0, 'binary.XzCcSXArLMKFLjFRyHJe', 'nchar.上海市阳县崇文丁路I座 548147', 1630000000086) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631305000000, -568876147, 4115921496929882007, 21876, 118, -766463785.889029, -63.898006, 0, 'binary.WPbsfDfyJKEoZCpyfhuq', 'nchar.青海省贵阳市大兴彭街W座 571855', 1630000000087) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, -1963919658, -7528759656724736232, -15462, -49, 82.771605, 228.648728, 0, 'binary.gvsIwjOVqnvwPYubIriV', 'nchar.浙江省合肥县蓟州蒋路s座 450606', 1630000000087) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631304999999, 862001048, 8289879675128879554, 10897, 54, -31560046.871248, -2510800476075.919922, 1, 'binary.ZakYSeKJSEQfAnbEPpdl', 'nchar.湖南省秀华市锡山牟街K座 639697', 1630000000087) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, 1928593809, 5104651015083770612, 32255, 58, 804488379.612065, 9429749762380.500000, 1, 'binary.wgTbKbFYQRosfKvUQSkH', 'nchar.重庆市桂珍市梁平乌鲁木齐街N座 153448', 1630000000087) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000001, -1547198379, -350643391542151214, -13828, -103, 64477812.167046, -2903.526723, 1, 'binary.vJieiALrJbMqqRhnMbqe', 'nchar.澳门特别行政区丹丹市丰都南宁街Q座 989300', 1630000000088) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000001, -134242385, -3869570405276188197, -32581, -104, 43107832.702441, 44056.657014, 1, 'binary.IkbIEnMQPVXBWOjcanNo', 'nchar.香港特别行政区东莞县丰都朱路U座 249243', 1630000000088) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631305000000, -597984033, -3361517834367129147, 75, -104, -7409.500037, -91.360942, 0, 'binary.TNnOUCoNllHJVskqaqWO', 'nchar.上海市玉梅市和平李街k座 583138', 1630000000087) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631320000000, 904033245, 1599598424557218108, -30005, -24, 4541.289136, -54.239926, 0, 'binary.iEBPEAmKXlpBUXwoALUt', 'nchar.吉林省刚县璧山刘路g座 682546', 1630000000088) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, -2079364817, -654287221384079565, 15782, -26, 748.766841, -4.965685, 0, 'binary.WEJkxJeFLZZByzCPoRdY', 'nchar.福建省太原县永川兴城街w座 961260', 1630000000088) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631319999999, 526847966, 5775445950760637967, 14304, 60, 81172141354.356293, 870255843.925099, 1, 'binary.MdntNFfvvyCStMNHBKIQ', 'nchar.广西壮族自治区合肥县西夏刘路q座 990741', 1630000000088) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, 537739308, 5090468389474927294, 22579, 33, 3433128.844123, 3872298849645.379883, 1, 'binary.abGKYFWBPWLVPYEvggCd', 'nchar.陕西省马鞍山市海港韩路B座 339379', 1630000000088) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000001, -795092969, -3711201539316715587, -11107, -98, -286579.413055, -3.739221, 1, 'binary.thHeWmYWAcgObBqqwqqB', 'nchar.四川省成县大东张街u座 740678', 1630000000089) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000001, -1559644142, -9027525835433685991, -2711, 0, 5557460.651989, -40007573640075.203125, 1, 'binary.NglLAfkhOUvNACXvlutl', 'nchar.西藏自治区丽市兴山萧路O座 326707', 1630000000089) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631320000000, -1107831086, -1823478878718094475, -27832, -90, 99212399503.662796, -74696134214106.500000, 0, 'binary.drlMMrbdDWnwUwkvFszC', 'nchar.青海省利市西夏辛集街A座 820543', 1630000000088) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631335000000, -1747665689, -3268703357637921126, -3204, 118, -7.843951, 5236837072088.639648, 0, 'binary.fKLvQiBIrygGolyqKaVe', 'nchar.福建省淮安县金平海口街k座 430777', 1630000000089) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, -828294034, 2657326032914454094, 21837, 8, -69702504398061.000000, -4450579.238900, 0, 'binary.esplmUmrLBFMyAKMDAzP', 'nchar.福建省北镇市兴山长沙路D座 908303', 1630000000089) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631334999999, 1804609738, 6857866489996684420, 32112, 37, -2477401204.802390, -3021.525948, 1, 'binary.DCLURcwhhaDhNKDcTLYq', 'nchar.安徽省西宁县江北郑州路R座 729863', 1630000000089) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, 437917493, 1733585568524751775, 14155, 38, 3358226.279225, -1184.537102, 1, 'binary.KYgSrTTQzSmndNwhbsxY', 'nchar.黑龙江省丽丽县双滦包路i座 494311', 1630000000089) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000001, -1727537263, -5460926119155540560, -9690, -95, 9289065.542646, 7.188845, 1, 'binary.yMKTfSNsTiIkXhhIQCtC', 'nchar.新疆维吾尔自治区柳州县东丽高街d座 298894', 1630000000090) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000001, -2120093363, -4777070161903130182, -8197, -30, -383000.885433, 9678.947342, 1, 'binary.xYtVuqPnZXiCBdRdskEu', 'nchar.海南省沈阳市大兴邓街P座 552544', 1630000000090) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631335000000, 1750088050, 3697839127781862571, -675, -53, -99724.435061, -28255356.108959, 0, 'binary.ZoJveGLeqQgcoitfoHqY', 'nchar.广东省丽丽市城北呼和浩特路P座 380188', 1630000000089) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 1880035325, -4802629856794707608, 22068, 100, 718062376.187030, -506130.634445, 0, 'binary.CfGiHufKQulMxwAItOfy', 'nchar.山西省想市黄浦邯郸路X座 818941', 1630000000090) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 165546984, -6984734151301447256, -16166, 96, -1236884449.480500, -639029206.988874, 0, 'binary.AvOGTRfAVrYaxEAHhJKk', 'nchar.四川省坤市清河文街M座 728309', 1630000000090) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631349999999, 1514387404, 2958688219263646841, 21127, 20, -46918158932248.000000, 2353213017.825450, 1, 'binary.bfrSvHvqHlFluwivEgxS', 'nchar.山西省玉市白云马街p座 592361', 1630000000090) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 697337569, 7135356390013225753, 32569, 116, -6611897.211005, -1.935866, 1, 'binary.ETaNjgpPPzwvOkzPbLMs', 'nchar.台湾省敏市朝阳余路L座 739835', 1630000000090) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000001, -1737290956, -4744278138504720755, -23704, -13, -20682554.975530, -5505407.317856, 1, 'binary.lqhFKjcHzvbkdWVoYtwW', 'nchar.河南省兰英县朝阳天津街A座 102658', 1630000000091) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000001, -1792783313, -6546177515445472740, -17996, -82, -2785.518995, 406175.862275, 1, 'binary.ZSgzqADQgOFLGnOSTTgl', 'nchar.云南省丽华县浔阳黄街u座 463279', 1630000000091) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631350000000, 1675171705, 4482392883505129283, 3877, 60, -529410318.578000, 87242703.165205, 0, 'binary.pPrjATzIWSjGnAQlByfg', 'nchar.澳门特别行政区玉珍县南溪刘路J座 234624', 1630000000090) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631365000000, -120996932, 7464290233605180116, -21030, 89, -7265.737889, 38451.519083, 0, 'binary.NdYvAIsMTcBJbRZBuEMZ', 'nchar.四川省成都县六枝特武汉路u座 796761', 1630000000091) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, 1626331139, -3444691871293469179, 1200, 69, -2719528.747073, 35.186089, 0, 'binary.SfqGXOabBHmYWmwbjAmD', 'nchar.江西省彬县清浦丁街f座 768134', 1630000000091) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631364999999, 1201642931, 3048123187871650645, 5183, 51, 5407076381615.250000, -94.505271, 1, 'binary.pCxeaejTSQlmBlttHhTg', 'nchar.江苏省六盘水市兴山宋街z座 229812', 1630000000091) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, 606859861, 3791676681420995058, 12977, 20, -11.820203, -60910962808.638702, 1, 'binary.dkXKTqEjhmKoFQauhwjf', 'nchar.甘肃省慧县淄川何街t座 472618', 1630000000091) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000001, -639050810, -5288142338710158760, -32726, -35, 1750485995849.620117, -5947212091.364470, 1, 'binary.hQfgaAPxYdJOaIkPUQJT', 'nchar.甘肃省宁德县孝南北京街f座 436022', 1630000000092) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000001, -592064320, -9093756286157608735, -19004, -37, -60515553533903.898438, 4852.484564, 1, 'binary.DyLlxTKVpmVQoztjwPWn', 'nchar.辽宁省武汉县徐汇王路u座 155320', 1630000000092) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631365000000, -168837427, 5524670347645121945, -687, 105, -33530.533779, -857595946215.329956, 0, 'binary.dWHryceFSxVdkHDrNfYa', 'nchar.山东省博市六枝特银川路k座 264255', 1630000000091) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631380000000, -144505842, 9031683748543148709, 30839, 37, 75951623525931.296875, -356608030036.447021, 0, 'binary.YAmFEuHOeLIMgXCOGnRf', 'nchar.黑龙江省秀珍市白云广州街Z座 558462', 1630000000092) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, -202930007, 2539365322373297240, 30035, -11, 3987179954447.959961, 166663.706087, 0, 'binary.aQPbQkWFDKuugEWfofTC', 'nchar.陕西省建国市江北阜新街O座 931583', 1630000000092) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631379999999, 340191939, 190224560698109215, 7127, 125, 41587961603.659103, -37646578.644303, 1, 'binary.ZClsyheJcXtiVXWSTmJC', 'nchar.内蒙古自治区南京市吉区海门街X座 981848', 1630000000092) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, 1141439964, 5703107595817904511, 17318, 70, 20244282326280.398438, -374.976792, 1, 'binary.ECmfqstJJJGVfXRSlipY', 'nchar.山东省秀珍市长寿嘉禾街H座 668408', 1630000000092) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000001, -822942850, -5193915838519507313, -6726, -109, 12.862186, -308.656139, 1, 'binary.fiNIskGvJlLuWiySSzcR', 'nchar.辽宁省太原市合川沈路K座 765732', 1630000000093) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000001, -1084726805, -1799418590653312591, -316, -61, -43.289435, -5628.101135, 1, 'binary.nYWtrJedHHgNuftngNzv', 'nchar.福建省小红市沈北新王街j座 802142', 1630000000093) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631380000000, 2145419646, 1580239203992480528, -14574, 100, -94259.299402, 2214154.396490, 0, 'binary.GZbUYTSXnACDqnrhbNFC', 'nchar.青海省淑华市沙湾阜新路Q座 663492', 1630000000092) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631395000000, -752146752, 8221071117818639487, -24048, -125, -4922542596.502000, 69.541336, 0, 'binary.DpfvmRBpvyGTcVOJPyJo', 'nchar.江西省刚县房山安街D座 955283', 1630000000093) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, -878950287, -8422331468601695661, -13558, 43, 339844.201301, 9.764866, 0, 'binary.RefVNTHmgyVTPXRzLkLd', 'nchar.浙江省马鞍山县海陵王街B座 867693', 1630000000093) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631394999999, 1783522942, 6078410931639616068, 9495, 98, 727091818.991834, 8165776010706.599609, 1, 'binary.dtkNAQMwyepAJbdqweVh', 'nchar.河北省天津市沈河贵阳街N座 113675', 1630000000093) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, 1165805677, 1114429668955803199, 26742, 75, 965648.432436, 67940.285447, 1, 'binary.STtpcklCmoPcTqcnVEYp', 'nchar.湖北省荆门县静安拉萨街x座 767367', 1630000000093) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000001, -1314056119, -4268557454181639559, -17681, -70, 9569.520951, -3.709641, 1, 'binary.eAxSwIhCTWRsxHmnyhOO', 'nchar.四川省建华县沙市沈阳路f座 959538', 1630000000094) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000001, -1195931348, -3234878011308908109, -8224, -81, 68272155.434104, 2277301.750441, 1, 'binary.cCbIERRNYVmzqgdEkLyn', 'nchar.宁夏回族自治区长春市山亭海门路i座 441560', 1630000000094) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631395000000, 1044986483, 708621525043415947, 23174, 25, -87522059062286.000000, 91518.977374, 0, 'binary.KWZOCXHGUnswxAQvgLbX', 'nchar.甘肃省上海市大东合肥路v座 576201', 1630000000093) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631410000000, -883689952, -6467532462337897809, -20092, 56, 2802216728037.279785, -9.303156, 0, 'binary.qwPhaldJXqLHFGOeyrHw', 'nchar.甘肃省南宁市孝南彭街T座 566484', 1630000000094) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 972297407, -5877490557670044529, -27447, 44, 40.846479, -4231.348093, 0, 'binary.DwlvVHWtVzbpJAeCWIMr', 'nchar.山西省莹县魏都张街A座 689281', 1630000000094) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631409999999, 1032579814, 2390187777440316314, 19171, 78, -63005465989151.203125, 5961100651288.120117, 1, 'binary.VAqPpjxEFxcYSxnzUAwL', 'nchar.贵州省辽阳县静安安街c座 964229', 1630000000094) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, 241342224, 8497667456001609204, 18172, 104, -906790925488.797974, 36707840437.474998, 1, 'binary.FLLdUdBRMRgvzrMSflyN', 'nchar.甘肃省哈尔滨县翔安拉萨路L座 242629', 1630000000094) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000001, -1560976462, -1082805904642279032, -24035, -84, 5518722837.437610, 99553268986.260300, 1, 'binary.yXYqlIPBknzcrQaHQVBK', 'nchar.山西省福州县白云乌鲁木齐街I座 977466', 1630000000095) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000001, -1279270656, -127272614642938342, -18488, -105, 820203443079.204956, -39324.928647, 1, 'binary.vcWaeUuquQmpiNmaMFTJ', 'nchar.重庆市红市友好张家港街h座 853577', 1630000000095) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631410000000, -810546809, 4707087124402946180, 14228, -33, 401.288641, -573136.630366, 0, 'binary.oglpZSvAEAkKdUTqMhCy', 'nchar.安徽省大冶县锡山高路Q座 272097', 1630000000094) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631425000000, -1291692620, -3976582859582309880, 99, -107, -643212263540.171997, 91650599.279398, 0, 'binary.QtWoywiZBLMnNCtJddrC', 'nchar.海南省凤英市新城单路Y座 332919', 1630000000095) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, -1636741671, 6149617241982859450, -29331, -23, 53068143981197.398438, 507.769638, 0, 'binary.rwiKvwHMRitScXUVmJhW', 'nchar.陕西省辛集县清浦李路H座 631016', 1630000000095) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631424999999, 2070489093, 3407988531094502611, 31121, 105, -797.417546, -5114.871991, 1, 'binary.VKDzqKpnljjtWvPxhJtw', 'nchar.台湾省淑英县黄浦齐齐哈尔街c座 143348', 1630000000095) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, 1536720383, 3880153804085171184, 11796, 82, 45989.231323, 92.649541, 1, 'binary.tbEDrcIJyjZTjRieTqRC', 'nchar.上海市北镇市怀柔阜新路R座 405512', 1630000000095) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000001, -955300022, -7851370978801274222, -14072, -90, -91217.776216, -9476202.110944, 1, 'binary.wYLmyHTheIfPsDKWcbvP', 'nchar.河南省丹市金平潮州路n座 514612', 1630000000096) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000001, -1019361248, -57576815854175575, -24873, -4, -32984292266917.800781, 58.748899, 1, 'binary.HNDdWAeDLFDTfTKRiPvW', 'nchar.云南省桂芝市长寿王街C座 833966', 1630000000096) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631425000000, 1923224809, -2907800501658168815, -9896, 49, -33656345.811017, -671677543.932121, 0, 'binary.PJVocGHwGGXYBPOAxHSo', 'nchar.内蒙古自治区柳县崇文王街n座 759363', 1630000000095) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631440000000, 1147520901, -7715180528849786436, -17616, -76, 6.545288, -50974540.927085, 0, 'binary.vSYpvPIBkKueVfRtelNq', 'nchar.江西省亮市孝南合肥路t座 737452', 1630000000096) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, -113454687, -5823484358741527503, 28969, 121, -64687699.492563, -79961693.274913, 0, 'binary.eNlgFeimPdtilNzjMToG', 'nchar.辽宁省通辽县东丽廖路E座 551853', 1630000000096) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631439999999, 1457091374, 8334151643776472564, 2662, 82, 323215.440926, -9805209441096.250000, 1, 'binary.rMLCVjMtmxFxYRYbvQQh', 'nchar.安徽省武汉县城东萧路w座 417063', 1630000000096) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, 199139943, 2468266181580361696, 6282, 45, 938749.350559, -728777906039.301025, 1, 'binary.QCaxrlOInebbkEcnmLYI', 'nchar.天津市拉萨县魏都李街D座 325624', 1630000000096) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000001, -1074274106, -6116630471404653549, -13682, -70, 513137.327759, -588918896.561140, 1, 'binary.TyKIxNLJfeZgAKvWAFfB', 'nchar.海南省济南县闵行西宁路m座 451609', 1630000000097) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000001, -565301435, -4026730978460667946, -19020, -77, -8.983572, -43.480820, 1, 'binary.ydTJczfbrhMoGjzUmQfP', 'nchar.香港特别行政区柳州市璧山张家港街l座 596812', 1630000000097) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631440000000, 625692002, -7425751090292284243, 10228, 1, 750.678719, -90.988373, 0, 'binary.xzExJqxxmHYoAUDKciSV', 'nchar.湖北省婷县龙潭兰州街n座 199763', 1630000000096) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 2072736931, -3329284412762274677, 12268, -69, 5910.358156, 997044108049.300049, 0, 'binary.aBqZKnHzsSpmRSwSQPYY', 'nchar.山西省郑州市孝南王路e座 167062', 1630000000097) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 1463682034, 990280305241763342, -3488, 89, 419973.481918, -55750492618.939903, 0, 'binary.OybEvgqNWyaCiEEvxpeR', 'nchar.青海省荆门县孝南东莞路a座 305732', 1630000000097) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631454999999, 1438955591, 8811422557823062850, 15737, 98, 1754297.282564, 567752594.816600, 1, 'binary.OMjmDDXPGGJaZZRMlpqF', 'nchar.上海市台北市黄浦周街v座 378966', 1630000000097) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 1828878609, 7873820745339300844, 23138, 106, 9.263627, 946712871592.369995, 1, 'binary.wDtceAzvXHGjCqbAGulx', 'nchar.浙江省邯郸县长寿张路V座 106929', 1630000000097) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000001, -1388085018, -6219226483229634812, -21928, -116, -896801017371.463989, 91302.519671, 1, 'binary.YMYcdmnGDpGZSmUvXpYa', 'nchar.重庆市刚县江北北镇路o座 294056', 1630000000098) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000001, -1225032046, -2776099211554722026, -11329, -26, 22948.709678, -47848.985287, 1, 'binary.muQMoYipLCOlIkSyKxUk', 'nchar.山东省郑州市合川张家港街X座 960693', 1630000000098) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631455000000, 1161217978, 1842781937217393115, 17682, -46, 286163234.229566, -2.426507, 0, 'binary.CRKBkwUTzvgcBPDSNWiy', 'nchar.广西壮族自治区浩市安次苏街L座 929528', 1630000000097) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631470000000, -821482750, -6759948063516530731, -16375, 89, -84205.305774, 768242.497026, 0, 'binary.vMJVGyqMTgaGrLmmbZIE', 'nchar.天津市琴市江北孙街m座 262672', 1630000000098) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 717271992, 7986931673188494197, -9479, 52, 4711.839591, -5371732.797787, 0, 'binary.kPRwzfEKYpYgWRWaeGzP', 'nchar.重庆市乌鲁木齐市闵行南京路R座 883727', 1630000000098) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631469999999, 1077620215, 2558543784745208971, 25857, 52, -47179130892.244499, -20848774363.230000, 1, 'binary.BQGtKbxrxZCgjfTUjLJe', 'nchar.黑龙江省岩市海陵天津街v座 971806', 1630000000098) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 1977913093, 3861242852069723630, 20743, 59, 9464902993.309589, -5024.160475, 1, 'binary.lzQOJDAUyRQtQDjNrWZJ', 'nchar.甘肃省齐齐哈尔市华龙沈阳街I座 777477', 1630000000098) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000001, -428527206, -547566441621156192, -12808, -31, -74278620895119.906250, 50537841781.787697, 1, 'binary.FkYBhPfxSWPoKPiVwaNT', 'nchar.吉林省南宁市平山张路Z座 293046', 1630000000099) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000001, -1181912125, -427873392135772867, -13447, -55, 3457499617.219500, -7038680.198896, 1, 'binary.ozhZqPebRJOHopCsXWpU', 'nchar.上海市伟市闵行北京街i座 107708', 1630000000099) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631470000000, 744599275, 5384505142686943052, 30013, -71, -4.424027, -5.984967, 0, 'binary.RcgJmYOFSSkGHfTHlqvY', 'nchar.陕西省东县大东窦路E座 174935', 1630000000098) ;''') + tdSql.execute(f'''insert into stable_1_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double , q_bool , q_binary , q_nchar, q_ts) values(1631485000000, -1492370578, 4514526083062070789, 31730, -102, -717.383970, 52910.663266, 0, 'binary.jsHQYTHKPARRqiGzZKOz', 'nchar.澳门特别行政区邯郸县丰都宋街W座 598200', 1630000000099) ;''') + tdSql.execute(f'''insert into regular_table_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, 963480452, 5458394487863298148, -21434, 125, -1656.168120, 5768934065.605500, 0, 'binary.rCnbtltacJIWEVZWlXVF', 'nchar.湖北省合肥县龙潭马路o座 149144', 1630000000099) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631484999999, 224551324, 4933122153940335639, 2537, 23, -4225000.312926, -7035665314327.099609, 1, 'binary.RgTacOhlKMDWwPcrKsZK', 'nchar.山西省杭州市长寿沈阳街K座 225610', 1630000000099) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, 336564804, 2661017413987846119, 26066, 127, 2271.487346, 26543272352747.398438, 1, 'binary.FKgzwzTQzUHULzcaEmAd', 'nchar.贵州省成都市清河李路w座 620531', 1630000000099) ;''') + tdSql.execute(f'''insert into stable_1_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000001, -1922225927, -8001645480178976724, -25777, -111, -789464.882012, -360094093.372387, 1, 'binary.pxrkoGUZiddEAkYHiVjK', 'nchar.湖北省太原市永川吴路b座 333484', 1630000000100) ;''') + tdSql.execute(f'''insert into regular_table_2 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000001, -1213998634, -4431806460604559923, -22976, -105, -49517009982577.796875, -959039647940.217041, 1, 'binary.VxnbNGXEiRmRkEseJUFg', 'nchar.西藏自治区张家港县浔阳张路z座 876862', 1630000000100) ;''') + tdSql.execute(f'''insert into stable_2_1 (ts , q_int , q_bigint , q_smallint , q_tinyint , q_float , q_double, q_bool , q_binary , q_nchar, q_ts) values(1631485000000, -1591388398, 1970187840324683506, 20196, -59, 28171713.207903, 6.598821, 0, 'binary.fDSGiJNSRdOTovcESQFl', 'nchar.山西省梅市沙市王街r座 815669', 1630000000099) ;''') + tdSql.execute(f'''flush database table_lastrow_2_2;''') + + def basic_query(self): + # TD-23175 + tdSql.query(f'''select _wstart, LAST_ROW(q_float) from stable_2_1 where q_ts <= now +1h and _C0 <= now +1h and q_bool in (0 , 1) interval(11s) order by ts''') + tdSql.checkRows(100) + + # TD-23178 + tdSql.query(f'''select count(*) from (select LAST(q_tinyint) from stable_2 where q_ts between 1600000000000 and now +1h interval(12s) sliding(3s) order by ts)''') + tdSql.checkData(0, 0, 400) + tdSql.query(f'''select count(*) from (select LAST(q_tinyint) from stable_2 where q_ts between 1600000000000 and now +1h interval(12s) sliding(3s) order by ts)''') + tdSql.checkData(0, 0, 400) + + # TD-23173 + tdSql.query(f'''select _wstart,_wend,LAST_ROW(q_nchar) from stable_1 interval(17d) sliding(5d) order by _wstart;''') + tdSql.checkRows(7) + tdSql.checkData(0, 2, 'nchar.安徽省玲市西夏海口街o座 109427') + tdSql.checkData(1, 2, 'nchar.黑龙江省芳县海陵贺路c座 606589') + tdSql.checkData(2, 2, 'nchar.江苏省冬梅县六枝特张路g座 635526') + tdSql.checkData(3, 2, 'nchar.甘肃省宁德县孝南北京街f座 436022') + tdSql.checkData(4, 2, 'nchar.湖北省太原市永川吴路b座 333484') + tdSql.checkData(5, 2, 'nchar.湖北省太原市永川吴路b座 333484') + tdSql.checkData(6, 2, 'nchar.湖北省太原市永川吴路b座 333484') + + tdSql.query(f'''select _wstart,_wend,LAST_ROW(q_nchar) from stable_1 where q_ts between 1600000000000 and now +1h interval(17d) sliding(5d) order by _wstart;''') + tdSql.checkRows(7) + tdSql.checkData(0, 2, 'nchar.安徽省玲市西夏海口街o座 109427') + tdSql.checkData(1, 2, 'nchar.黑龙江省芳县海陵贺路c座 606589') + tdSql.checkData(2, 2, 'nchar.江苏省冬梅县六枝特张路g座 635526') + tdSql.checkData(3, 2, 'nchar.甘肃省宁德县孝南北京街f座 436022') + tdSql.checkData(4, 2, 'nchar.湖北省太原市永川吴路b座 333484') + tdSql.checkData(5, 2, 'nchar.湖北省太原市永川吴路b座 333484') + tdSql.checkData(6, 2, 'nchar.湖北省太原市永川吴路b座 333484') + + # TD-23222 + tdSql.query(f'''select LAST_ROW(q_tinyint) from stable_2 where q_ts between 1600000000000 and now +1h and _c0 <= now +1h and (q_nchar like 'nchar%' or q_binary = '0' or q_nchar = 'nchar_' ) and (q_bool = 0 or q_bool = 1 or q_bool is null) interval(15a,9a);''') + tdSql.checkRows(100) + + def run(self): + self.prepare_data() + self.basic_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index 91719aa21b0ce4245a096686aa1f5089e10ef542..c00e3b7d56af46f662ddb855a34e154e7c723b80 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -140,7 +140,7 @@ class TDTestCase: val2 = paraDict["ctbNum"] - 1 # select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 sqlStr = f"select count(*), t1, t2, t3, t4, t5, t6 from %s where t1 > %d and t1 < %d group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0"%(paraDict["stbName"], val1, val2) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) @@ -152,7 +152,7 @@ class TDTestCase: # select count(*), t3, t4 from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0 sqlStr = f"select count(*), t3, t4 from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 2 offset 0"%(paraDict["stbName"]) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(2) tdSql.checkData(0, 0, paraDict["rowsPerTbl"]) @@ -164,7 +164,7 @@ class TDTestCase: # select count(*) from $stb where t2 like '%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1 sqlStr = f"select count(*) from %s where t2 like '%%' and t1 > 2 and t1 < 5 group by t3, t4 order by t3 desc limit 1 offset 1"%(paraDict["stbName"]) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) @@ -179,18 +179,18 @@ class TDTestCase: tb = paraDict["ctbPrefix"] + '0' # select _wstart, max(c1) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1) limit 10 offset 1 sqlStr = f"select _wstart, max(c1) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1) limit 10 offset 1"%(tb, ts0, tsu) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) - tdSql.checkData(0, 0, "18-09-17 09:05:00.000") + tdSql.checkData(0, 0, "2018-09-17 09:05:00.000") tdSql.checkData(0, 1, -1) tdSql.checkData(1, 1, 1) - tdSql.checkData(9, 0, "18-09-17 09:50:00.000") + tdSql.checkData(9, 0, "2018-09-17 09:50:00.000") tdSql.checkData(9, 1, 5) - + tb5 = paraDict["ctbPrefix"] + '5' sqlStr = f"select max(c1), min(c2) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2) limit 10 offset 1"%(tb5, ts0, tsu) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, -1) @@ -206,22 +206,22 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb, ts0, tsu, limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 0) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 8200" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(8200) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100000;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 8190;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, 5) @@ -231,7 +231,7 @@ class TDTestCase: sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10 offset 10001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 0, -1000) @@ -240,13 +240,13 @@ class TDTestCase: tdSql.checkData(3, 0, 2) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 10000 offset 10001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(9998) sqlStr = f"select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000) limit 100 offset 20001;" - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(0) @@ -254,7 +254,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(linear) limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 0) @@ -272,7 +272,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d interval(5m) fill(prev) limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) @@ -280,7 +280,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit / 2 + 10 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(limit) tdSql.checkData(0, 1, 5) @@ -298,7 +298,7 @@ class TDTestCase: limit = paraDict["rowsPerTbl"] offset = limit * 2 - 11 sqlStr = f"select _wstart,max(c1), min(c2), sum(c3), avg(c4), stddev(c5), spread(c6), first(c7), last(c8), first(c9) from %s where ts >= %d and ts <= %d and c1 = 5 interval(5m) fill(value, -1, -2 ,-3, -4 , -5, -6 ,-7 ,'-8', '-9') limit %d offset %d"%(tb,ts0,tsu,limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(10) tdSql.checkData(0, 1, -1) @@ -314,7 +314,7 @@ class TDTestCase: ### [TBASE-350] ## stb + interval + fill + group by + limit offset sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') limit 2 offset 10" - print("====sql:%s"%(sqlStr)) + tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(2) @@ -322,7 +322,7 @@ class TDTestCase: offset = paraDict["rowsPerTbl"] * 2 offset = offset - 2 sqlStr = f"select max(c1), min(c2), sum(c3), avg(c4), first(c7), last(c8), first(c9) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 partition by t1 interval(5m) fill(value, -1, -2, -3, -4 ,-7 ,'-8', '-9') order by t1 limit %d offset %d"%(limit, offset) - print("====sql:%s"%(sqlStr)) + # tdLog.info("====sql:%s"%(sqlStr)) tdSql.query(sqlStr) tdSql.checkRows(1) tdSql.checkData(0, 0, 9) @@ -339,7 +339,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() + # tdSql.prepare() self.prepareTestEnv() self.tmqCase1() diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index d659a682b10ea9876595c0680d02295774967d15..798d0e290b8bf88ac6c9a6230651d6c44e17c68b 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -716,8 +716,6 @@ class TDTestCase: tdSql.checkRows(2) # partition by col - tdSql.query(f"select c1 , mavg(c1,3) from {dbname}.stb1 partition by c1") - tdSql.checkRows(0) tdSql.query(f"select c1 , mavg(c1,1) from {dbname}.stb1 partition by c1") tdSql.checkRows(40) tdSql.query(f"select c1, c2, c3, c4, mavg(c1,3) from {dbname}.stb1 partition by tbname ") diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index b8da02b9a6da6bfadc93d301d135464a1c7566f1..ba6ab53fc7c2503bfcdcd1b79474dc22f2717211 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -20,8 +20,8 @@ class TDTestCase: intData = [] floatData = [] tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(t0 tinyint, t1 float, loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags(5, 5.5, 'beijing')") for i in range(self.rowNum): tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) @@ -55,13 +55,20 @@ class TDTestCase: tdSql.checkData(0, 1, np.max(intData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb_1") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) + # check tags + tdSql.query(f"select max(t0) from {dbname}.stb") + tdSql.checkData(0,0,5) + + tdSql.query(f"select max(t1) from {dbname}.stb") + tdSql.checkData(0,0,5.5) + def max_check_ntb_base(self, dbname="db"): tdSql.prepare() intData = [] diff --git a/tests/system-test/2-query/mode.py b/tests/system-test/2-query/mode.py index 72bae552618f7e772481035c079a2d9507045d79..8321ef167585ac31d7d8086bfa24ccb34926ecb8 100644 --- a/tests/system-test/2-query/mode.py +++ b/tests/system-test/2-query/mode.py @@ -53,6 +53,7 @@ class TDTestCase: ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:32.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' @@ -64,61 +65,61 @@ class TDTestCase: tdSql.query(f"select ts,mode(c1) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 9) tdSql.query(f"select ts,mode(c2) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -99999999999999999) tdSql.query(f"select ts,mode(c3) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -999) tdSql.query(f"select ts,mode(c4) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -99) tdSql.query(f"select ts,mode(c5) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -9.99) tdSql.query(f"select ts,mode(c6) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -1e+21) tdSql.query(f"select ts,mode(c7) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, True) tdSql.query(f"select ts,mode(c8) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 'binary9') tdSql.query(f"select ts,mode(c9) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 'nchar9') tdSql.query(f"select ts,c3,c5,c8,mode(c1) from {dbname}.tb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2022, 12, 31, 1, 1, 36)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2022, 12, 31, 1, 1, 36).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -999) tdSql.checkData(0, 2, -9.99) tdSql.checkData(0, 3, 'binary9') @@ -128,61 +129,61 @@ class TDTestCase: tdSql.query(f"select ts,mode(c1) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 9) tdSql.query(f"select ts,mode(c2) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -99999) tdSql.query(f"select ts,mode(c3) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -999) tdSql.query(f"select ts,mode(c4) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 2)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 2).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -99) tdSql.query(f"select ts,mode(c5) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -9.99) tdSql.query(f"select ts,mode(c6) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, -99.99) tdSql.query(f"select ts,mode(c7) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, True) tdSql.query(f"select ts,mode(c8) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 'binary9') tdSql.query(f"select ts,mode(c9) from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 'nchar9') tdSql.query(f"select ts,mode(c1),c3,c5,c8 from {dbname}.stb") tdSql.checkRows(1) ts = tdSql.getData(0, 0) - tdSql.checkEqual(ts, datetime.datetime(2020, 12, 11, 0, 0, 3)) + tdSql.checkEqual(ts.astimezone(datetime.timezone.utc), datetime.datetime(2020, 12, 11, 0, 0, 3).astimezone(datetime.timezone.utc)) tdSql.checkData(0, 1, 9) tdSql.checkData(0, 2, -999) tdSql.checkData(0, 3, -9.99) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 6557aad05f7dc0d3021f1e0bbba8dfb08964ad0f..1b843defcef09ceb8bece595cae1b4b7611c33ae 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -6144,7 +6144,7 @@ class TDTestCase: startTime = time.time() - self.function_before_26() + #self.function_before_26() self.math_nest(['UNIQUE']) self.math_nest(['MODE']) @@ -6157,9 +6157,9 @@ class TDTestCase: # self.math_nest(['MAVG']) # self.math_nest(['HYPERLOGLOG']) # self.math_nest(['TAIL']) - # self.math_nest(['CSUM']) - # self.math_nest(['statecount','stateduration']) - # self.math_nest(['HISTOGRAM']) + self.math_nest(['CSUM']) + self.math_nest(['statecount','stateduration']) + self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) diff --git a/tests/system-test/2-query/nestedQuery_26.py b/tests/system-test/2-query/nestedQuery_26.py new file mode 100755 index 0000000000000000000000000000000000000000..9d5f31d1e067969f30f74d3f3de41236e346712e --- /dev/null +++ b/tests/system-test/2-query/nestedQuery_26.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .nestedQuery import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime = time.time() + + self.function_before_26() + + # self.math_nest(['UNIQUE']) + # self.math_nest(['MODE']) + # self.math_nest(['SAMPLE']) + + # self.math_nest(['ABS','SQRT']) + # self.math_nest(['SIN','COS','TAN','ASIN','ACOS','ATAN']) + # self.math_nest(['POW','LOG']) + # self.math_nest(['FLOOR','CEIL','ROUND']) + # self.math_nest(['MAVG']) + # self.math_nest(['HYPERLOGLOG']) + # self.math_nest(['TAIL']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) + + # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) + # self.str_nest(['LENGTH','CHAR_LENGTH']) + # self.str_nest(['SUBSTR']) + # self.str_nest(['CONCAT']) + # self.str_nest(['CONCAT_WS']) + # self.time_nest(['CAST']) #放到time里起来弄 + # self.time_nest(['CAST_1']) + # self.time_nest(['CAST_2']) + # self.time_nest(['CAST_3']) + # self.time_nest(['CAST_4']) + + + + # self.time_nest(['NOW','TODAY']) + # self.time_nest(['TIMEZONE']) + # self.time_nest(['TIMETRUNCATE']) + # self.time_nest(['TO_ISO8601']) + # self.time_nest(['TO_UNIXTIMESTAMP']) + # self.time_nest(['ELAPSED']) + #self.time_nest(['TIMEDIFF_1']) + #self.time_nest(['TIMEDIFF_2']) + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/nestedQuery_math.py b/tests/system-test/2-query/nestedQuery_math.py index 2d0bbcb352a4cfc732a2ccd4ec83604733146d32..3e37a2c15e6f2db8d657bc9dcbd001b1ec8c7aea 100755 --- a/tests/system-test/2-query/nestedQuery_math.py +++ b/tests/system-test/2-query/nestedQuery_math.py @@ -34,9 +34,9 @@ class TDTestCase(TDTestCase): self.math_nest(['MAVG']) self.math_nest(['HYPERLOGLOG']) self.math_nest(['TAIL']) - self.math_nest(['CSUM']) - self.math_nest(['statecount','stateduration']) - self.math_nest(['HISTOGRAM']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index f26f0abda0db5edb6c54261eb7e85f72daaafea2..a05e05704279bfa4a96df9a755da3dcd78126a38 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,7 +22,7 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - tdSql.checkData(0, 0, 271) + tdSql.checkData(0, 0, 277) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py index 148b89fc58ba481ef53d3735ce50a8957ec10278..8685b112068b5fbba72db009f6bd707a74dd6dbc 100644 --- a/tests/system-test/2-query/out_of_order.py +++ b/tests/system-test/2-query/out_of_order.py @@ -35,7 +35,10 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + taosdFileName = "taosd" + if platform.system().lower() == 'windows': + taosdFileName = "taosd.exe" + if (taosdFileName in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] @@ -52,7 +55,10 @@ class TDTestCase: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("%staosBenchmark -d %s -t %d -n %d -O %d -a %d -b float,double,nchar\(200\),binary\(50\) -T 50 -y " % (binPath,dbname,tables,per_table_num,order,replica)) + cmd = "%staosBenchmark -d %s -t %d -n %d -O %d -a %d -b float,double,nchar\(200\),binary\(50\) -T 50 -y " % (binPath,dbname,tables,per_table_num,order,replica) + if platform.system().lower() == 'windows': + cmd = "%staosBenchmark -d %s -t %d -n %d -O %d -a %d -b float,double,nchar(200),binary(50) -T 50 -y " % (binPath,dbname,tables,per_table_num,order,replica) + os.system(cmd) def sql_base(self,dbname): self.check_sub(dbname) @@ -162,19 +168,18 @@ class TDTestCase: sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname tdSql.query(sql) - num = tdSql.getData(0,0) + # 目前不需要了 + # num = tdSql.getData(0,0) - for i in range(0,num): - sql1 = "select count(*) from %s.d%d" %(dbname,i) - tdSql.query(sql1) - sql1_result = tdSql.getData(0,0) - tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + # for i in range(0,num): + # sql1 = "select count(*) from %s.d%d" %(dbname,i) + # tdSql.query(sql1) + # sql1_result = tdSql.getData(0,0) + # tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + def check_out_of_order(self,dbname,tables,per_table_num,order,replica): self.run_benchmark(dbname,tables,per_table_num,order,replica) - print("sleep 10 seconds") - #time.sleep(10) - print("sleep 10 seconds finish") self.run_sql(dbname) @@ -182,7 +187,7 @@ class TDTestCase: startTime = time.time() #self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1) - self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1) + self.check_out_of_order('db1',random.randint(50,100),random.randint(10000,20000),random.randint(1,5),1) # self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index fec553436c04f6b1cca231dc917874de4a883f5f..9c4091d0dfb58c05484f32cfbdd27a55c4b20dbd 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -292,9 +292,9 @@ class TDTestCase: maxQnode=tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") minQnode=tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionQnode=tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionallQnode=tdSql.queryResult # tdSql.query("select * from information_schema.ins_qnodes;") @@ -306,10 +306,10 @@ class TDTestCase: tdSql.checkData(0, 0, "%s"%maxQnode) tdSql.query("select min(c1) from stb11;") tdSql.checkData(0, 0, "%s"%minQnode) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionVnode=tdSql.queryResult assert unionQnode == unionVnode - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionallVnode=tdSql.queryResult assert unionallQnode == unionallVnode @@ -340,9 +340,9 @@ class TDTestCase: assert maxQnode==tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") assert minQnode==tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionQnode==tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionallQnode==tdSql.queryResult # tdSql.query("select * from information_schema.ins_qnodes;") @@ -354,8 +354,8 @@ class TDTestCase: assert maxQnode==tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") assert minQnode==tdSql.getData(0,0) - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") # tdSql.execute("create qnode on dnode %s"%dnodeId) @@ -387,9 +387,9 @@ class TDTestCase: assert maxQnode==tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") assert minQnode==tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionQnode==tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionallQnode==tdSql.queryResult queryPolicy=1 @@ -412,9 +412,9 @@ class TDTestCase: assert maxQnode==tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") assert minQnode==tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionQnode==tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") assert unionallQnode==tdSql.queryResult # test case : queryPolicy = 2 @@ -443,8 +443,8 @@ class TDTestCase: tdSql.execute("use db1;") tdSql.error("select max(c1) from stb10;") tdSql.error("select min(c1) from stb11;") - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") tdSql.query("select max(c1) from stb10_0;") tdSql.query("select min(c1) from stb11_0;") @@ -464,9 +464,9 @@ class TDTestCase: maxQnode=tdSql.getData(0,0) tdSql.query("select min(c1) from stb11;") minQnode=tdSql.getData(0,0) - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionQnode=tdSql.queryResult - tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.query("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") unionallQnode=tdSql.queryResult # tdSql.query("select * from information_schema.ins_qnodes;") @@ -478,8 +478,8 @@ class TDTestCase: tdSql.error("select max(c1) from stb10;") tdSql.error("select min(c1) from stb11;") - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") - tdSql.error("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") + tdSql.error("select c0,c1 from(select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000) order by c0,c1;") # run case def run(self): diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index a09d6a2b856bca40f1df55673c7d8e555f1df0f6..a43c2e635eb771687a237cabdb61d1311f29a1de 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -171,6 +171,7 @@ class TDTestCase: if any(parm in condition.lower().strip() for parm in condition_exception): print(f"case in {line}: ", end='') + print(f"condition : {condition}: ", end='') return tdSql.error(self.sample_query_form( sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, table_expr=table_expr, condition=condition @@ -391,16 +392,6 @@ class TDTestCase: self.checksample(**case25) case26 = {"k": 1000} self.checksample(**case26) - case27 = { - "table_expr": f"{DBNAME}.stb1", - "condition": "group by tbname slimit 1 " - } - self.checksample(**case27) # with slimit - case28 = { - "table_expr": f"{DBNAME}.stb1", - "condition": "group by tbname slimit 1 soffset 1" - } - self.checksample(**case28) # with soffset pass diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index ec6309c71ad295eb504d8f97b493a062a044fed1..f96ed8a3ff3e6656c9b81e5f82efa1fed6d3bb36 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -24,7 +24,7 @@ class TDTestCase: tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, dbname="sml_db"): + def checkContent(self, dbname="sml_db"): simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg) @@ -102,7 +102,7 @@ class TDTestCase: def run(self): tdSql.prepare() - self.checkFileContent() + self.checkContent() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/timeerror.py b/tests/system-test/2-query/timeerror.py new file mode 100644 index 0000000000000000000000000000000000000000..f43dc91de6a7c9faeeb3f86d92214f309f02c3e3 --- /dev/null +++ b/tests/system-test/2-query/timeerror.py @@ -0,0 +1,69 @@ +import taos +import sys +from time import sleep +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + rows = 10 + + tdSql.execute("create database dbus PRECISION 'us' ") + tdSql.execute("create database dbns PRECISION 'ns' ") + tdSql.execute("create database dbms PRECISION 'ms' ") + dball = ["dbns","dbus","dbms"] + for i in range(len(dball)): + dbname = dball[i] + stb = f"{dbname}.stb1" + if i == 0 : + qts = 1678883666951061471 + qts1 = 1678883666951061471 + + qtime = "2023-03-15 20:34:26.951061471" + elif i == 1 : + qts = 1678883666951061 + qts1 = 1678883666951061 + + qtime = "2023-03-15 20:34:26.951061" + else: + qts = 1678883666951 + qts1 = 1678883666953 + + qtime = "2023-03-15 20:34:26.951" + + tdSql.execute(f"use {dbname}") + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + f'''create table if not exists {stb} + (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) + tags(location binary(64), type int, isused bool , family nchar(64))''' + ) + tdSql.execute(f"create table {dbname}.t1 using {stb} tags('beijing', 1, 1, 'nchar1')") + tdSql.execute(f"create table {dbname}.t2 using {stb} tags('shanghai', 2, 0, 'nchar2')") + tdSql.execute(f"create table {dbname}.t3 using {stb} tags('shanghai', 3, 0, 'nchar3')") + + tdLog.printNoPrefix("==========step2:insert data") + tdSql.execute( + f"insert into {dbname}.t3 values ({qts}, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdLog.printNoPrefix("==========step3:query timestamp type") + tdSql.query(f"select ts from {dbname}.t3 limit 1") + tdSql.checkData(0,0,qtime) + tdSql.checkData(0,0,qts) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 4f415550b8e762fd2bb88838f75865e88a77d5c4..0e2975cdba02fc7b5a6ffda879b56f712649b43f 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -211,15 +211,16 @@ class TDTestCase: tdSql.query(f"select model,state_changed,count(state_changed) FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT _wstart,model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs from {dbname}.diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model interval(10m)) partition BY model) where state_changed =1 partition BY model,state_changed ;") - sql=f"select model,ctc from (SELECT model,count(state_changed) as ctc FROM (SELECT model,diff(broken_down) AS state_changed FROM (SELECT model,cast(cast(floor(2*(sum(nzs)/count(nzs))) as bool) as int) AS broken_down FROM (SELECT ts,model, cast(cast(status as bool) as int) AS nzs from {dbname}.diagnostics WHERE ts >= 1451606400000 AND ts < 1451952001000 ) WHERE ts >= 1451606400000 AND ts < 1451952001000 partition BY model interval(10m)) partition BY model) WHERE state_changed = 1 partition BY model )where model is null;" - - # for i in range(2): - # tdSql.query("%s"%sql) - # quertR1=tdSql.queryResult - # for j in range(50): - # tdSql.query("%s"%sql) - # quertR2=tdSql.queryResult - # assert quertR1 == quertR2 , "%s != %s ,The results of multiple queries are different" %(quertR1,quertR2) + sql=f"SELECT model,count(state_changed) FROM (SELECT _rowts,model,diff(broken_down) AS state_changed FROM (SELECT ts,model,tb,cast(cast(floor(2*(nzs)) as bool) as int) AS broken_down FROM (SELECT _wstart as ts,model,tbname as tb, sum(cast(cast(status as bool) as int))/count(cast(cast(status as bool) as int)) AS nzs FROM {dbname}.diagnostics WHERE ts >= 1451606400000 AND ts < 1451952001000 partition BY tbname,model interval(10m))order by ts) partition BY tb,model ) WHERE state_changed = 1 partition BY model;" + tdSql.query(f"{sql}") + tdSql.checkRows(46) + for i in range(2): + tdSql.query("%s"%sql) + quertR1=tdSql.queryResult + for j in range(50): + tdSql.query("%s"%sql) + quertR2=tdSql.queryResult + assert quertR1 == quertR2 , "%s != %s ,The results of multiple queries are different" %(quertR1,quertR2) #it's already supported: diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index f21fae0d7b9a4908a6112d2f5ea7cc214bd961bf..61451f03b1927352f9d0a227925f33227458c12f 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -18,7 +18,10 @@ class MyDnodes(TDDnodes): def __init__(self ,dnodes_lists): super(MyDnodes,self).__init__() self.dnodes = dnodes_lists # dnode must be TDDnode instance - self.simDeployed = False + if platform.system().lower() == 'windows': + self.simDeployed = True + else: + self.simDeployed = False class TDTestCase: noConn = True @@ -29,7 +32,7 @@ class TDTestCase: self.master_dnode = self.TDDnodes.dnodes[0] self.host=self.master_dnode.cfgDict["fqdn"] conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - tdSql.init(conn1.cursor()) + tdSql.init(conn1.cursor(), True) def getBuildPath(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index 9dd3c568054948d7062c2f87f145149509b1b575..c4b30a16f71ba1331fd3f146aa3245375326588e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -19,7 +19,10 @@ class MyDnodes(TDDnodes): def __init__(self ,dnodes_lists): super(MyDnodes,self).__init__() self.dnodes = dnodes_lists # dnode must be TDDnode instance - self.simDeployed = False + if platform.system().lower() == 'windows': + self.simDeployed = True + else: + self.simDeployed = False class TDTestCase: @@ -111,6 +114,8 @@ class TDTestCase: dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + if platform.system().lower() == 'windows': + cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"" tdLog.debug(cmd) if os.system(cmd) != 0: raise Exception("failed to execute system command. cmd: %s" % cmd) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 69cf378da3da0b12e56242125c49bfe33afcb3a6..080b431ffe54f43e725e22d84a3f365f8bfc4cea 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -57,7 +57,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -149,6 +149,7 @@ class TDTestCase: topicFromStb = 'topic_stb_column' topicFromCtb = 'topic_ctb_column' + tdSql.execute("alter database %s wal_retention_period 3600" % (parameterDict['dbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName'])) diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index 39d519fec1be19009860408dd02d26630070af44..77d43149cd611225d2ee8f7b017013b0fc784bfb 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -44,6 +44,7 @@ class TDTestCase: def wrong_topic(self): tdSql.prepare() tdSql.execute('use db') + tdSql.execute('alter database db wal_retention_period 3600') stbname = f'db.{tdCom.getLongName(5, "letters")}' tag_dict = { 't0':'int' @@ -75,4 +76,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 808a4935e30ccec4d06863e54e5e2ff3e1ba635d..44f58279be2df9cc97ee504babebbbfda1793f44 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -67,6 +67,7 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 8386c22cd0430128a1b873f3f5150ef51704b245..0f4f1694c17701d9d4361c788980215ae418db5e 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -67,6 +67,7 @@ class TDTestCase: tdLog.info("flush db to let data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict['dbName'])) return def tmqCase1(self): diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index 04f5aac559b060e17b23618cea1007e41ffb858d..e0d1e2c5b6190ed28f9e8ad9f3e4c8b15d57bb51 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -60,7 +60,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") tdSql.query("drop database if exists %s "%(cdbName)) - tdSql.query("create database %s vgroups 1"%(cdbName)) + tdSql.query("create database %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index 4371a909c253b3aaf1f7f4d5fad931ee35665519..7d11684ed81fd79584253e032f67cae1bd833de8 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -32,34 +32,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -134,6 +106,7 @@ class TDTestCase: paraDict['ctbNum'] = self.ctbNum paraDict['rowsPerTbl'] = self.rowsPerTbl + tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' queryString = "select ts, c1, c2 from %s.%s where t4 == 'beijing' or t4 == 'changsha' "%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/raw_block_interface_test.py b/tests/system-test/7-tmq/raw_block_interface_test.py new file mode 100644 index 0000000000000000000000000000000000000000..1e89de1cce397e8c64e2a78bb6fb8614d011d47d --- /dev/null +++ b/tests/system-test/7-tmq/raw_block_interface_test.py @@ -0,0 +1,54 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def checkData(self): + tdSql.execute('use db_raw') + tdSql.query("select * from d1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 120) + + tdSql.query("select * from d2") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + return + + def check(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/write_raw_block_test'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkData() + + return + + def run(self): + tdSql.prepare() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 04224fbc7e86d177829f898855067986aae81746..95c1839964eeb20ac9aa1672713527d197c179b1 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -60,7 +60,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") tdSql.query("drop database if exists %s "%(cdbName)) - tdSql.query("create database %s vgroups 1"%(cdbName)) + tdSql.query("create database %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -115,7 +115,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 6b48a6d57076b4baf54d7c08db7cd8a625fab29c..3f862ae0472f6a1312ac5f2ea680adbd4b95f172 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -45,6 +45,7 @@ class TDTestCase: tmqCom.initConsumerTable() tmqCom.create_database(tsql=tdSql, dbName=paraDict["dbName"],dropFlag=paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=paraDict['replica']) + tdSql.execute("alter database %s wal_retention_period 3600"%(paraDict["dbName"])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index c4a7c8cae5551c6374d683f941148f8f054dd53c..7ee5fce5a81ffb50341a61497f5616b03040d998 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -106,6 +106,7 @@ class TDTestCase: # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + tdSql.execute("alter database dbt wal_retention_period 3600") tdLog.info("create topics from stb1") topicFromStb1 = 'topic_UpperCase_stb1' # queryString = "select ts, c1, c2 from %s.%s where t4 == 'shanghai' or t4 == 'changsha'"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index c380d201b2d8827c6e3d12f03a0168ed47af0362..71b7fdef5d6a2fc072b7a21ef4205ea7a651bce2 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index 0fa9bcfbd461c0498d301e7b702a2ee3f8dd8aa4..631813e7ab864d486411bbd437079181ba9711bb 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -13,11 +13,11 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - #rpcDebugFlagVal = '143' + # rpcDebugFlagVal = '143' #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} - #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + # updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) def init(self, conn, logSql, replicaVar=1): @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -180,7 +180,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -278,7 +278,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -345,7 +345,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) tdSql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(parameterDict['dbName'], parameterDict['stbName'])) tdLog.info("create topics from db") @@ -415,7 +415,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index 50ef52cb15a5336395154813619976c7933f2dc7..ed13fcbe06be1c82590752b7257528190b972192 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -180,7 +180,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -262,7 +262,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index c5ae44214a0cea58f2c4f3cfee03dd0c3e5918a2..605e862950799b81fe57451a94e62ba086fa0a47 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -99,7 +99,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -123,8 +123,9 @@ class TDTestCase: pre_insert = "insert into " sql = pre_insert - t = time.time() - startTs = int(round(t * 1000)) + # t = 1678609778776 #time.time() + t = 1600000000000 + startTs = t #int(round(t * 1000)) #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) for i in range(ctbNum): sql += " %s_%d values "%(stbName,i) @@ -179,8 +180,8 @@ class TDTestCase: parameterDict['cfg'] = cfgPath self.initConsumerTable() - tdLog.info("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdLog.info("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -196,7 +197,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() @@ -267,7 +268,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'], parameterDict['replica'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -283,7 +284,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d replica %d" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) + tdSql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600" %(parameterDict2['dbName'], parameterDict2['vgroups'], parameterDict2['replica'])) prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) prepareEnvThread2.start() diff --git a/tests/system-test/7-tmq/subscribeDb2.py b/tests/system-test/7-tmq/subscribeDb2.py index d045842e45f4f526e5916081774177b22847ef11..60cde8d7f0940a8427e7f76ae31f5c682e36ca57 100644 --- a/tests/system-test/7-tmq/subscribeDb2.py +++ b/tests/system-test/7-tmq/subscribeDb2.py @@ -53,7 +53,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -100,7 +100,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -185,7 +185,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -263,7 +263,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 747ea3ba86c56a4008014e044fad5de6f8536f32..f82f89f1b9f37ab77ccea3368ccd1d5c51d861e7 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) tdSql.query("drop table if exists %s.notifyinfo "%(cdbName)) @@ -122,7 +122,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -203,7 +203,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -280,7 +280,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index c14d3b27b11fa8e36bd216f23a2371018358aaf0..764362c70882e0851d3cd5eb80ac4f0096a71113 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -65,6 +65,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 3ff0b25ff6b4585a8abc4061e7e46abced77d15e..9dcbf5b351ff80023fc0eea9476d3d436364ff03 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -226,7 +226,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -307,7 +307,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 1463cad627a23dd106d6afb78a4ee274345a840f..06aa18c9b6c79a1ee11a4e236340023c08714137 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -228,7 +228,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -303,7 +303,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -315,7 +315,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -333,7 +333,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") @@ -386,7 +386,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -398,7 +398,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -416,7 +416,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != (expectrowcnt * (1 + 1/4)): + if totalConsumeRows < (expectrowcnt * (1 + 1/4)): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index edbe1bc3c6dbcbef26a9068c3d644db03d1d9d12..25d09c38e2060dc0c5ef213a0c36a4bde4738866 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -245,8 +245,8 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) + tdLog.info("act consume rows: %d, expect consume rows greater than or equal to: %d"%(totalConsumeRows, expectrowcnt/4)) + if totalConsumeRows < expectrowcnt/4: tdLog.exit("tmq consume rows error!") self.initConsumerInfoTable() @@ -267,8 +267,8 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.info("act consume rows: %d, expect consume rows greater than or equal to: %d"%(totalConsumeRows, expectrowcnt)) + if totalConsumeRows < expectrowcnt: tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicFromStb1) @@ -320,7 +320,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 6c3e122902cc907dacd8297d95f27c6289cad176..cdbc41a59351c0f67205244ffdae69f32bf59e6b 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 10 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -251,6 +251,7 @@ class TDTestCase: tdLog.info("start consume 1 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -271,6 +272,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -338,7 +340,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -380,6 +382,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -394,7 +397,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*2: + if totalConsumeRows < expectrowcnt*2: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 025f403282e46eab4893062069bcea820124203b..6f3230e68791c9c1806abf576936cd5878bd7b06 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -215,7 +215,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -233,7 +234,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -269,7 +270,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt-10000: + if totalConsumeRows < expectrowcnt-10000: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000)) tdLog.exit("tmq consume rows error!") @@ -328,7 +329,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -346,7 +348,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -415,7 +417,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -433,7 +436,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -445,7 +448,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -467,7 +470,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -502,7 +505,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -520,7 +524,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -532,7 +536,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -555,7 +559,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*(1/2+1/4): + if totalConsumeRows < expectrowcnt*(1/2+1/4): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4))) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index 6aa3da66a46b316c55b2586bfc4c82f96fc33a8c..ed467b09b32aa862eabd1ef22c1872aeedf8d7d0 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -114,7 +114,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return @@ -231,7 +231,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -305,7 +305,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index cdcdadbcbb396d1e7228bba7779549e86646871c..0740830696987a6009db020de14bdc768d56c69a 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -138,34 +138,6 @@ class TDTestCase: else: tdLog.exit("three mnodes is not ready in 10s ") - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -200,6 +172,7 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create topics from stb with filter") # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) @@ -211,7 +184,7 @@ class TDTestCase: # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 # because taosd switch, may be consume duplication data topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 @@ -251,11 +224,12 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0])) + if expectRowsList[0] > resultList[0]: tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + if expectRowsList[0] == resultList[0]: + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index a70678219f15ba94b7f9ceb1f81b9aa3881660d4..1a8b0693b86cae8d12eaeb11f3af91f587a23a6f 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -65,6 +65,7 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -175,6 +176,7 @@ class TDTestCase: queryStringList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" %(paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ntb") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index 41073d83ae13b5a3828bbc758d8d1d2c600f8ca6..5d0af636b206e5c13ca403dd35c70fabeb2601cd 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 04d0744ab595279ac834e62669cb100df113d253..cb5a40642aab7ba2053780b898fa498e2c8b49a3 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -5,6 +5,7 @@ import time import socket import os import threading +import math from util.log import * from util.sql import * @@ -21,34 +22,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -80,6 +53,7 @@ class TDTestCase: tdLog.info("insert data") tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create topics from stb with filter") queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[0], queryString) @@ -109,7 +83,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -135,7 +109,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -161,7 +135,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index b9dac62833ba45dc63fca65e1b6f642265b655dd..b4fec94dcc3a2a266a35400dc789ee13aa463caf 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -73,6 +45,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -109,7 +82,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -134,7 +107,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -159,7 +132,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 895da95e5d5d8280b9e420b2803681674dab5bc2..f63c70a4c68ba0d88149326f3cd15bab764b6165 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -10,7 +10,7 @@ ################################################################### # -*- coding: utf-8 -*- - +import math from asyncore import loop from collections import defaultdict import subprocess @@ -129,11 +129,16 @@ class TMQCom: def stopTmqSimProcess(self, processorName): psCmd = "unset LD_PRELOAD; ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName) + if platform.system().lower() == 'windows': + psCmd = "ps -ef|grep -w %s|grep -v grep | awk '{print $2}'"%(processorName) processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") onlyKillOnceWindows = 0 while(processID): if not platform.system().lower() == 'windows' or (onlyKillOnceWindows == 0 and platform.system().lower() == 'windows'): - killCmd = "unset LD_PRELOAD; kill -INT %s > /dev/null 2>&1" % processID + if platform.system().lower() == 'windows': + killCmd = "kill -INT %s > /dev/nul 2>&1" % processID + else: + killCmd = "unset LD_PRELOAD; kill -INT %s > /dev/null 2>&1" % processID os.system(killCmd) onlyKillOnceWindows = 1 time.sleep(0.2) @@ -462,18 +467,24 @@ class TMQCom: for i in range(0,skipRowsOfCons): consumeFile.readline() - lines = 0 while True: dst = queryFile.readline() src = consumeFile.readline() - lines += 1 - if dst: - if dst != src: - tdLog.info("src row: %s"%src) - tdLog.info("dst row: %s"%dst) - tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines)) - else: + dstSplit = dst.split(',') + srcSplit = src.split(',') + + if not dst or not src: break + if len(dstSplit) != len(srcSplit): + tdLog.exit("consumerId %d consume rows len is not match the rows by direct query,len(dstSplit):%d != len(srcSplit):%d, dst:%s, src:%s" + %(consumerId, len(dstSplit), len(srcSplit), dst, src)) + + for i in range(len(dstSplit)): + if srcSplit[i] != dstSplit[i]: + srcFloat = float(srcSplit[i]) + dstFloat = float(dstSplit[i]) + if not math.isclose(srcFloat, dstFloat, abs_tol=1e-9): + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) return def getResultFileByTaosShell(self, consumerId, queryString): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index f372a2b7428a77b02b8dcdb5c4c546756ef7e1f7..3ad1d097e11b85f04f1d0cc9b2cd75a9ea1bdb0a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index c7f95f6f41cbf72e8d64c3a9bd8218dfd7a1c6d5..fdd683d08daba5625017029a3363b379d2a206b1 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 26dacf514d4f66273d36f6ac3fe49ee17602747a..f05f0abeff3f3f6dabde18ac3d3543d9df032f4e 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index d6f100041b15c205e8294cca964353d880b9ecc4..75b49a34fc583217e5ee0bb81324abc8421ed31a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index 87832ac0ef2169ad2895928ceb20121ae17ecba9..26f7a9fb4d1300907807651a5e2fd9dd80579725 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -16,6 +16,8 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: + updatecfgDict = {"tsdbDebugFlag":135} + def __init__(self): self.vgroups = 4 self.ctbNum = 10 @@ -54,6 +56,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index 8ed4a6df973b57f7302d5a2c193debffbf7286a1..cca29c178db281b86d145b8623f72a2dd8df1366 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 6a03f0f75192dda0b8b8394b1733bdec02222bda..00d2491c97b0a178fbfd289cf44e2617107fdfe9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index c11159c6e551d60b90aab16bb7c90e4b26021ac2..3b1dbae443f0e8d6f71dc3c8127975790c23401a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index 37946d0c22dae353ac62cb8e564c1361115bf8f9..a799fa57195fe84065c7bb0b295dc7c0f14cdd09 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index 8ff01444f81c5b5bdb52c3885970faca5b779a1b..f0bedbb187b9987e8f367f1afed7b69fb4c41001 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -18,8 +18,8 @@ from tmqCommon import * class TDTestCase: def __init__(self): self.vgroups = 4 - self.ctbNum = 3000 - self.rowsPerTbl = 70 + self.ctbNum = 1000 + self.rowsPerTbl = 100 def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -112,15 +113,15 @@ class TDTestCase: # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") consumerId = 3 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp3, enable.auto.commit:true, auto.commit.interval.ms:100, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") @@ -131,8 +132,8 @@ class TDTestCase: resultList = tmqCom.selectConsumeResult(expectRows) actConsumeTotalRows = resultList[0] + resultList[1] - if not (totalRowsInserted == actConsumeTotalRows): - tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + tdLog.info("sum of two consume rows: %d should be greater than or equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + if not (totalRowsInserted <= actConsumeTotalRows): tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) @@ -188,7 +189,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + keyList = 'group.id:cgrp4, enable.auto.commit:true, auto.commit.interval.ms:100, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") @@ -216,9 +217,9 @@ class TDTestCase: actConsumeTotalRows = resultList[0] - if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted): - tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) - tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) + tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + if not (actConsumeTotalRows > 0 and actConsumeTotalRows <= totalRowsInserted): tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index da8ac6c57deaf99d9871134489b290381b570306..a63927dd8bdee0483cc943ececaac1855248ef57 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -18,7 +18,7 @@ from tmqCommon import * class TDTestCase: def __init__(self): self.vgroups = 4 - self.ctbNum = 10 + self.ctbNum = 4 self.rowsPerTbl = 10000 def init(self, conn, logSql, replicaVar=1): @@ -41,7 +41,7 @@ class TDTestCase: 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, - 'batchNum': 10, + 'batchNum': 1000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 3, 'showMsg': 1, @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -112,15 +113,15 @@ class TDTestCase: # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") consumerId = 3 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + keyList = 'group.id:cgrp3, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 - expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") @@ -130,9 +131,9 @@ class TDTestCase: expectRows = 2 resultList = tmqCom.selectConsumeResult(expectRows) actConsumeTotalRows = resultList[0] + resultList[1] - - if not (totalRowsInserted == actConsumeTotalRows): - tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + + tdLog.info("sum of two consume rows: %d should be greater than or equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + if not (totalRowsInserted <= actConsumeTotalRows): tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) @@ -188,7 +189,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + keyList = 'group.id:cgrp4, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") @@ -216,7 +217,7 @@ class TDTestCase: actConsumeTotalRows = resultList[0] - if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted): + if not (actConsumeTotalRows > 0 and actConsumeTotalRows <= totalRowsInserted): tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index b910caf420de5efd6fef315e220920599f946435..8fcc991d4e3904da04480c7bbc28cac49cf5c068 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -41,7 +41,7 @@ class TDTestCase: 'ctbStartIdx': 0, 'ctbNum': 10, 'rowsPerTbl': 10000, - 'batchNum': 10, + 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 3, 'showMsg': 1, @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -87,7 +88,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 25, + 'pollDelay': 10, 'showMsg': 1, 'showRow': 1, 'snapshot': 1} @@ -116,7 +117,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + keyList = 'group.id:cgrp3, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) consumerId = 4 @@ -131,8 +132,8 @@ class TDTestCase: resultList = tmqCom.selectConsumeResult(expectRows) actConsumeTotalRows = resultList[0] + resultList[1] + tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) if not (totalRowsInserted == actConsumeTotalRows): - tdLog.info("sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) @@ -188,7 +189,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + keyList = 'group.id:cgrp4, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") @@ -213,12 +214,11 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - actConsumeTotalRows = resultList[0] - if not (actConsumeTotalRows > 0 and actConsumeTotalRows < totalRowsInserted): - tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) - tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + tdLog.info("act consume rows: %d"%(actConsumeTotalRows)) + tdLog.info("and second consume rows should be between 0 and %d"%(totalRowsInserted)) + if not ((actConsumeTotalRows > 0) and (actConsumeTotalRows <= totalRowsInserted)): tdLog.exit("%d tmq consume rows error!"%consumerId) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index 02093a2d889f1f7138e29055145bc63318dd2873..d146dca4497b9f60d204ca067f3bcd299c204dae 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -73,6 +45,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index b09efdd1e6a67ac0312f45e4277302d177ca34a4..aa9c8d25d0364635b11ba6c867cad67f2085c08e 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -237,10 +238,10 @@ class TDTestCase: if self.snapshot == 0: consumerId = 2 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 3 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4)) topicList = topicFromStb1 ifcheckdata = 1 @@ -270,7 +271,7 @@ class TDTestCase: if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: - if totalConsumeRows != totalRowsFromQuery: + if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 1!") # tmqCom.checkFileContent(consumerId, queryString) @@ -323,7 +324,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 4 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 5 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) @@ -369,9 +370,10 @@ class TDTestCase: tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) if self.snapshot == 0: - if totalConsumeRows != expectrowcnt: + if (totalConsumeRows != expectrowcnt): tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: + tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt)) if not ((totalConsumeRows >= totalRowsFromQuery) and (totalConsumeRows <= expectrowcnt)): tdLog.exit("tmq consume rows error with snapshot = 1!") @@ -381,9 +383,115 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 3 end ...... ") + def tmqCase4(self): + tdLog.printNoPrefix("======== test case 4: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 5000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 15, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdSql.query("flush database %s"%(paraDict['dbName'])) + + tmqCom.initConsumerTable() + tdLog.info("create topics from stb1") + topicFromStb1 = 'topic_stb1' + queryString = "select ts, c1, c2 from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicFromStb1, queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + # paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + consumerId = 1 + if self.snapshot == 0: + consumerId = 4 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + elif self.snapshot == 1: + consumerId = 5 + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) + + topicList = topicFromStb1 + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + # del some data + rowsOfDelete = int(self.rowsPerTbl / 4 ) + paraDict["endTs"] = paraDict["startTs"] + rowsOfDelete - 1 + # pDeleteThread = self.asyncDeleteData(paraDict) + self.threadFunctionForDeletaData(paraDict) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + + # update to 1/4 rows and insert 3/4 new rows + paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl * 3 / 4) + # paraDict['rowsPerTbl'] = self.rowsPerTbl + # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"], + # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + pInsertThread.join() + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query(queryString) + totalRowsFromQuery = tdSql.getRows() + + tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) + + if self.snapshot == 0: + tdLog.info("consume rows should be %d"%(expectrowcnt)) + if (totalConsumeRows != expectrowcnt): + tdLog.exit("tmq consume rows error with snapshot = 0!") + elif self.snapshot == 1: + # If data writing has not started before consumer get snapshot, will consume 10000 from wal, and consumer 7500 from tsdb; + tdLog.info("consume rows should be %d, "%(expectrowcnt)) + if (totalConsumeRows != expectrowcnt): + tdLog.exit("tmq consume rows error with snapshot = 1!") + + # tmqCom.checkFileContent(consumerId, queryString) + + tdSql.query("drop topic %s"%topicFromStb1) + + tdLog.printNoPrefix("======== test case 4 end ...... ") + def run(self): - # tdSql.prepare() + tdSql.prepare() tdLog.printNoPrefix("=============================================") tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.snapshot = 0 @@ -409,6 +517,17 @@ class TDTestCase: self.prepareTestEnv() self.tmqCase3() + # tdLog.printNoPrefix("=============================================") + # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + # self.snapshot = 0 + # self.prepareTestEnv() + # self.tmqCase4() + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + self.prepareTestEnv() + self.tmqCase4() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index e59e3d6ecda6e13127eba3f087029ad281cc55b3..7a47cd6025ccffab89656caf017f21aa2005c1c6 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDnode.py b/tests/system-test/7-tmq/tmqDnode.py index e1d6d91e2d620fe6a1ca7b28a5762047eaaa9bed..b96ca24ac5ad372eb5e3b15fbbebc63adeb9f63e 100644 --- a/tests/system-test/7-tmq/tmqDnode.py +++ b/tests/system-test/7-tmq/tmqDnode.py @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index a44ff916e5a22c92e68d743333e4c4b921929111..648d629e5cc2e5cbcc8d60cf1599ba93c37d080a 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -136,7 +137,7 @@ class TDTestCase: tdLog.info("================= restart dnode ===========================") tdDnodes.stoptaosd(1) tdDnodes.starttaosd(1) - # time.sleep(3) + time.sleep(5) tdLog.info(" restart taosd end and wait to check consume result") expectRows = 1 @@ -145,7 +146,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - tdSql.query(queryString) + tdSql.query(queryString, None, 50) totalRowsFromQury = tdSql.getRows() tdLog.info("act consume rows: %d, act query rows: %d"%(totalConsumeRows, totalRowsFromQury)) @@ -186,6 +187,7 @@ class TDTestCase: tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") @@ -220,7 +222,7 @@ class TDTestCase: tdLog.info("================= restart dnode ===========================") tdDnodes.stoptaosd(1) tdDnodes.starttaosd(1) - # time.sleep(3) + time.sleep(5) tdLog.info("create some new child table and insert data ") paraDict["batchNum"] = 100 @@ -234,7 +236,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - tdSql.query(queryString) + tdSql.query(queryString, None, 50) totalRowsFromQuery = tdSql.getRows() tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, totalRowsFromQuery)) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 982cc0a6313f8482031290acd89ea9f556dd5be3..cff55a1239b37414d234185d2326f089c1bb8f41 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1,wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index 6c49fae299ac68aac99892acf5de3011df0ab1a0..7a9c1bbb8c808fe62e8fede76640ac78e19f2f30 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -57,6 +57,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -100,7 +101,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)): + if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows <= expectrowcnt)): tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") @@ -131,7 +132,7 @@ class TDTestCase: 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'endTs': 0, - 'pollDelay': 10, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -143,6 +144,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -193,7 +195,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)): + if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows <= expectrowcnt)): tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 3fc5a2fdc70b64ec95324c370625f5397287ae7d..0b9cb7e66a2d481f6162f1e5e92d8c1f68dcb300 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -57,6 +57,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") @@ -143,6 +144,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdLog.info("start create database....") tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("start create normal tables....") tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) tdLog.info("start insert data into normal tables....") diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index 3789632984c91e49358ea21124824138642162ea..a94747e574cf1dd5b10f875dcf6c6590a3532aca 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -64,6 +64,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (self.paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index c9e34136cc1e59af173205a8e0cb2796acb1a14a..587baf12aab02a3ae3e7d4d9783cafab6ac0ceca 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -54,6 +54,7 @@ class TDTestCase: # tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index a39bac8dd194360196ccad80c1352b6028474a0b..b2038f1644e70cd2a6f4f7c2eb979ac58ff9a27e 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -116,7 +116,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqModule.py b/tests/system-test/7-tmq/tmqModule.py index d6b4aff9380bb8f5337ce4b52088e105449a1d07..187152c9ce9a1fd6afba1a718cd82e7e81ae0e4f 100644 --- a/tests/system-test/7-tmq/tmqModule.py +++ b/tests/system-test/7-tmq/tmqModule.py @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/7-tmq/tmqMultiConsumer.py b/tests/system-test/7-tmq/tmqMultiConsumer.py new file mode 100644 index 0000000000000000000000000000000000000000..cc217e0c4c2c3b185a6e7cba656dbcf1e4740ac2 --- /dev/null +++ b/tests/system-test/7-tmq/tmqMultiConsumer.py @@ -0,0 +1,174 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math +# from tests.pytest.util.common import TDCom +# from tests.pytest.util.log import TDLog + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 32 + self.ctbNum = 100 + self.rowsPerTbl = 1000 + self.snapshot = 1 + self.replicaVar = 3 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tmqCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + # tdLog.info("restart taosd to ensure that the data falls into the disk") + # tdDnodes.stop(1) + # tdDnodes.start(1) + # tdSql.query("flush database %s"%(paraDict['dbName'])) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + paraDict['snapshot'] = self.snapshot + + topicNameList = ['topic1', 'topic2'] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s where (sin(c2) >= 0) and (c1 %% 4 == 0) and (ts >= %d) and (t4 like 'shanghai')"%(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9379) + queryString = "select ts, acos(c1), ceil(pow(c1,3)) from %s.%s "%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 4 + topicList = topicNameList[0] + ',' + topicNameList[0] + ',' + topicNameList[1] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + consumerId = 1 + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + # continue to insert new rows + paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl) + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + pInsertThread.join() + + expectRows = 2 + resultList = tmqCom.selectConsumeResult(expectRows) + actConsumeTotalRows = resultList[0] + resultList[1] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(actConsumeTotalRows, expectrowcnt)) + + if not ((expectrowcnt <= actConsumeTotalRows) or ((resultList[0] == 0) and (resultList[1] >= expectrowcnt)) or ((resultList[1] == 0) and (resultList[0] >= expectrowcnt))): + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt, actConsumeTotalRows)) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 31ddc1b0f831df5d6e10f4337536488ddeaa892e..e9234f6c7aea99cf339be76ed799b34d3ec44721 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -51,6 +51,7 @@ class TDTestCase: consumerIdList = [0, 1, 2, 3] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 85222a941bf466631feedd1ead0387ef5f984096..7f322dc2583201bce47f40a9c1883daabadf8e42 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -94,6 +94,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replica) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index 297429b495b4bbc42e6d53a58295ef306b5d6903..bee38ca8ee192c542ebc2c09718d4e4224904c1e 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -50,7 +50,7 @@ class TDTestCase: def create_udf_function(self): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -116,6 +88,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +136,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -199,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -226,7 +200,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -265,6 +239,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -309,7 +284,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -336,7 +311,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index 9c139b50de71902d743236c503d6cca12cb8296a..d3b64d2b2150521c1a3bc98fe41734390609084f 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -50,7 +50,7 @@ class TDTestCase: def create_udf_function(self): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -116,6 +88,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +136,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -199,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -226,7 +200,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -265,6 +239,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -309,7 +284,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -336,7 +311,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 8593fd4f1e5969d9eb8083a66200301a6524d8c1..5da1625cb125091dddc6ddbe3e635dc404352bbb 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -50,7 +50,7 @@ class TDTestCase: def create_udf_function(self): # create scalar functions - tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + tdSql.execute("create function udf1 as '%s' outputtype int;"%self.libudf1) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -116,6 +88,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -163,6 +136,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -199,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) @@ -227,7 +201,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -266,6 +240,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() # tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + # tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) # tdLog.info("create stb") # tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) # tdLog.info("create ctb") @@ -310,7 +285,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -337,7 +312,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 2fa6600fb99efa22035091da6d75ac00d6b55afa..8fdf7748a3425b1efe7966949a1098e5867b6ee2 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -207,7 +208,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 if self.snapshot == 0: - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/2)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1+ 1/2 + 1/2)) elif self.snapshot == 1: expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1)) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index daffff44c1cf0dda7c4ecf5ac2dfd3dadfdf5504..8b67f6f8252e5e3682284c5c2d5b4295308db26c 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -128,7 +129,7 @@ class TDTestCase: ifManualCommit = 1 keyList = 'group.id:cgrp1,\ enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ + auto.commit.interval.ms:200,\ auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -213,7 +214,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl consumerId = 1 if self.snapshot == 0: - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/2) * (1/2*3)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] *(2 + 1/2*1/2*2 + 1/2 + 1/4)) elif self.snapshot == 1: expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/2)) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index 6b8c10de271b28726639baad8648e2586c02f9a1..5a35c4f5ee84c950ee7a611e5be93fe13371b031 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index 3975013e747f09d8cc0548b5e32f9f01c63fb9a4..84617efae4d78cf57cf6d522a7b3fbf50321e5f2 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -55,6 +55,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index 4f595788daee975f26cd2e30c1a0fe973e35aa68..6a9bb0ae924d0c0a6366ae6ccb0a6f8a1cec6e0d 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -54,6 +54,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=self.replicaVar, wal_retention_size=-1, wal_retention_period=-1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index db06700284d6de78139aef2f2678e2eb75d379fd..aeb28063e47253b085fe664e3109dcb4855539c9 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -51,7 +51,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index 51934ccd4408cc002060049f8c32fb6706a41fd3..c8ddeca3c3426c22efe3c4ddbbc6ad5b5992a108 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -50,7 +50,7 @@ class TDTestCase: return cur def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" diff --git a/tests/system-test/99-TDcase/TD-15557.py b/tests/system-test/99-TDcase/TD-15557.py index 884c028a6567d62ce5141eb5ad16240830168f51..f9e676f4f89d9e356d9ede59793f2b8cc431039b 100644 --- a/tests/system-test/99-TDcase/TD-15557.py +++ b/tests/system-test/99-TDcase/TD-15557.py @@ -64,7 +64,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -141,7 +141,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -214,7 +214,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -298,7 +298,7 @@ class TDTestCase: 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index 5ea652b4fb6fe5b441ea4a946026115431ec24b2..cc4c3f3b39a6ff6259b62a18d39d91265d87ba13 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -52,7 +52,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -95,7 +95,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): - tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) pre_create = "create table" @@ -176,7 +176,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -238,7 +238,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() @@ -304,7 +304,7 @@ class TDTestCase: self.initConsumerTable() - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + tdSql.execute("create database if not exists %s vgroups %d wal_retention_period 3600" %(parameterDict['dbName'], parameterDict['vgroups'])) prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() diff --git a/tests/system-test/99-TDcase/TD-16025.py b/tests/system-test/99-TDcase/TD-16025.py index 8c9fa9319f8071db436e4f71a597dc7d5de8758e..3e25b8c8334013b5dbb56b77ead3473994cd8e9e 100644 --- a/tests/system-test/99-TDcase/TD-16025.py +++ b/tests/system-test/99-TDcase/TD-16025.py @@ -59,7 +59,7 @@ class TDTestCase: def initConsumerTable(self,cdbName='cdb'): tdLog.info("create consume database, and consume info table, and consume result table") - tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("create database if not exists %s vgroups 1 wal_retention_period 3600"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) @@ -110,7 +110,7 @@ class TDTestCase: if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) - tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tsql.execute("create database if not exists %s vgroups %d replica %d wal_retention_period 3600"%(dbName, vgroups, replica)) tdLog.debug("complete to create database %s"%(dbName)) return diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index f57fae752f8d2d07f730f791ce0b5a63eb7dc832..2e23002059c5d157b3e1f6edf2f21a5d291739bc 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -73,6 +45,7 @@ class TDTestCase: expectRowsList = [] tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) tdLog.info("create ctb") @@ -113,7 +86,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -139,7 +112,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -165,7 +138,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 0f83468754783ee5adca78113c7a8df5c78bbe99..5f68a5b7389d3ea095a8886e2801dd82fb6af8d9 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -53,6 +53,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -97,6 +98,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -181,6 +183,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("create ctb") @@ -267,6 +270,7 @@ class TDTestCase: tmqCom.initConsumerTable() tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) tdLog.info("insert data by auto create ctb") diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index 2862f4a78d5391098f0493afa35e482c5a032817..6956e88aec7d0c8adb84c8af9aff31cb6d80fa1f 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -65,6 +65,7 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + tdSql.execute("alter database %s wal_retention_period 3600" % (paraDict['dbName'])) self.paraDict["stbName"] = 'stb1' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) diff --git a/tests/system-test/runAllOne.sh b/tests/system-test/runAllOne.sh new file mode 100644 index 0000000000000000000000000000000000000000..5a8d358d989355127f3971d1127aeb43fa0323bd --- /dev/null +++ b/tests/system-test/runAllOne.sh @@ -0,0 +1,688 @@ +# start -N3 +echo " ********** -N 3 *************" +python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 +python3 ./test.py -f 1-insert/alter_database.py -P +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -P +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -P +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -P +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -P +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -P +python3 ./test.py -f 1-insert/alter_stable.py -P +python3 ./test.py -f 1-insert/alter_table.py -P +python3 ./test.py -f 1-insert/boundary.py -P +python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -P +python3 ./test.py -f 1-insert/table_comment.py -P +python3 ./test.py -f 1-insert/time_range_wise.py -P +python3 ./test.py -f 1-insert/block_wise.py -P +python3 ./test.py -f 1-insert/create_retentions.py -P +python3 ./test.py -f 1-insert/mutil_stage.py -P +python3 ./test.py -f 1-insert/table_param_ttl.py -P +python3 ./test.py -f 1-insert/table_param_ttl.py -P -R +python3 ./test.py -f 1-insert/update_data_muti_rows.py -P +python3 ./test.py -f 1-insert/db_tb_name_check.py -P +python3 ./test.py -f 1-insert/InsertFuturets.py -P +python3 ./test.py -f 1-insert/insert_wide_column.py -P +python3 ./test.py -f 2-query/nestedQuery.py -P +python3 ./test.py -f 2-query/nestedQuery_str.py -P +python3 ./test.py -f 2-query/nestedQuery_math.py -P +python3 ./test.py -f 2-query/nestedQuery_time.py -P +python3 ./test.py -f 2-query/nestedQuery_26.py -P +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py -P +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 3 +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 4 +python3 ./test.py -f 7-tmq/tmqShow.py -P +python3 ./test.py -f 7-tmq/tmqDropStb.py -P +python3 ./test.py -f 7-tmq/subscribeStb0.py -P +python3 ./test.py -f 7-tmq/subscribeStb1.py -P +python3 ./test.py -f 7-tmq/subscribeStb2.py -P +python3 ./test.py -f 7-tmq/subscribeStb3.py -P +python3 ./test.py -f 7-tmq/subscribeDb0.py -P -N 3 -n 3 +python3 ./test.py -f 1-insert/delete_stable.py -P +python3 ./test.py -f 2-query/out_of_order.py -P -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -P +python3 ./test.py -f 2-query/insert_null_none.py -P +python3 ./test.py -f 2-query/insert_null_none.py -P -R +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 2 +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 3 +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 4 +python3 ./test.py -f 1-insert/database_pre_suf.py -P +python3 ./test.py -f 2-query/concat.py -P -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -P -Q 2 +python3 ./test.py -f 2-query/out_of_order.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 3 +python3 ./test.py -f 7-tmq/create_wrong_topic.py -P +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -P -N 3 +python3 ./test.py -f 7-tmq/basic5.py -P +python3 ./test.py -f 7-tmq/subscribeDb.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeDb1.py -P +python3 ./test.py -f 7-tmq/subscribeDb2.py -P +python3 ./test.py -f 7-tmq/subscribeDb3.py -P +python3 ./test.py -f 7-tmq/subscribeDb4.py -P +python3 ./test.py -f 7-tmq/subscribeStb.py -P +python3 ./test.py -f 7-tmq/subscribeStb4.py -P +python3 ./test.py -f 7-tmq/db.py -P +python3 ./test.py -f 7-tmq/tmqError.py -P +python3 ./test.py -f 7-tmq/schema.py -P +python3 ./test.py -f 7-tmq/stbFilter.py -P +python3 ./test.py -f 7-tmq/tmqCheckData.py -P +python3 ./test.py -f 7-tmq/tmqCheckData1.py -P +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -P +python3 ./test.py -f 7-tmq/tmqAlterSchema.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -P +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -P +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -P +python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -P +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -P +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -P +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -P +python3 ./test.py -f 7-tmq/tmq_taosx.py -P +python3 ./test.py -f 7-tmq/raw_block_interface_test.py -P +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -P +python3 ./test.py -f 99-TDcase/TD-19201.py -P +python3 ./test.py -f 99-TDcase/TD-21561.py -P +python3 ./test.py -f 0-others/taosShell.py -P +python3 ./test.py -f 0-others/taosShellError.py -P +python3 ./test.py -f 0-others/taosShellNetChk.py -P +python3 ./test.py -f 0-others/telemetry.py -P +python3 ./test.py -f 0-others/backquote_check.py -P +python3 ./test.py -f 0-others/taosdMonitor.py -P +python3 ./test.py -f 0-others/udfTest.py -P +python3 ./test.py -f 0-others/udf_create.py -P +python3 ./test.py -f 0-others/udf_restart_taosd.py -P +python3 ./test.py -f 0-others/udf_cfg1.py -P +python3 ./test.py -f 0-others/udf_cfg2.py -P +python3 ./test.py -f 0-others/cachemodel.py -P +python3 ./test.py -f 0-others/sysinfo.py -P +python3 ./test.py -f 0-others/user_control.py -P +python3 ./test.py -f 0-others/user_manage.py -P +python3 ./test.py -f 0-others/fsync.py -P +python3 ./test.py -f 0-others/tag_index_basic.py -P +python3 ./test.py -f 0-others/show.py -P +python3 ./test.py -f 0-others/information_schema.py -P +python3 ./test.py -f 2-query/abs.py -P +python3 ./test.py -f 2-query/abs.py -P -R +python3 ./test.py -f 2-query/and_or_for_byte.py -P +python3 ./test.py -f 2-query/and_or_for_byte.py -P -R +python3 ./test.py -f 2-query/apercentile.py -P +python3 ./test.py -f 2-query/apercentile.py -P -R +python3 ./test.py -f 2-query/arccos.py -P +python3 ./test.py -f 2-query/arccos.py -P -R +python3 ./test.py -f 2-query/arcsin.py -P +python3 ./test.py -f 2-query/arcsin.py -P -R +python3 ./test.py -f 2-query/arctan.py -P +python3 ./test.py -f 2-query/arctan.py -P -R +python3 ./test.py -f 2-query/avg.py -P +python3 ./test.py -f 2-query/avg.py -P -R +python3 ./test.py -f 2-query/between.py -P +python3 ./test.py -f 2-query/between.py -P -R +python3 ./test.py -f 2-query/bottom.py -P +python3 ./test.py -f 2-query/bottom.py -P -R +python3 ./test.py -f 2-query/cast.py -P +python3 ./test.py -f 2-query/cast.py -P -R +python3 ./test.py -f 2-query/ceil.py -P +python3 ./test.py -f 2-query/ceil.py -P -R +python3 ./test.py -f 2-query/char_length.py -P +python3 ./test.py -f 2-query/char_length.py -P -R +python3 ./test.py -f 2-query/check_tsdb.py -P +python3 ./test.py -f 2-query/check_tsdb.py -P -R +python3 ./test.py -f 2-query/concat.py -P +python3 ./test.py -f 2-query/concat.py -P -R +python3 ./test.py -f 2-query/concat_ws.py -P +python3 ./test.py -f 2-query/concat_ws.py -P -R +python3 ./test.py -f 2-query/concat_ws2.py -P +python3 ./test.py -f 2-query/concat_ws2.py -P -R +python3 ./test.py -f 2-query/cos.py -P +python3 ./test.py -f 2-query/cos.py -P -R +python3 ./test.py -f 2-query/count_partition.py -P +python3 ./test.py -f 2-query/count_partition.py -P -R +python3 ./test.py -f 2-query/count.py -P +python3 ./test.py -f 2-query/count.py -P -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -R +python3 ./test.py -f 2-query/db.py -P +python3 ./test.py -f 2-query/diff.py -P +python3 ./test.py -f 2-query/diff.py -P -R +python3 ./test.py -f 2-query/distinct.py -P +python3 ./test.py -f 2-query/distinct.py -P -R +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -R +python3 ./test.py -f 2-query/distribute_agg_avg.py -P +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -R +python3 ./test.py -f 2-query/distribute_agg_count.py -P +python3 ./test.py -f 2-query/distribute_agg_count.py -P -R +python3 ./test.py -f 2-query/distribute_agg_max.py -P +python3 ./test.py -f 2-query/distribute_agg_max.py -P -R +python3 ./test.py -f 2-query/distribute_agg_min.py -P +python3 ./test.py -f 2-query/distribute_agg_min.py -P -R +python3 ./test.py -f 2-query/distribute_agg_spread.py -P +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -R +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -R +python3 ./test.py -f 2-query/distribute_agg_sum.py -P +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -R +python3 ./test.py -f 2-query/explain.py -P +python3 ./test.py -f 2-query/explain.py -P -R +python3 ./test.py -f 2-query/first.py -P +python3 ./test.py -f 2-query/first.py -P -R +python3 ./test.py -f 2-query/floor.py -P +python3 ./test.py -f 2-query/floor.py -P -R +python3 ./test.py -f 2-query/function_null.py -P +python3 ./test.py -f 2-query/function_null.py -P -R +python3 ./test.py -f 2-query/function_stateduration.py -P +python3 ./test.py -f 2-query/function_stateduration.py -P -R +python3 ./test.py -f 2-query/histogram.py -P +python3 ./test.py -f 2-query/histogram.py -P -R +python3 ./test.py -f 2-query/hyperloglog.py -P +python3 ./test.py -f 2-query/hyperloglog.py -P -R +python3 ./test.py -f 2-query/interp.py -P +python3 ./test.py -f 2-query/interp.py -P -R +python3 ./test.py -f 2-query/irate.py -P +python3 ./test.py -f 2-query/irate.py -P -R +python3 ./test.py -f 2-query/join.py -P +python3 ./test.py -f 2-query/join.py -P -R +python3 ./test.py -f 2-query/last_row.py -P +python3 ./test.py -f 2-query/last_row.py -P -R +python3 ./test.py -f 2-query/last.py -P +python3 ./test.py -f 2-query/last.py -P -R +python3 ./test.py -f 2-query/leastsquares.py -P +python3 ./test.py -f 2-query/leastsquares.py -P -R +python3 ./test.py -f 2-query/length.py -P +python3 ./test.py -f 2-query/length.py -P -R +python3 ./test.py -f 2-query/limit.py -P +python3 ./test.py -f 2-query/log.py -P +python3 ./test.py -f 2-query/log.py -P -R +python3 ./test.py -f 2-query/lower.py -P +python3 ./test.py -f 2-query/lower.py -P -R +python3 ./test.py -f 2-query/ltrim.py -P +python3 ./test.py -f 2-query/ltrim.py -P -R +python3 ./test.py -f 2-query/mavg.py -P +python3 ./test.py -f 2-query/mavg.py -P -R +python3 ./test.py -f 2-query/max_partition.py -P +python3 ./test.py -f 2-query/max_partition.py -P -R +python3 ./test.py -f 2-query/max_min_last_interval.py -P +python3 ./test.py -f 2-query/last_row_interval.py -P +python3 ./test.py -f 2-query/max.py -P +python3 ./test.py -f 2-query/max.py -P -R +python3 ./test.py -f 2-query/min.py -P +python3 ./test.py -f 2-query/min.py -P -R +python3 ./test.py -f 2-query/mode.py -P +python3 ./test.py -f 2-query/mode.py -P -R +python3 ./test.py -f 2-query/Now.py -P +python3 ./test.py -f 2-query/Now.py -P -R +python3 ./test.py -f 2-query/percentile.py -P +python3 ./test.py -f 2-query/percentile.py -P -R +python3 ./test.py -f 2-query/pow.py -P +python3 ./test.py -f 2-query/pow.py -P -R +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -R +python3 ./test.py -f 2-query/round.py -P +python3 ./test.py -f 2-query/round.py -P -R +python3 ./test.py -f 2-query/rtrim.py -P +python3 ./test.py -f 2-query/rtrim.py -P -R +python3 ./test.py -f 1-insert/drop.py -P -N 3 -M 3 -i False -n 3 +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -P -N 3 -n 3 +python3 ./test.py -f 2-query/db.py -P -N 3 -n 3 -R +python3 ./test.py -f 2-query/stablity.py -P +python3 ./test.py -f 2-query/stablity_1.py -P +python3 ./test.py -f 2-query/elapsed.py -P +python3 ./test.py -f 2-query/csum.py -P +python3 ./test.py -f 2-query/function_diff.py -P +python3 ./test.py -f 2-query/tagFilter.py -P +python3 ./test.py -f 2-query/projectionDesc.py -P +python3 ./test.py -f 2-query/queryQnode.py -P +python3 ./test.py -f 6-cluster/5dnode1mnode.py -P + + +# -N 4 +echo " ********** -N 4 *************" +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -P -N 4 -M 1 +python3 ./test.py -f 2-query/varchar.py -P -R +python3 ./test.py -f 2-query/case_when.py -P +python3 ./test.py -f 2-query/case_when.py -P -R +python3 ./test.py -f 2-query/blockSMA.py -P +python3 ./test.py -f 2-query/blockSMA.py -P -R +python3 ./test.py -f 2-query/projectionDesc.py -P +python3 ./test.py -f 2-query/projectionDesc.py -P -R +python3 ./test.py -f 1-insert/update_data.py -P +python3 ./test.py -f 1-insert/tb_100w_data_order.py -P +python3 ./test.py -f 1-insert/delete_childtable.py -P +python3 ./test.py -f 1-insert/delete_normaltable.py -P +python3 ./test.py -f 1-insert/keep_expired.py -P +python3 ./test.py -f 1-insert/drop.py -P +python3 ./test.py -f 2-query/join2.py -P +python3 ./test.py -f 2-query/union1.py -P +python3 ./test.py -f 2-query/concat2.py -P +python3 ./test.py -f 2-query/json_tag.py -P +python3 ./test.py -f 2-query/nestedQueryInterval.py -P + + +# -N 5 +echo " ********** -N 5 *************" +python3 ./test.py -f 6-cluster/5dnode2mnode.py -P -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3 +python3 ./test.py -f 0-others/taosdShell.py -P -N 5 -M 3 -Q 3 +python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -P -N 5 +python3 ./test.py -f 2-query/timezone.py -P +python3 ./test.py -f 2-query/timezone.py -P -R +python3 ./test.py -f 2-query/To_iso8601.py -P +python3 ./test.py -f 2-query/To_iso8601.py -P -R +python3 ./test.py -f 2-query/To_unixtimestamp.py -P +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -R +python3 ./test.py -f 2-query/Today.py -P +python3 ./test.py -f 2-query/Today.py -P -R +python3 ./test.py -f 2-query/top.py -P +python3 ./test.py -f 2-query/top.py -P -R +python3 ./test.py -f 2-query/tsbsQuery.py -P +python3 ./test.py -f 2-query/tsbsQuery.py -P -R +python3 ./test.py -f 2-query/ttl_comment.py -P +python3 ./test.py -f 2-query/ttl_comment.py -P -R +python3 ./test.py -f 2-query/twa.py -P +python3 ./test.py -f 2-query/twa.py -P -R +python3 ./test.py -f 2-query/union.py -P +python3 ./test.py -f 2-query/union.py -P -R +python3 ./test.py -f 2-query/unique.py -P +python3 ./test.py -f 2-query/unique.py -P -R +python3 ./test.py -f 2-query/upper.py -P +python3 ./test.py -f 2-query/upper.py -P -R +python3 ./test.py -f 2-query/varchar.py -P + +# -N6 +echo " ********** -N 6 *************" +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 0-others/balance_vgroups_r1.py -P -N 6 + +python3 ./test.py -f 2-query/sample.py -P +python3 ./test.py -f 2-query/sample.py -P -R +python3 ./test.py -f 2-query/sin.py -P +python3 ./test.py -f 2-query/sin.py -P -R +python3 ./test.py -f 2-query/smaTest.py -P +python3 ./test.py -f 2-query/smaTest.py -P -R +python3 ./test.py -f 2-query/sml.py -P +python3 ./test.py -f 2-query/sml.py -P -R +python3 ./test.py -f 2-query/spread.py -P +python3 ./test.py -f 2-query/spread.py -P -R +python3 ./test.py -f 2-query/sqrt.py -P +python3 ./test.py -f 2-query/sqrt.py -P -R +python3 ./test.py -f 2-query/statecount.py -P +python3 ./test.py -f 2-query/statecount.py -P -R +python3 ./test.py -f 2-query/stateduration.py -P +python3 ./test.py -f 2-query/stateduration.py -P -R +python3 ./test.py -f 2-query/substr.py -P +python3 ./test.py -f 2-query/substr.py -P -R +python3 ./test.py -f 2-query/sum.py -P +python3 ./test.py -f 2-query/sum.py -P -R +python3 ./test.py -f 2-query/tail.py -P +python3 ./test.py -f 2-query/tail.py -P -R +python3 ./test.py -f 2-query/tan.py -P +python3 ./test.py -f 2-query/tan.py -P -R +python3 ./test.py -f 2-query/Timediff.py -P +python3 ./test.py -f 2-query/Timediff.py -P -R +python3 ./test.py -f 2-query/timetruncate.py -P +python3 ./test.py -f 2-query/timetruncate.py -P -R + +# N-7 +echo " ********** -N 7 *************" +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -P -N 7 -M 3 -C 6 -n 3 + +python3 ./test.py -f 2-query/between.py -P -Q 2 +python3 ./test.py -f 2-query/distinct.py -P -Q 2 +python3 ./test.py -f 2-query/varchar.py -P -Q 2 +python3 ./test.py -f 2-query/ltrim.py -P -Q 2 +python3 ./test.py -f 2-query/rtrim.py -P -Q 2 +python3 ./test.py -f 2-query/length.py -P -Q 2 +python3 ./test.py -f 2-query/char_length.py -P -Q 2 +python3 ./test.py -f 2-query/upper.py -P -Q 2 +python3 ./test.py -f 2-query/lower.py -P -Q 2 +python3 ./test.py -f 2-query/join.py -P -Q 2 +python3 ./test.py -f 2-query/join2.py -P -Q 2 +python3 ./test.py -f 2-query/cast.py -P -Q 2 +python3 ./test.py -f 2-query/substr.py -P -Q 2 +python3 ./test.py -f 2-query/union.py -P -Q 2 +python3 ./test.py -f 2-query/union1.py -P -Q 2 +python3 ./test.py -f 2-query/concat.py -P -Q 2 +python3 ./test.py -f 2-query/concat2.py -P -Q 2 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 2 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 2 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 2 +python3 ./test.py -f 2-query/spread.py -P -Q 2 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 2 +python3 ./test.py -f 2-query/explain.py -P -Q 2 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 2 +python3 ./test.py -f 2-query/timezone.py -P -Q 2 +python3 ./test.py -f 2-query/Now.py -P -Q 2 +python3 ./test.py -f 2-query/Today.py -P -Q 2 +python3 ./test.py -f 2-query/max.py -P -Q 2 +python3 ./test.py -f 2-query/min.py -P -Q 2 +python3 ./test.py -f 2-query/mode.py -P -Q 2 +python3 ./test.py -f 2-query/count.py -P -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 2 +python3 ./test.py -f 2-query/last.py -P -Q 2 +python3 ./test.py -f 2-query/first.py -P -Q 2 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 2 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 2 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 2 +python3 ./test.py -f 2-query/diff.py -P -Q 2 +python3 ./test.py -f 2-query/Timediff.py -P -Q 2 +python3 ./test.py -f 2-query/json_tag.py -P -Q 2 +python3 ./test.py -f 2-query/top.py -P -Q 2 +python3 ./test.py -f 2-query/bottom.py -P -Q 2 +python3 ./test.py -f 2-query/percentile.py -P -Q 2 +python3 ./test.py -f 2-query/apercentile.py -P -Q 2 +python3 ./test.py -f 2-query/abs.py -P -Q 2 +python3 ./test.py -f 2-query/ceil.py -P -Q 2 +python3 ./test.py -f 2-query/floor.py -P -Q 2 +python3 ./test.py -f 2-query/round.py -P -Q 2 +python3 ./test.py -f 2-query/log.py -P -Q 2 +python3 ./test.py -f 2-query/pow.py -P -Q 2 +python3 ./test.py -f 2-query/sqrt.py -P -Q 2 +python3 ./test.py -f 2-query/sin.py -P -Q 2 +python3 ./test.py -f 2-query/cos.py -P -Q 2 +python3 ./test.py -f 2-query/tan.py -P -Q 2 +python3 ./test.py -f 2-query/arcsin.py -P -Q 2 +python3 ./test.py -f 2-query/arccos.py -P -Q 2 +python3 ./test.py -f 2-query/arctan.py -P -Q 2 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 2 +python3 ./test.py -f 2-query/interp.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 2 +python3 ./test.py -f 2-query/stablity.py -P -Q 2 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 2 +python3 ./test.py -f 2-query/avg.py -P -Q 2 +python3 ./test.py -f 2-query/elapsed.py -P -Q 2 +python3 ./test.py -f 2-query/csum.py -P -Q 2 +python3 ./test.py -f 2-query/mavg.py -P -Q 2 +python3 ./test.py -f 2-query/sample.py -P -Q 2 +python3 ./test.py -f 2-query/function_diff.py -P -Q 2 +python3 ./test.py -f 2-query/unique.py -P -Q 2 +python3 ./test.py -f 2-query/stateduration.py -P -Q 2 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 2 +python3 ./test.py -f 2-query/statecount.py -P -Q 2 +python3 ./test.py -f 2-query/tail.py -P -Q 2 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 2 +python3 ./test.py -f 2-query/twa.py -P -Q 2 +python3 ./test.py -f 2-query/irate.py -P -Q 2 +python3 ./test.py -f 2-query/function_null.py -P -Q 2 +python3 ./test.py -f 2-query/count_partition.py -P -Q 2 +python3 ./test.py -f 2-query/max_partition.py -P -Q 2 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 2 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 2 +python3 ./test.py -f 2-query/last_row.py -P -Q 2 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 2 +python3 ./test.py -f 2-query/sml.py -P -Q 2 +python3 ./test.py -f 2-query/case_when.py -P -Q 2 +python3 ./test.py -f 2-query/blockSMA.py -P -Q 2 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 2 +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 2 + +python3 ./test.py -f 2-query/between.py -P -Q 3 +python3 ./test.py -f 2-query/distinct.py -P -Q 3 +python3 ./test.py -f 2-query/varchar.py -P -Q 3 +python3 ./test.py -f 2-query/ltrim.py -P -Q 3 +python3 ./test.py -f 2-query/rtrim.py -P -Q 3 +python3 ./test.py -f 2-query/length.py -P -Q 3 +python3 ./test.py -f 2-query/char_length.py -P -Q 3 +python3 ./test.py -f 2-query/upper.py -P -Q 3 +python3 ./test.py -f 2-query/lower.py -P -Q 3 +python3 ./test.py -f 2-query/join.py -P -Q 3 +python3 ./test.py -f 2-query/join2.py -P -Q 3 +python3 ./test.py -f 2-query/cast.py -P -Q 3 +python3 ./test.py -f 2-query/substr.py -P -Q 3 +python3 ./test.py -f 2-query/union.py -P -Q 3 +python3 ./test.py -f 2-query/union1.py -P -Q 3 +python3 ./test.py -f 2-query/concat2.py -P -Q 3 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 3 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 3 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 3 +python3 ./test.py -f 2-query/spread.py -P -Q 3 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 3 +python3 ./test.py -f 2-query/explain.py -P -Q 3 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 3 +python3 ./test.py -f 2-query/timezone.py -P -Q 3 +python3 ./test.py -f 2-query/Now.py -P -Q 3 +python3 ./test.py -f 2-query/Today.py -P -Q 3 +python3 ./test.py -f 2-query/max.py -P -Q 3 +python3 ./test.py -f 2-query/min.py -P -Q 3 +python3 ./test.py -f 2-query/mode.py -P -Q 3 +python3 ./test.py -f 2-query/count.py -P -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 3 +python3 ./test.py -f 2-query/last.py -P -Q 3 +python3 ./test.py -f 2-query/first.py -P -Q 3 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 3 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 3 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 3 +python3 ./test.py -f 2-query/diff.py -P -Q 3 +python3 ./test.py -f 2-query/Timediff.py -P -Q 3 +python3 ./test.py -f 2-query/json_tag.py -P -Q 3 +python3 ./test.py -f 2-query/top.py -P -Q 3 +python3 ./test.py -f 2-query/bottom.py -P -Q 3 +python3 ./test.py -f 2-query/percentile.py -P -Q 3 +python3 ./test.py -f 2-query/apercentile.py -P -Q 3 +python3 ./test.py -f 2-query/abs.py -P -Q 3 +python3 ./test.py -f 2-query/ceil.py -P -Q 3 +python3 ./test.py -f 2-query/floor.py -P -Q 3 +python3 ./test.py -f 2-query/round.py -P -Q 3 +python3 ./test.py -f 2-query/log.py -P -Q 3 +python3 ./test.py -f 2-query/pow.py -P -Q 3 +python3 ./test.py -f 2-query/sqrt.py -P -Q 3 +python3 ./test.py -f 2-query/sin.py -P -Q 3 +python3 ./test.py -f 2-query/cos.py -P -Q 3 +python3 ./test.py -f 2-query/tan.py -P -Q 3 +python3 ./test.py -f 2-query/arcsin.py -P -Q 3 +python3 ./test.py -f 2-query/arccos.py -P -Q 3 +python3 ./test.py -f 2-query/arctan.py -P -Q 3 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 3 +python3 ./test.py -f 2-query/stablity.py -P -Q 3 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 3 +python3 ./test.py -f 2-query/avg.py -P -Q 3 +python3 ./test.py -f 2-query/elapsed.py -P -Q 3 +python3 ./test.py -f 2-query/csum.py -P -Q 3 +python3 ./test.py -f 2-query/mavg.py -P -Q 3 +python3 ./test.py -f 2-query/sample.py -P -Q 3 +python3 ./test.py -f 2-query/function_diff.py -P -Q 3 +python3 ./test.py -f 2-query/unique.py -P -Q 3 +python3 ./test.py -f 2-query/stateduration.py -P -Q 3 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 3 +python3 ./test.py -f 2-query/statecount.py -P -Q 3 +python3 ./test.py -f 2-query/tail.py -P -Q 3 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 3 +python3 ./test.py -f 2-query/twa.py -P -Q 3 +python3 ./test.py -f 2-query/irate.py -P -Q 3 +python3 ./test.py -f 2-query/function_null.py -P -Q 3 +python3 ./test.py -f 2-query/count_partition.py -P -Q 3 +python3 ./test.py -f 2-query/max_partition.py -P -Q 3 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 3 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 3 +python3 ./test.py -f 2-query/last_row.py -P -Q 3 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 3 +python3 ./test.py -f 2-query/sml.py -P -Q 3 +python3 ./test.py -f 2-query/interp.py -P -Q 3 +python3 ./test.py -f 2-query/case_when.py -P -Q 3 +python3 ./test.py -f 2-query/blockSMA.py -P -Q 3 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 3 +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 3 +python3 ./test.py -f 2-query/between.py -P -Q 4 +python3 ./test.py -f 2-query/distinct.py -P -Q 4 +python3 ./test.py -f 2-query/varchar.py -P -Q 4 +python3 ./test.py -f 2-query/ltrim.py -P -Q 4 +python3 ./test.py -f 2-query/rtrim.py -P -Q 4 +python3 ./test.py -f 2-query/length.py -P -Q 4 +python3 ./test.py -f 2-query/char_length.py -P -Q 4 +python3 ./test.py -f 2-query/upper.py -P -Q 4 +python3 ./test.py -f 2-query/lower.py -P -Q 4 +python3 ./test.py -f 2-query/join.py -P -Q 4 +python3 ./test.py -f 2-query/join2.py -P -Q 4 +python3 ./test.py -f 2-query/substr.py -P -Q 4 +python3 ./test.py -f 2-query/union.py -P -Q 4 +python3 ./test.py -f 2-query/union1.py -P -Q 4 +python3 ./test.py -f 2-query/concat.py -P -Q 4 +python3 ./test.py -f 2-query/concat2.py -P -Q 4 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 4 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 4 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 4 +python3 ./test.py -f 2-query/spread.py -P -Q 4 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 4 +python3 ./test.py -f 2-query/explain.py -P -Q 4 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 4 +python3 ./test.py -f 2-query/timezone.py -P -Q 4 +python3 ./test.py -f 2-query/Now.py -P -Q 4 +python3 ./test.py -f 2-query/Today.py -P -Q 4 +python3 ./test.py -f 2-query/max.py -P -Q 4 +python3 ./test.py -f 2-query/min.py -P -Q 4 +python3 ./test.py -f 2-query/mode.py -P -Q 4 +python3 ./test.py -f 2-query/count.py -P -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 4 +python3 ./test.py -f 2-query/last.py -P -Q 4 +python3 ./test.py -f 2-query/first.py -P -Q 4 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 4 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 4 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 4 +python3 ./test.py -f 2-query/diff.py -P -Q 4 +python3 ./test.py -f 2-query/Timediff.py -P -Q 4 +python3 ./test.py -f 2-query/json_tag.py -P -Q 4 +python3 ./test.py -f 2-query/top.py -P -Q 4 +python3 ./test.py -f 2-query/bottom.py -P -Q 4 +python3 ./test.py -f 2-query/percentile.py -P -Q 4 +python3 ./test.py -f 2-query/apercentile.py -P -Q 4 +python3 ./test.py -f 2-query/abs.py -P -Q 4 +python3 ./test.py -f 2-query/ceil.py -P -Q 4 +python3 ./test.py -f 2-query/floor.py -P -Q 4 +python3 ./test.py -f 2-query/round.py -P -Q 4 +python3 ./test.py -f 2-query/log.py -P -Q 4 +python3 ./test.py -f 2-query/pow.py -P -Q 4 +python3 ./test.py -f 2-query/sqrt.py -P -Q 4 +python3 ./test.py -f 2-query/sin.py -P -Q 4 +python3 ./test.py -f 2-query/cos.py -P -Q 4 +python3 ./test.py -f 2-query/tan.py -P -Q 4 +python3 ./test.py -f 2-query/arcsin.py -P -Q 4 +python3 ./test.py -f 2-query/arccos.py -P -Q 4 +python3 ./test.py -f 2-query/arctan.py -P -Q 4 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 4 +python3 ./test.py -f 2-query/stablity.py -P -Q 4 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 4 +python3 ./test.py -f 2-query/avg.py -P -Q 4 +python3 ./test.py -f 2-query/elapsed.py -P -Q 4 +python3 ./test.py -f 2-query/csum.py -P -Q 4 +python3 ./test.py -f 2-query/mavg.py -P -Q 4 +python3 ./test.py -f 2-query/sample.py -P -Q 4 +python3 ./test.py -f 2-query/cast.py -P -Q 4 +python3 ./test.py -f 2-query/function_diff.py -P -Q 4 +python3 ./test.py -f 2-query/unique.py -P -Q 4 +python3 ./test.py -f 2-query/stateduration.py -P -Q 4 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 4 +python3 ./test.py -f 2-query/statecount.py -P -Q 4 +python3 ./test.py -f 2-query/tail.py -P -Q 4 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 4 +python3 ./test.py -f 2-query/twa.py -P -Q 4 +python3 ./test.py -f 2-query/irate.py -P -Q 4 +python3 ./test.py -f 2-query/function_null.py -P -Q 4 +python3 ./test.py -f 2-query/count_partition.py -P -Q 4 +python3 ./test.py -f 2-query/max_partition.py -P -Q 4 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 4 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 4 +python3 ./test.py -f 2-query/last_row.py -P -Q 4 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 4 +python3 ./test.py -f 2-query/sml.py -P -Q 4 +python3 ./test.py -f 2-query/interp.py -P -Q 4 +python3 ./test.py -f 2-query/case_when.py -P -Q 4 +python3 ./test.py -f 2-query/insert_select.py -P +python3 ./test.py -f 2-query/insert_select.py -P -R +python3 ./test.py -f 2-query/insert_select.py -P -Q 2 +python3 ./test.py -f 2-query/insert_select.py -P -Q 3 +python3 ./test.py -f 2-query/insert_select.py -P -Q 4 +python3 ./test.py -f 2-query/out_of_order.py -P -R +python3 ./test.py -f 2-query/blockSMA.py -P -Q 4 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 4 +python3 ./test.py -f 2-query/odbc.py -P +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 4 +python3 ./test.py -f 99-TDcase/TD-20582.py -P \ No newline at end of file diff --git a/tests/system-test/test-win.bat b/tests/system-test/test-win.bat new file mode 100644 index 0000000000000000000000000000000000000000..662adf3987a254c62da0fb68708e7b0d48cdfb21 --- /dev/null +++ b/tests/system-test/test-win.bat @@ -0,0 +1,96 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +if "%1" == "full" ( + echo Windows Taosd Full Test + set /a exitNum=0 + del /Q /F failed.txt + set caseFile="win-test-file" + if not "%2" == "" ( + set caseFile="%2" + ) + for /F "usebackq tokens=*" %%i in (!caseFile!) do ( + call :CheckSkipCase %%i + if !skipCase! == false ( + set line=%%i + if "!line:~,7!" == "python3" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt || set errorlevel=8 + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) + ) + ) + exit /b !exitNum! +) +echo Windows Taosd Test +for /F "usebackq tokens=*" %%i in (win-test-file) do ( + for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( + set /a a+=1 + set timeNow=!time! + echo !a! Processing %%i + call :GetTimeSeconds !timeNow! + set time1=!_timeTemp! + echo Start at !timeNow! + call %%i ARG1 > result_!a!.txt 2>error_!a!.txt || set errorlevel=8 + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit /b 8 ) else ( call :colorEcho 0a "Success" &echo. ) + ) +) +@REM echo Linux Taosd Test +@REM for /F "usebackq tokens=*" %%i in (simpletest.bat) do ( +@REM for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( +@REM set /a a+=1 +@REM echo !a! Processing %%i +@REM call :GetTimeSeconds !time! +@REM set time1=!_timeTemp! +@REM echo Start at !time! +@REM call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt +@REM if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +@REM ) +@REM ) +exit /b + +:colorEcho +set timeNow=%time% +call :GetTimeSeconds %timeNow% +set time2=%_timeTemp% +set /a interTime=%time2% - %time1% +echo End at %timeNow% , cast %interTime%s +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i +goto :eof + +:GetTimeSeconds +set tt=%1 +set tt=%tt:.= % +set tt=%tt::= % +set tt=%tt: 0= % +set /a index=1 +for %%a in (%tt%) do ( + if !index! EQU 1 ( + set /a hh=%%a + )^ + else if !index! EQU 2 ( + set /a mm=%%a + + )^ + else if !index! EQU 3 ( + set /a ss=%%a + ) + set /a index=index+1 +) +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +goto :eof + +:CheckSkipCase +set skipCase=false +if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=false ) +if "%*" == "python3 ./test.py -f 2-query/queryQnode.py" ( set skipCase=false ) +echo %* | grep "\-R" && set skipCase=true +:goto eof \ No newline at end of file diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 0c62c182f70eda8e20a7c5d91ebe360bfa2475ad..7082bb0f22f0ae2ab98a5c8ca8e87f895e1dea54 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -13,6 +13,7 @@ # pip install src/connector/python/ # -*- coding: utf-8 -*- +import os import sys import getopt import subprocess @@ -22,6 +23,7 @@ import json import platform import socket import threading +import importlib import toml sys.path.append("../pytest") @@ -53,8 +55,39 @@ def checkRunTimeError(): if hwnd: os.system("TASKKILL /F /IM taosd.exe") +# +# run case on previous cluster +# +def runOnPreviousCluster(host, config, fileName): + print("enter run on previeous") + + # load case module + sep = "/" + if platform.system().lower() == 'windows': + sep = os.sep + moduleName = fileName.replace(".py", "").replace(sep, ".") + uModule = importlib.import_module(moduleName) + case = uModule.TDTestCase() + + # create conn + conn = taos.connect(host, config) + + # run case + case.init(conn, False) + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed" % (fileName)) + # stop + case.stop() + + if __name__ == "__main__": + # + # analysis paramaters + # fileName = "all" deployPath = "" masterIp = "" @@ -75,8 +108,9 @@ if __name__ == "__main__": replicaVar = 1 asan = False independentMnode = True - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:a', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode']) + previousCluster = False + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:aP', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -101,6 +135,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') + tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') sys.exit(0) @@ -182,6 +217,12 @@ if __name__ == "__main__": if key in ['-n', '--replicaVar']: replicaVar = value + if key in ['-P', '--previous']: + previousCluster = True + + # + # do exeCmd command + # if not execCmd == "": if restful: tAdapter.init(deployPath) @@ -191,6 +232,9 @@ if __name__ == "__main__": exec(execCmd) quit() + # + # do stop option + # if (stop != 0): if (valgrind == 0): toBeKilled = "taosd" @@ -248,6 +292,9 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') + # + # get hostname + # if masterIp == "": host = socket.gethostname() else: @@ -256,8 +303,20 @@ if __name__ == "__main__": host = config["host"] except Exception as r: host = masterIp - tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + # + # do previousCluster option + # + if previousCluster: + tdDnodes.init(deployPath, masterIp) + runOnPreviousCluster(host, tdDnodes.getSimCfgPath(), fileName) + tdLog.info("run on previous cluster end.") + quit() + + # + # windows run + # if platform.system().lower() == 'windows': fileName = fileName.replace("/", os.sep) if (masterIp == "" and not fileName == "0-others\\udf_create.py"): @@ -291,9 +350,9 @@ if __name__ == "__main__": print(r) else: pass - if restful: - tAdapter.init(deployPath, masterIp) - tAdapter.stop(force_kill=True) + # if restful: + tAdapter.init(deployPath, masterIp) + tAdapter.stop(force_kill=True) if dnodeNums == 1 : tdDnodes.deploy(1,updateCfgDict) @@ -306,7 +365,7 @@ if __name__ == "__main__": if queryPolicy != 1: queryPolicy=int(queryPolicy) if restful: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") else: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) @@ -343,7 +402,7 @@ if __name__ == "__main__": if not restful: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) else: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") # tdLog.info(tdDnodes.getSimCfgPath(),host) if createDnodeNums == 1: createDnodeNums=dnodeNums @@ -359,7 +418,7 @@ if __name__ == "__main__": if queryPolicy != 1: queryPolicy=int(queryPolicy) if restful: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") else: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) @@ -382,11 +441,44 @@ if __name__ == "__main__": if not restful: conn = taos.connect(host="%s"%(host), config=tdDnodes.sim.getCfgDir()) else: - conn = taosrest.connect(url=f"http://{host}:6041") - if is_test_framework: - tdCases.runOneWindows(conn, fileName) + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + + if testCluster: + tdLog.info("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) else: - tdCases.runAllWindows(conn) + tdLog.info("Procedures for testing self-deployment") + if not restful: + conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + else: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + + if fileName == "all": + tdCases.runAllWindows(conn) + else: + tdCases.runOneWindows(conn, fileName, replicaVar) + + if restart: + if fileName == "all": + tdLog.info("not need to query ") + else: + sp = fileName.rsplit(".", 1) + if len(sp) == 2 and sp[1] == "py": + tdDnodes.stopAll() + tdDnodes.start(1) + time.sleep(1) + if not restful: + conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + else: + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + tdLog.info("query test after taosd restart") + tdCases.runOneWindows(conn, sp[0] + "_" + "restart.py", replicaVar) + else: + tdLog.info("not need to query") else: tdDnodes.setKillValgrind(killValgrind) tdDnodes.init(deployPath, masterIp) @@ -418,6 +510,7 @@ if __name__ == "__main__": tAdapter.stop(force_kill=True) if dnodeNums == 1 : + # dnode is one tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -431,7 +524,7 @@ if __name__ == "__main__": if not restful: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) else: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") # tdSql.init(conn.cursor()) # tdSql.execute("create qnode on dnode 1") # tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) @@ -458,6 +551,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : + # dnode > 1 cluster tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) tdDnodes = ClusterDnodes(dnodeslist) @@ -476,10 +570,11 @@ if __name__ == "__main__": tAdapter.deploy(adapter_cfg_dict) tAdapter.start() + # create taos connect if not restful: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) else: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") print(tdDnodes.getSimCfgPath(),host) if createDnodeNums == 1: createDnodeNums=dnodeNums @@ -494,10 +589,11 @@ if __name__ == "__main__": except Exception as r: print(r) + # do queryPolicy option if queryPolicy != 1: queryPolicy=int(queryPolicy) if restful: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") else: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) @@ -515,6 +611,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") + # run case if testCluster: tdLog.info("Procedures for testing cluster") if fileName == "all": @@ -526,13 +623,14 @@ if __name__ == "__main__": if not restful: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) else: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") if fileName == "all": tdCases.runAllLinux(conn) else: tdCases.runOneLinux(conn, fileName, replicaVar) + # do restart option if restart: if fileName == "all": tdLog.info("not need to query ") @@ -545,13 +643,14 @@ if __name__ == "__main__": if not restful: conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) else: - conn = taosrest.connect(url=f"http://{host}:6041") + conn = taosrest.connect(url=f"http://{host}:6041",timezone="utc") tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py", replicaVar) else: tdLog.info("not need to query") + # close for end if conn is not None: conn.close() if asan: diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file new file mode 100644 index 0000000000000000000000000000000000000000..7e68c40fd8b355d3d3cd40161bb2df09886e8570 --- /dev/null +++ b/tests/system-test/win-test-file @@ -0,0 +1,637 @@ +python3 ./test.py -f 1-insert/database_pre_suf.py +python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 +python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShellError.py +python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 0-others/telemetry.py +python3 ./test.py -f 0-others/backquote_check.py +python3 ./test.py -f 0-others/taosdMonitor.py +python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +python3 ./test.py -f 0-others/cachemodel.py +python3 ./test.py -f 0-others/sysinfo.py +python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/user_manage.py +python3 ./test.py -f 0-others/fsync.py +python3 ./test.py -f 0-others/compatibility.py +python3 ./test.py -f 0-others/tag_index_basic.py +# python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +python3 ./test.py -f 1-insert/alter_stable.py +python3 ./test.py -f 1-insert/alter_table.py +python3 ./test.py -f 1-insert/boundary.py +python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +python3 ./test.py -f 1-insert/table_comment.py +python3 ./test.py -f 1-insert/time_range_wise.py +python3 ./test.py -f 1-insert/block_wise.py +python3 ./test.py -f 1-insert/create_retentions.py +python3 ./test.py -f 1-insert/mutil_stage.py +python3 ./test.py -f 1-insert/table_param_ttl.py +python3 ./test.py -f 1-insert/table_param_ttl.py -R +python3 ./test.py -f 1-insert/update_data_muti_rows.py +python3 ./test.py -f 1-insert/db_tb_name_check.py +python3 ./test.py -f 1-insert/InsertFuturets.py +python3 ./test.py -f 0-others/show.py +python3 ./test.py -f 0-others/information_schema.py +python3 ./test.py -f 2-query/abs.py +python3 ./test.py -f 2-query/abs.py -R +python3 ./test.py -f 2-query/and_or_for_byte.py +python3 ./test.py -f 2-query/and_or_for_byte.py -R +python3 ./test.py -f 2-query/apercentile.py +python3 ./test.py -f 2-query/apercentile.py -R +python3 ./test.py -f 2-query/arccos.py +python3 ./test.py -f 2-query/arccos.py -R +python3 ./test.py -f 2-query/arcsin.py +python3 ./test.py -f 2-query/arcsin.py -R +python3 ./test.py -f 2-query/arctan.py +python3 ./test.py -f 2-query/arctan.py -R +python3 ./test.py -f 2-query/avg.py +python3 ./test.py -f 2-query/avg.py -R +python3 ./test.py -f 2-query/between.py +python3 ./test.py -f 2-query/between.py -R +python3 ./test.py -f 2-query/bottom.py +python3 ./test.py -f 2-query/bottom.py -R +python3 ./test.py -f 2-query/cast.py +python3 ./test.py -f 2-query/cast.py -R +python3 ./test.py -f 2-query/ceil.py +python3 ./test.py -f 2-query/ceil.py -R +python3 ./test.py -f 2-query/char_length.py +python3 ./test.py -f 2-query/char_length.py -R +python3 ./test.py -f 2-query/check_tsdb.py +python3 ./test.py -f 2-query/check_tsdb.py -R +python3 ./test.py -f 2-query/concat.py +python3 ./test.py -f 2-query/concat.py -R +python3 ./test.py -f 2-query/concat_ws.py +python3 ./test.py -f 2-query/concat_ws.py -R +python3 ./test.py -f 2-query/concat_ws2.py +python3 ./test.py -f 2-query/concat_ws2.py -R +python3 ./test.py -f 2-query/cos.py +python3 ./test.py -f 2-query/cos.py -R +python3 ./test.py -f 2-query/count_partition.py +python3 ./test.py -f 2-query/count_partition.py -R +python3 ./test.py -f 2-query/count.py +python3 ./test.py -f 2-query/count.py -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +python3 ./test.py -f 2-query/db.py +python3 ./test.py -f 2-query/db.py -N 3 -n 3 -R +python3 ./test.py -f 2-query/diff.py +python3 ./test.py -f 2-query/diff.py -R +python3 ./test.py -f 2-query/distinct.py +python3 ./test.py -f 2-query/distinct.py -R +python3 ./test.py -f 2-query/distribute_agg_apercentile.py +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +python3 ./test.py -f 2-query/distribute_agg_avg.py +python3 ./test.py -f 2-query/distribute_agg_avg.py -R +python3 ./test.py -f 2-query/distribute_agg_count.py +python3 ./test.py -f 2-query/distribute_agg_count.py -R +python3 ./test.py -f 2-query/distribute_agg_max.py +python3 ./test.py -f 2-query/distribute_agg_max.py -R +python3 ./test.py -f 2-query/distribute_agg_min.py +python3 ./test.py -f 2-query/distribute_agg_min.py -R +python3 ./test.py -f 2-query/distribute_agg_spread.py +python3 ./test.py -f 2-query/distribute_agg_spread.py -R +python3 ./test.py -f 2-query/distribute_agg_stddev.py +python3 ./test.py -f 2-query/distribute_agg_stddev.py -R +python3 ./test.py -f 2-query/distribute_agg_sum.py +python3 ./test.py -f 2-query/distribute_agg_sum.py -R +python3 ./test.py -f 2-query/explain.py +python3 ./test.py -f 2-query/explain.py -R +python3 ./test.py -f 2-query/first.py +python3 ./test.py -f 2-query/first.py -R +python3 ./test.py -f 2-query/floor.py +python3 ./test.py -f 2-query/floor.py -R +python3 ./test.py -f 2-query/function_null.py +python3 ./test.py -f 2-query/function_null.py -R +python3 ./test.py -f 2-query/function_stateduration.py +python3 ./test.py -f 2-query/function_stateduration.py -R +python3 ./test.py -f 2-query/histogram.py +python3 ./test.py -f 2-query/histogram.py -R +python3 ./test.py -f 2-query/hyperloglog.py +python3 ./test.py -f 2-query/hyperloglog.py -R +python3 ./test.py -f 2-query/interp.py +python3 ./test.py -f 2-query/interp.py -R +python3 ./test.py -f 2-query/irate.py +python3 ./test.py -f 2-query/irate.py -R +python3 ./test.py -f 2-query/join.py +python3 ./test.py -f 2-query/join.py -R +python3 ./test.py -f 2-query/last_row.py +python3 ./test.py -f 2-query/last_row.py -R +python3 ./test.py -f 2-query/last.py +python3 ./test.py -f 2-query/last.py -R +python3 ./test.py -f 2-query/leastsquares.py +python3 ./test.py -f 2-query/leastsquares.py -R +python3 ./test.py -f 2-query/length.py +python3 ./test.py -f 2-query/length.py -R +python3 ./test.py -f 2-query/limit.py +python3 ./test.py -f 2-query/log.py +python3 ./test.py -f 2-query/log.py -R +python3 ./test.py -f 2-query/lower.py +python3 ./test.py -f 2-query/lower.py -R +python3 ./test.py -f 2-query/ltrim.py +python3 ./test.py -f 2-query/ltrim.py -R +python3 ./test.py -f 2-query/mavg.py +python3 ./test.py -f 2-query/mavg.py -R +python3 ./test.py -f 2-query/max_partition.py +python3 ./test.py -f 2-query/max_partition.py -R +python3 ./test.py -f 2-query/max_min_last_interval.py +python3 ./test.py -f 2-query/max.py +python3 ./test.py -f 2-query/max.py -R +python3 ./test.py -f 2-query/min.py +python3 ./test.py -f 2-query/min.py -R +python3 ./test.py -f 2-query/mode.py +python3 ./test.py -f 2-query/mode.py -R +python3 ./test.py -f 2-query/Now.py +python3 ./test.py -f 2-query/Now.py -R +python3 ./test.py -f 2-query/percentile.py +python3 ./test.py -f 2-query/percentile.py -R +python3 ./test.py -f 2-query/pow.py +python3 ./test.py -f 2-query/pow.py -R +python3 ./test.py -f 2-query/query_cols_tags_and_or.py +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R +python3 ./test.py -f 2-query/round.py +python3 ./test.py -f 2-query/round.py -R +python3 ./test.py -f 2-query/rtrim.py +python3 ./test.py -f 2-query/rtrim.py -R +python3 ./test.py -f 2-query/sample.py +python3 ./test.py -f 2-query/sample.py -R +python3 ./test.py -f 2-query/sin.py +python3 ./test.py -f 2-query/sin.py -R +python3 ./test.py -f 2-query/smaTest.py +python3 ./test.py -f 2-query/smaTest.py -R +python3 ./test.py -f 2-query/sml.py +python3 ./test.py -f 2-query/sml.py -R +python3 ./test.py -f 2-query/spread.py +python3 ./test.py -f 2-query/spread.py -R +python3 ./test.py -f 2-query/sqrt.py +python3 ./test.py -f 2-query/sqrt.py -R +python3 ./test.py -f 2-query/statecount.py +python3 ./test.py -f 2-query/statecount.py -R +python3 ./test.py -f 2-query/stateduration.py +python3 ./test.py -f 2-query/stateduration.py -R +python3 ./test.py -f 2-query/substr.py +python3 ./test.py -f 2-query/substr.py -R +python3 ./test.py -f 2-query/sum.py +python3 ./test.py -f 2-query/sum.py -R +python3 ./test.py -f 2-query/tail.py +python3 ./test.py -f 2-query/tail.py -R +python3 ./test.py -f 2-query/tan.py +python3 ./test.py -f 2-query/tan.py -R +python3 ./test.py -f 2-query/Timediff.py +python3 ./test.py -f 2-query/Timediff.py -R +python3 ./test.py -f 2-query/timetruncate.py +python3 ./test.py -f 2-query/timetruncate.py -R +python3 ./test.py -f 2-query/timezone.py +python3 ./test.py -f 2-query/timezone.py -R +python3 ./test.py -f 2-query/To_iso8601.py +python3 ./test.py -f 2-query/To_iso8601.py -R +python3 ./test.py -f 2-query/To_unixtimestamp.py +python3 ./test.py -f 2-query/To_unixtimestamp.py -R +python3 ./test.py -f 2-query/Today.py +python3 ./test.py -f 2-query/Today.py -R +python3 ./test.py -f 2-query/top.py +python3 ./test.py -f 2-query/top.py -R +python3 ./test.py -f 2-query/tsbsQuery.py +python3 ./test.py -f 2-query/tsbsQuery.py -R +python3 ./test.py -f 2-query/ttl_comment.py +python3 ./test.py -f 2-query/ttl_comment.py -R +python3 ./test.py -f 2-query/twa.py +python3 ./test.py -f 2-query/twa.py -R +python3 ./test.py -f 2-query/union.py +python3 ./test.py -f 2-query/union.py -R +python3 ./test.py -f 2-query/unique.py +python3 ./test.py -f 2-query/unique.py -R +python3 ./test.py -f 2-query/upper.py +python3 ./test.py -f 2-query/upper.py -R +python3 ./test.py -f 2-query/varchar.py +python3 ./test.py -f 2-query/varchar.py -R +python3 ./test.py -f 2-query/case_when.py +python3 ./test.py -f 2-query/case_when.py -R +python3 ./test.py -f 2-query/blockSMA.py +python3 ./test.py -f 2-query/blockSMA.py -R +python3 ./test.py -f 1-insert/update_data.py +python3 ./test.py -f 1-insert/tb_100w_data_order.py +python3 ./test.py -f 1-insert/delete_stable.py +python3 ./test.py -f 1-insert/delete_childtable.py +python3 ./test.py -f 1-insert/delete_normaltable.py +python3 ./test.py -f 1-insert/keep_expired.py +python3 ./test.py -f 1-insert/drop.py +python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 +python3 ./test.py -f 2-query/join2.py +python3 ./test.py -f 2-query/union1.py +python3 ./test.py -f 2-query/concat2.py +python3 ./test.py -f 2-query/json_tag.py +python3 ./test.py -f 2-query/nestedQuery.py +python3 ./test.py -f 2-query/nestedQueryInterval.py +python3 ./test.py -f 2-query/nestedQuery_str.py +python3 ./test.py -f 2-query/nestedQuery_math.py +python3 ./test.py -f 2-query/nestedQuery_time.py +python3 ./test.py -f 2-query/stablity.py +python3 ./test.py -f 2-query/stablity_1.py +python3 ./test.py -f 2-query/elapsed.py +python3 ./test.py -f 2-query/csum.py +python3 ./test.py -f 2-query/function_diff.py +python3 ./test.py -f 2-query/tagFilter.py +python3 ./test.py -f 2-query/queryQnode.py +python3 ./test.py -f 6-cluster/5dnode1mnode.py +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +python3 ./test.py -f 7-tmq/create_wrong_topic.py +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 +python3 ./test.py -f 7-tmq/basic5.py +python3 ./test.py -f 7-tmq/subscribeDb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeDb0.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeDb1.py +python3 ./test.py -f 7-tmq/subscribeDb2.py +python3 ./test.py -f 7-tmq/subscribeDb3.py +python3 ./test.py -f 7-tmq/subscribeDb4.py +python3 ./test.py -f 7-tmq/subscribeStb.py +python3 ./test.py -f 7-tmq/subscribeStb0.py +python3 ./test.py -f 7-tmq/subscribeStb1.py +python3 ./test.py -f 7-tmq/subscribeStb2.py +python3 ./test.py -f 7-tmq/subscribeStb3.py +python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py +python3 ./test.py -f 7-tmq/tmqError.py +python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqCheckData1.py +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py +python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py +python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqDropStb.py +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py +python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +python3 ./test.py -f 7-tmq/tmq_taosx.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 99-TDcase/TD-19201.py +python3 ./test.py -f 99-TDcase/TD-21561.py +python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 +python3 ./test.py -f 2-query/between.py -Q 2 +python3 ./test.py -f 2-query/distinct.py -Q 2 +python3 ./test.py -f 2-query/varchar.py -Q 2 +python3 ./test.py -f 2-query/ltrim.py -Q 2 +python3 ./test.py -f 2-query/rtrim.py -Q 2 +python3 ./test.py -f 2-query/length.py -Q 2 +python3 ./test.py -f 2-query/char_length.py -Q 2 +python3 ./test.py -f 2-query/upper.py -Q 2 +python3 ./test.py -f 2-query/lower.py -Q 2 +python3 ./test.py -f 2-query/join.py -Q 2 +python3 ./test.py -f 2-query/join2.py -Q 2 +python3 ./test.py -f 2-query/cast.py -Q 2 +python3 ./test.py -f 2-query/substr.py -Q 2 +python3 ./test.py -f 2-query/union.py -Q 2 +python3 ./test.py -f 2-query/union1.py -Q 2 +python3 ./test.py -f 2-query/concat.py -Q 2 +python3 ./test.py -f 2-query/concat2.py -Q 2 +python3 ./test.py -f 2-query/concat_ws.py -Q 2 +python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +python3 ./test.py -f 2-query/spread.py -Q 2 +python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +python3 ./test.py -f 2-query/explain.py -Q 2 +python3 ./test.py -f 2-query/leastsquares.py -Q 2 +python3 ./test.py -f 2-query/timezone.py -Q 2 +python3 ./test.py -f 2-query/Now.py -Q 2 +python3 ./test.py -f 2-query/Today.py -Q 2 +python3 ./test.py -f 2-query/max.py -Q 2 +python3 ./test.py -f 2-query/min.py -Q 2 +python3 ./test.py -f 2-query/mode.py -Q 2 +python3 ./test.py -f 2-query/count.py -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +python3 ./test.py -f 2-query/last.py -Q 2 +python3 ./test.py -f 2-query/first.py -Q 2 +python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +python3 ./test.py -f 2-query/timetruncate.py -Q 2 +python3 ./test.py -f 2-query/diff.py -Q 2 +python3 ./test.py -f 2-query/Timediff.py -Q 2 +python3 ./test.py -f 2-query/json_tag.py -Q 2 +python3 ./test.py -f 2-query/top.py -Q 2 +python3 ./test.py -f 2-query/bottom.py -Q 2 +python3 ./test.py -f 2-query/percentile.py -Q 2 +python3 ./test.py -f 2-query/apercentile.py -Q 2 +python3 ./test.py -f 2-query/abs.py -Q 2 +python3 ./test.py -f 2-query/ceil.py -Q 2 +python3 ./test.py -f 2-query/floor.py -Q 2 +python3 ./test.py -f 2-query/round.py -Q 2 +python3 ./test.py -f 2-query/log.py -Q 2 +python3 ./test.py -f 2-query/pow.py -Q 2 +python3 ./test.py -f 2-query/sqrt.py -Q 2 +python3 ./test.py -f 2-query/sin.py -Q 2 +python3 ./test.py -f 2-query/cos.py -Q 2 +python3 ./test.py -f 2-query/tan.py -Q 2 +python3 ./test.py -f 2-query/arcsin.py -Q 2 +python3 ./test.py -f 2-query/arccos.py -Q 2 +python3 ./test.py -f 2-query/arctan.py -Q 2 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +python3 ./test.py -f 2-query/interp.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +python3 ./test.py -f 2-query/stablity.py -Q 2 +python3 ./test.py -f 2-query/stablity_1.py -Q 2 +python3 ./test.py -f 2-query/avg.py -Q 2 +python3 ./test.py -f 2-query/elapsed.py -Q 2 +python3 ./test.py -f 2-query/csum.py -Q 2 +python3 ./test.py -f 2-query/mavg.py -Q 2 +python3 ./test.py -f 2-query/sample.py -Q 2 +python3 ./test.py -f 2-query/function_diff.py -Q 2 +python3 ./test.py -f 2-query/unique.py -Q 2 +python3 ./test.py -f 2-query/stateduration.py -Q 2 +python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +python3 ./test.py -f 2-query/statecount.py -Q 2 +python3 ./test.py -f 2-query/tail.py -Q 2 +python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +python3 ./test.py -f 2-query/twa.py -Q 2 +python3 ./test.py -f 2-query/irate.py -Q 2 +python3 ./test.py -f 2-query/function_null.py -Q 2 +python3 ./test.py -f 2-query/count_partition.py -Q 2 +python3 ./test.py -f 2-query/max_partition.py -Q 2 +python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 +python3 ./test.py -f 2-query/last_row.py -Q 2 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +python3 ./test.py -f 2-query/sml.py -Q 2 +python3 ./test.py -f 2-query/case_when.py -Q 2 +python3 ./test.py -f 2-query/blockSMA.py -Q 2 +python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 +python3 ./test.py -f 2-query/between.py -Q 3 +python3 ./test.py -f 2-query/distinct.py -Q 3 +python3 ./test.py -f 2-query/varchar.py -Q 3 +python3 ./test.py -f 2-query/ltrim.py -Q 3 +python3 ./test.py -f 2-query/rtrim.py -Q 3 +python3 ./test.py -f 2-query/length.py -Q 3 +python3 ./test.py -f 2-query/char_length.py -Q 3 +python3 ./test.py -f 2-query/upper.py -Q 3 +python3 ./test.py -f 2-query/lower.py -Q 3 +python3 ./test.py -f 2-query/join.py -Q 3 +python3 ./test.py -f 2-query/join2.py -Q 3 +python3 ./test.py -f 2-query/cast.py -Q 3 +python3 ./test.py -f 2-query/substr.py -Q 3 +python3 ./test.py -f 2-query/union.py -Q 3 +python3 ./test.py -f 2-query/union1.py -Q 3 +python3 ./test.py -f 2-query/concat.py -Q 3 +python3 ./test.py -f 2-query/concat2.py -Q 3 +python3 ./test.py -f 2-query/concat_ws.py -Q 3 +python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +python3 ./test.py -f 2-query/spread.py -Q 3 +python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +python3 ./test.py -f 2-query/explain.py -Q 3 +python3 ./test.py -f 2-query/leastsquares.py -Q 3 +python3 ./test.py -f 2-query/timezone.py -Q 3 +python3 ./test.py -f 2-query/Now.py -Q 3 +python3 ./test.py -f 2-query/Today.py -Q 3 +python3 ./test.py -f 2-query/max.py -Q 3 +python3 ./test.py -f 2-query/min.py -Q 3 +python3 ./test.py -f 2-query/mode.py -Q 3 +python3 ./test.py -f 2-query/count.py -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +python3 ./test.py -f 2-query/last.py -Q 3 +python3 ./test.py -f 2-query/first.py -Q 3 +python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +python3 ./test.py -f 2-query/timetruncate.py -Q 3 +python3 ./test.py -f 2-query/diff.py -Q 3 +python3 ./test.py -f 2-query/Timediff.py -Q 3 +python3 ./test.py -f 2-query/json_tag.py -Q 3 +python3 ./test.py -f 2-query/top.py -Q 3 +python3 ./test.py -f 2-query/bottom.py -Q 3 +python3 ./test.py -f 2-query/percentile.py -Q 3 +python3 ./test.py -f 2-query/apercentile.py -Q 3 +python3 ./test.py -f 2-query/abs.py -Q 3 +python3 ./test.py -f 2-query/ceil.py -Q 3 +python3 ./test.py -f 2-query/floor.py -Q 3 +python3 ./test.py -f 2-query/round.py -Q 3 +python3 ./test.py -f 2-query/log.py -Q 3 +python3 ./test.py -f 2-query/pow.py -Q 3 +python3 ./test.py -f 2-query/sqrt.py -Q 3 +python3 ./test.py -f 2-query/sin.py -Q 3 +python3 ./test.py -f 2-query/cos.py -Q 3 +python3 ./test.py -f 2-query/tan.py -Q 3 +python3 ./test.py -f 2-query/arcsin.py -Q 3 +python3 ./test.py -f 2-query/arccos.py -Q 3 +python3 ./test.py -f 2-query/arctan.py -Q 3 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 3 +python3 ./test.py -f 2-query/stablity.py -Q 3 +python3 ./test.py -f 2-query/stablity_1.py -Q 3 +python3 ./test.py -f 2-query/avg.py -Q 3 +python3 ./test.py -f 2-query/elapsed.py -Q 3 +python3 ./test.py -f 2-query/csum.py -Q 3 +python3 ./test.py -f 2-query/mavg.py -Q 3 +python3 ./test.py -f 2-query/sample.py -Q 3 +python3 ./test.py -f 2-query/function_diff.py -Q 3 +python3 ./test.py -f 2-query/unique.py -Q 3 +python3 ./test.py -f 2-query/stateduration.py -Q 3 +python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +python3 ./test.py -f 2-query/statecount.py -Q 3 +python3 ./test.py -f 2-query/tail.py -Q 3 +python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +python3 ./test.py -f 2-query/twa.py -Q 3 +python3 ./test.py -f 2-query/irate.py -Q 3 +python3 ./test.py -f 2-query/function_null.py -Q 3 +python3 ./test.py -f 2-query/count_partition.py -Q 3 +python3 ./test.py -f 2-query/max_partition.py -Q 3 +python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 +python3 ./test.py -f 2-query/last_row.py -Q 3 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/interp.py -Q 3 +python3 ./test.py -f 2-query/case_when.py -Q 3 +python3 ./test.py -f 2-query/blockSMA.py -Q 3 +python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 +python3 ./test.py -f 2-query/between.py -Q 4 +python3 ./test.py -f 2-query/distinct.py -Q 4 +python3 ./test.py -f 2-query/varchar.py -Q 4 +python3 ./test.py -f 2-query/ltrim.py -Q 4 +python3 ./test.py -f 2-query/rtrim.py -Q 4 +python3 ./test.py -f 2-query/length.py -Q 4 +python3 ./test.py -f 2-query/char_length.py -Q 4 +python3 ./test.py -f 2-query/upper.py -Q 4 +python3 ./test.py -f 2-query/lower.py -Q 4 +python3 ./test.py -f 2-query/join.py -Q 4 +python3 ./test.py -f 2-query/join2.py -Q 4 +python3 ./test.py -f 2-query/substr.py -Q 4 +python3 ./test.py -f 2-query/union.py -Q 4 +python3 ./test.py -f 2-query/union1.py -Q 4 +python3 ./test.py -f 2-query/concat.py -Q 4 +python3 ./test.py -f 2-query/concat2.py -Q 4 +python3 ./test.py -f 2-query/concat_ws.py -Q 4 +python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +python3 ./test.py -f 2-query/spread.py -Q 4 +python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +python3 ./test.py -f 2-query/explain.py -Q 4 +python3 ./test.py -f 2-query/leastsquares.py -Q 4 +python3 ./test.py -f 2-query/timezone.py -Q 4 +python3 ./test.py -f 2-query/Now.py -Q 4 +python3 ./test.py -f 2-query/Today.py -Q 4 +python3 ./test.py -f 2-query/max.py -Q 4 +python3 ./test.py -f 2-query/min.py -Q 4 +python3 ./test.py -f 2-query/mode.py -Q 4 +python3 ./test.py -f 2-query/count.py -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +python3 ./test.py -f 2-query/last.py -Q 4 +python3 ./test.py -f 2-query/first.py -Q 4 +python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +python3 ./test.py -f 2-query/timetruncate.py -Q 4 +python3 ./test.py -f 2-query/diff.py -Q 4 +python3 ./test.py -f 2-query/Timediff.py -Q 4 +python3 ./test.py -f 2-query/top.py -Q 4 +python3 ./test.py -f 2-query/bottom.py -Q 4 +python3 ./test.py -f 2-query/percentile.py -Q 4 +python3 ./test.py -f 2-query/apercentile.py -Q 4 +python3 ./test.py -f 2-query/abs.py -Q 4 +python3 ./test.py -f 2-query/ceil.py -Q 4 +python3 ./test.py -f 2-query/floor.py -Q 4 +python3 ./test.py -f 2-query/round.py -Q 4 +python3 ./test.py -f 2-query/log.py -Q 4 +python3 ./test.py -f 2-query/pow.py -Q 4 +python3 ./test.py -f 2-query/sqrt.py -Q 4 +python3 ./test.py -f 2-query/sin.py -Q 4 +python3 ./test.py -f 2-query/cos.py -Q 4 +python3 ./test.py -f 2-query/tan.py -Q 4 +python3 ./test.py -f 2-query/arcsin.py -Q 4 +python3 ./test.py -f 2-query/arccos.py -Q 4 +python3 ./test.py -f 2-query/arctan.py -Q 4 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +python3 ./test.py -f 2-query/avg.py -Q 4 +python3 ./test.py -f 2-query/elapsed.py -Q 4 +python3 ./test.py -f 2-query/csum.py -Q 4 +python3 ./test.py -f 2-query/mavg.py -Q 4 +python3 ./test.py -f 2-query/sample.py -Q 4 +python3 ./test.py -f 2-query/cast.py -Q 4 +python3 ./test.py -f 2-query/function_diff.py -Q 4 +python3 ./test.py -f 2-query/unique.py -Q 4 +python3 ./test.py -f 2-query/tail.py -Q 4 +python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +python3 ./test.py -f 2-query/twa.py -Q 4 +python3 ./test.py -f 2-query/irate.py -Q 4 +python3 ./test.py -f 2-query/function_null.py -Q 4 +python3 ./test.py -f 2-query/count_partition.py -Q 4 +python3 ./test.py -f 2-query/max_partition.py -Q 4 +python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 +python3 ./test.py -f 2-query/last_row.py -Q 4 +python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +python3 ./test.py -f 2-query/sml.py -Q 4 +python3 ./test.py -f 2-query/interp.py -Q 4 +python3 ./test.py -f 2-query/case_when.py -Q 4 +python3 ./test.py -f 2-query/insert_select.py +python3 ./test.py -f 2-query/insert_select.py -R +python3 ./test.py -f 2-query/insert_select.py -Q 2 +python3 ./test.py -f 2-query/insert_select.py -Q 3 +python3 ./test.py -f 2-query/insert_select.py -Q 4 +python3 ./test.py -f 2-query/insert_null_none.py +python3 ./test.py -f 2-query/insert_null_none.py -R +python3 ./test.py -f 2-query/insert_null_none.py -Q 2 +python3 ./test.py -f 2-query/insert_null_none.py -Q 3 +python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +python3 ./test.py -f 2-query/out_of_order.py +python3 ./test.py -f 2-query/out_of_order.py -R +python3 ./test.py -f 2-query/out_of_order.py -Q 2 +python3 ./test.py -f 2-query/out_of_order.py -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -Q 4 +python3 ./test.py -f 2-query/max_min_data.py +python3 ./test.py -f 2-query/max_min_data.py -R +python3 ./test.py -f 2-query/max_min_data.py -Q 2 +python3 ./test.py -f 2-query/max_min_data.py -Q 3 +python3 ./test.py -f 2-query/max_min_data.py -Q 4 +python3 ./test.py -f 2-query/blockSMA.py -Q 4 +python3 ./test.py -f 2-query/odbc.py +python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 +python3 ./test.py -f 99-TDcase/TD-20582.py diff --git a/tests/unit-test/win-test-file b/tests/unit-test/win-test-file new file mode 100644 index 0000000000000000000000000000000000000000..7ddca165e55c15048841effe860d255befc3955f --- /dev/null +++ b/tests/unit-test/win-test-file @@ -0,0 +1 @@ +bash test.sh diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 600e33feab10fcd0b3c0ccee2de3cd61544e16d8..7b1dc3a541416335dca761aba8634679473ecdd2 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,10 +26,6 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () -IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) -ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) - IF (TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE () diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 113853bd8397fe194d0efc65a55f73f2aa865929..6345647e2f24b1a87cee8a6a0c60275907946dfe 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -28,6 +28,10 @@ #ifdef WEBSOCKET #include "taosws.h" + +#define SHELL_WS_TIMEOUT 30 +#define SHELL_WS_DSN_BUFF 256 +#define SHELL_WS_DSN_MASK 10 #endif #define SHELL_MAX_HISTORY_SIZE 1000 @@ -83,7 +87,7 @@ typedef struct { const char *clientVersion; char cusName[32]; char promptHeader[32]; - const char* promptContinue; + char promptContinue[32]; const char* osname; int32_t promptSize; char programVersion[256]; @@ -99,7 +103,7 @@ typedef struct { bool exit; #ifdef WEBSOCKET WS_TAOS* ws_conn; - bool stop_query; + bool stop_query; #endif } SShellObj; @@ -139,7 +143,7 @@ void shellExit(); void shellTestNetWork(); #ifdef WEBSOCKET -void shellCheckConnectMode(); +void shellCheckConnectMode(); // shellWebsocket.c int shell_conn_ws_server(bool first); int32_t shell_run_websocket(); diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 19275cce8228ff280b262c87e044a770f6a6a00d..52cb524e3bfb98875e4f93a20bf21fa185674a29 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -18,18 +18,7 @@ #endif #include "shellInt.h" - -#ifndef CUS_NAME - char cusName[] = "TDengine"; -#endif - -#ifndef CUS_PROMPT - char cusPrompt[] = "taos"; -#endif - -#ifndef CUS_EMAIL - char cusEmail[] = ""; -#endif +#include "version.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -58,9 +47,9 @@ #define SHELL_VERSION "Print program version." #ifdef WEBSOCKET -#define SHELL_DSN "The dsn to use when connecting to cloud server." -#define SHELL_REST "Use restful mode when connecting." -#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 10." +#define SHELL_DSN "Use dsn to connect to the cloud server or to a remote server which provides WebSocket connection." +#define SHELL_REST "Use RESTful mode when connecting." +#define SHELL_TIMEOUT "Set the timeout for websocket query in seconds, default is 30." #endif static int32_t shellParseSingleOpt(int32_t key, char *arg); @@ -92,7 +81,11 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); +#ifdef CUS_EMAIL + printf("\r\n\r\nReport bugs to %s.\r\n", CUS_EMAIL); +#else + printf("\r\n\r\nReport bugs to %s.\r\n", "support@taosdata.com"); +#endif } #ifdef LINUX @@ -104,7 +97,11 @@ void shellPrintHelp() { #endif const char *argp_program_version = version; -const char *argp_program_bug_address = cusEmail; +#ifdef CUS_EMAIL +const char *argp_program_bug_address = CUS_EMAIL; +#else +const char *argp_program_bug_address = "support@taosdata.com"; +#endif static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -127,7 +124,7 @@ static struct argp_option shellOptions[] = { #ifdef WEBSOCKET {"dsn", 'E', "DSN", 0, SHELL_DSN}, {"restful", 'R', 0, 0, SHELL_REST}, - {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, + {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {0}, @@ -145,7 +142,7 @@ static void shellParseArgsUseArgp(int argc, char *argv[]) { #endif #ifndef ARGP_ERR_UNKNOWN - #define ARGP_ERR_UNKNOWN E2BIG +#define ARGP_ERR_UNKNOWN E2BIG #endif static int32_t shellParseSingleOpt(int32_t key, char *arg) { @@ -223,9 +220,9 @@ static int32_t shellParseSingleOpt(int32_t key, char *arg) { pArgs->dsn = arg; pArgs->cloud = true; break; - case 'T': - pArgs->timeout = atoi(arg); - break; + case 'T': + pArgs->timeout = atoi(arg); + break; #endif case 'V': pArgs->is_version = true; @@ -246,7 +243,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 + || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } @@ -262,12 +260,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { return -1; } - if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || - key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' + || key[1] == 'a' || key[1] == 'c' || key[1] == 's' + || key[1] == 'f' || key[1] == 'd' || key[1] == 'w' + || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' #ifdef WEBSOCKET - || key[1] == 'E' || key[1] == 'T' + || key[1] == 'E' || key[1] == 'T' #endif - ) { + ) { if (i + 1 >= argc) { fprintf(stderr, "option %s requires an argument\r\n", key); return -1; @@ -279,12 +279,14 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { } shellParseSingleOpt(key[1], val); i++; - } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || - key[1] == 't' || key[1] == 'V' || key[1] == '?' || key[1] == 1 + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' + || key[1] == 'r' || key[1] == 'k' + || key[1] == 't' || key[1] == 'V' + || key[1] == '?' || key[1] == 1 #ifdef WEBSOCKET - ||key[1] == 'R' + ||key[1] == 'R' #endif - ) { + ) { shellParseSingleOpt(key[1], NULL); } else { fprintf(stderr, "invalid option %s\r\n", key); @@ -406,16 +408,33 @@ static int32_t shellCheckArgs() { int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); - shell.info.clientVersion = + shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; - strcpy(shell.info.cusName, cusName); - sprintf(shell.info.promptHeader, "%s> ", cusPrompt); - shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; +#ifdef CUS_NAME + strcpy(shell.info.cusName, CUS_NAME); +#else + strcpy(shell.info.cusName, "TDengine"); +#endif + char promptContinueFormat[32] = {0}; +#ifdef CUS_PROMPT + sprintf(shell.info.promptHeader, "%s> ", CUS_PROMPT); + sprintf(promptContinueFormat, "%%%zus> ", strlen(CUS_PROMPT)); +#else + sprintf(shell.info.promptHeader, "taos> "); + sprintf(promptContinueFormat, "%%%zus> ", strlen("taos")); +#endif + sprintf(shell.info.promptContinue, promptContinueFormat, " "); shell.info.promptSize = strlen(shell.info.promptHeader); +#ifdef TD_ENTERPRISE + snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), + "version: %s compatible_version: %s\ngitinfo: %s\ngitinfoOfInternal: %s\nbuildInfo: %s", version, + compatible_version, gitinfo, gitinfoOfInternal, buildinfo); +#else snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s compatible_version: %s\ngitinfo: %s\nbuildInfo: %s", version, compatible_version, gitinfo, buildinfo); +#endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) shell.info.osname = "Windows"; diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index a8986351b7325802ba6a121db56a1cbc5dfe1295..140720af815cf317bf7665c40750b2d0fb2ebd37 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -84,15 +84,15 @@ SWords shellCommands[] = { {"create table using tags(", 0, 0, NULL}, {"create database " " " - " ;", - 0, 0, NULL}, + " ;", 0, 0, NULL}, {"create dnode ", 0, 0, NULL}, {"create index on ()", 0, 0, NULL}, {"create mnode on dnode ;", 0, 0, NULL}, {"create qnode on dnode ;", 0, 0, NULL}, {"create stream into as select", 0, 0, NULL}, // 26 append sub sql {"create topic as select", 0, 0, NULL}, // 27 append sub sql - {"create function ", 0, 0, NULL}, + {"create function as outputtype language ", 0, 0, NULL}, + {"create aggregate function as outputtype bufsize language ", 0, 0, NULL}, {"create user pass sysinfo 0;", 0, 0, NULL}, {"create user pass sysinfo 1;", 0, 0, NULL}, {"describe ", 0, 0, NULL}, @@ -105,7 +105,7 @@ SWords shellCommands[] = { {"drop qnode on dnode ;", 0, 0, NULL}, {"drop user ;", 0, 0, NULL}, // 40 - {"drop function", 0, 0, NULL}, + {"drop function ;", 0, 0, NULL}, {"drop consumer group on ", 0, 0, NULL}, {"drop topic ;", 0, 0, NULL}, {"drop stream ;", 0, 0, NULL}, @@ -272,6 +272,8 @@ char* key_systable[] = { "ins_subscriptions", "ins_streams", "ins_stream_tasks", "ins_vnodes", "ins_user_privileges", "perf_connections", "perf_queries", "perf_consumers", "perf_trans", "perf_apps"}; +char* udf_language[] = {"\'Python\'", "\'C\'"}; + // // ------- global variant define --------- // @@ -291,25 +293,29 @@ bool waitAutoFill = false; #define WT_VAR_USERNAME 4 #define WT_VAR_TOPIC 5 #define WT_VAR_STREAM 6 -#define WT_VAR_ALLTABLE 7 -#define WT_VAR_FUNC 8 -#define WT_VAR_KEYWORD 9 -#define WT_VAR_TBACTION 10 -#define WT_VAR_DBOPTION 11 -#define WT_VAR_ALTER_DBOPTION 12 -#define WT_VAR_DATATYPE 13 -#define WT_VAR_KEYTAGS 14 -#define WT_VAR_ANYWORD 15 -#define WT_VAR_TBOPTION 16 -#define WT_VAR_USERACTION 17 -#define WT_VAR_KEYSELECT 18 -#define WT_VAR_SYSTABLE 19 - -#define WT_VAR_CNT 20 - -#define WT_FROM_DB_MAX 6 // max get content from db +#define WT_VAR_UDFNAME 7 + +#define WT_FROM_DB_MAX 7 // max get content from db #define WT_FROM_DB_CNT (WT_FROM_DB_MAX + 1) +#define WT_VAR_ALLTABLE 8 +#define WT_VAR_FUNC 9 +#define WT_VAR_KEYWORD 10 +#define WT_VAR_TBACTION 11 +#define WT_VAR_DBOPTION 12 +#define WT_VAR_ALTER_DBOPTION 13 +#define WT_VAR_DATATYPE 14 +#define WT_VAR_KEYTAGS 15 +#define WT_VAR_ANYWORD 16 +#define WT_VAR_TBOPTION 17 +#define WT_VAR_USERACTION 18 +#define WT_VAR_KEYSELECT 19 +#define WT_VAR_SYSTABLE 20 +#define WT_VAR_LANGUAGE 21 + +#define WT_VAR_CNT 22 + + #define WT_TEXT 0xFF char dbName[256] = ""; // save use database name; @@ -319,13 +325,13 @@ TdThreadMutex tiresMutex; // save thread handle obtain var name from db server TdThread* threads[WT_FROM_DB_CNT]; // obtain var name with sql from server -char varTypes[WT_VAR_CNT][64] = {"", "", "", "", "", - "", "", "", "", "", - "", "", "", "", "", - "", "", "", ""}; +char varTypes[WT_VAR_CNT][64] = { + "", "", "", "", "", "", "", + "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""}; char varSqls[WT_FROM_DB_CNT][64] = {"show databases;", "show stables;", "show tables;", "show dnodes;", - "show users;", "show topics;", "show streams;"}; + "show users;", "show topics;", "show streams;", "show functions;"}; // var words current cursor, if user press any one key except tab, cursorVar can be reset to -1 int cursorVar = -1; @@ -390,7 +396,8 @@ void showHelp() { create qnode on dnode ;\n\ create stream into as select ...\n\ create topic as select ...\n\ - create function ...\n\ + create function as outputtype language \'C\' | \'Python\' ;\n\ + create aggregate function as outputtype bufsize language \'C\' | \'Python\';\n\ create user pass ...\n\ ----- D ----- \n\ describe \n\ @@ -401,7 +408,7 @@ void showHelp() { drop mnode on dnode ;\n\ drop qnode on dnode ;\n\ drop user ;\n\ - drop function ;\n\ + drop function ;\n\ drop consumer group ... \n\ drop topic ;\n\ drop stream ;\n\ @@ -643,6 +650,7 @@ bool shellAutoInit() { GenerateVarType(WT_VAR_USERACTION, user_actions, sizeof(user_actions) / sizeof(char*)); GenerateVarType(WT_VAR_KEYSELECT, key_select, sizeof(key_select) / sizeof(char*)); GenerateVarType(WT_VAR_SYSTABLE, key_systable, sizeof(key_systable) / sizeof(char*)); + GenerateVarType(WT_VAR_LANGUAGE, udf_language, sizeof(udf_language) / sizeof(char*)); return true; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 05183a036c2ac6664d0e3a21900f8e47f3310b39..910b067d4e0bce5e289c136de29470591a446f2b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -102,10 +102,10 @@ int32_t shellRunSingleCommand(char *command) { } #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { - shellRunSingleCommandWebsocketImp(command); + shellRunSingleCommandWebsocketImp(command); } else { #endif - shellRunSingleCommandImp(command); + shellRunSingleCommandImp(command); #ifdef WEBSOCKET } #endif @@ -291,7 +291,9 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { } struct tm ptm = {0}; - taosLocalTime(&tt, &ptm); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { @@ -346,14 +348,23 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + if (tsEnableScience) { + taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); + } else { + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, length)) { - taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); - } else { + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9e", 23, GET_DOUBLE_VAL(val)); taosFprintfFile(pFile, "%s", buf); + } else { + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } } break; case TSDB_DATA_TYPE_BINARY: @@ -541,14 +552,23 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*" PRIu64, width, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%*.5f", width, GET_FLOAT_VAL(val)); + if (tsEnableScience) { + printf("%*e", width, GET_FLOAT_VAL(val)); + } else { + printf("%*.5f", width, GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, width)) { - printf("%*.15e", width, GET_DOUBLE_VAL(val)); + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); + printf("%*s", width, buf); } else { - printf("%s", buf); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, width)) { + printf("%*.15e", width, GET_DOUBLE_VAL(val)); + } else { + printf("%s", buf); + } } break; case TSDB_DATA_TYPE_BINARY: @@ -1025,15 +1045,15 @@ void *shellCancelHandler(void *arg) { } #ifdef WEBSOCKET - if (shell.args.restful || shell.args.cloud) { - shell.stop_query = true; - } else { + if (shell.args.restful || shell.args.cloud) { + shell.stop_query = true; + } else { #endif - if (shell.conn) { - taos_kill_query(shell.conn); - } + if (shell.conn) { + taos_kill_query(shell.conn); + } #ifdef WEBSOCKET - } + } #endif #ifdef WINDOWS printf("\n%s", shell.info.promptHeader); @@ -1083,21 +1103,21 @@ int32_t shellExecute() { SShellArgs *pArgs = &shell.args; #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { - if (shell_conn_ws_server(1)) { - return -1; - } + if (shell_conn_ws_server(1)) { + return -1; + } } else { #endif - if (shell.args.auth == NULL) { - shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); - } else { - shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); - } - - if (shell.conn == NULL) { - fflush(stdout); - return -1; - } + if (shell.args.auth == NULL) { + shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); + } else { + shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); + } + + if (shell.conn == NULL) { + fflush(stdout); + return -1; + } #ifdef WEBSOCKET } #endif @@ -1118,13 +1138,13 @@ int32_t shellExecute() { shellSourceFile(pArgs->file); } #ifdef WEBSOCKET - if (shell.args.restful || shell.args.cloud) { - ws_close(shell.ws_conn); - } else { + if (shell.args.restful || shell.args.cloud) { + ws_close(shell.ws_conn); + } else { #endif - taos_close(shell.conn); + taos_close(shell.conn); #ifdef WEBSOCKET - } + } #endif shellWriteHistory(); @@ -1148,9 +1168,9 @@ int32_t shellExecute() { if (!shell.args.restful && !shell.args.cloud) { #endif #ifndef WINDOWS - printfIntroduction(); + printfIntroduction(); #endif - shellGetGrantInfo(); + shellGetGrantInfo(); #ifdef WEBSOCKET } #endif diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 22b8e89959593e9da18124908fedac31bd6b45ed..bc5809ffe8b389ca57146a65f8c67d451f934b9e 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -45,7 +45,7 @@ void shellCrashHandler(int signum, void *sigInfo, void *context) { int main(int argc, char *argv[]) { shell.exit = false; #ifdef WEBSOCKET - shell.args.timeout = 10; + shell.args.timeout = SHELL_WS_TIMEOUT; shell.args.cloud = true; #endif diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 81c23035e21a225f98e8214bd5c3adcb25add72c..d8920cb4c3d79aacb441bf9a1f512f4a5a4f62b6 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -14,22 +14,51 @@ * along with this program. If not, see . */ #ifdef WEBSOCKET -#include "taosws.h" -#include "shellInt.h" +#include +#include int shell_conn_ws_server(bool first) { - shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); - if (!shell.ws_conn) { - fprintf(stderr, "failed to connect %s, reason: %s\n", - shell.args.dsn, ws_errstr(NULL)); + char cuttedDsn[SHELL_WS_DSN_BUFF] = {0}; + int dsnLen = strlen(shell.args.dsn); + snprintf(cuttedDsn, + ((dsnLen-SHELL_WS_DSN_MASK) > SHELL_WS_DSN_BUFF)? + SHELL_WS_DSN_BUFF:(dsnLen-SHELL_WS_DSN_MASK), + "%s", shell.args.dsn); + fprintf(stdout, "trying to connect %s****** ", cuttedDsn); + fflush(stdout); + for (int i = 0; i < shell.args.timeout; i++) { + shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); + if (NULL == shell.ws_conn) { + int errNo = ws_errno(NULL); + if (0xE001 == errNo) { + fprintf(stdout, "."); + fflush(stdout); + taosMsleep(1000); // sleep 1 second then try again + continue; + } else { + fprintf(stderr, "\nfailed to connect %s***, reason: %s\n", + cuttedDsn, ws_errstr(NULL)); + return -1; + } + } else { + break; + } + } + if (NULL == shell.ws_conn) { + fprintf(stdout, "\n timeout\n"); + fprintf(stderr, "\nfailed to connect %s***, reason: %s\n", + cuttedDsn, ws_errstr(NULL)); return -1; + } else { + fprintf(stdout, "\n"); } if (first && shell.args.restful) { - fprintf(stdout, "successfully connect to %s\n\n", + fprintf(stdout, "successfully connected to %s\n\n", shell.args.dsn); } else if (first && shell.args.cloud) { - fprintf(stdout, "successfully connect to cloud service\n"); + fprintf(stdout, "successfully connected to cloud service\n"); } + fflush(stdout); return 0; } @@ -118,7 +147,8 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { return numOfRows; } -static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute_time) { +static int dumpWebsocketToFile(const char* fname, WS_RES* wres, + double* pexecute_time) { char fullname[PATH_MAX] = {0}; if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { tstrncpy(fullname, fname, PATH_MAX); @@ -150,7 +180,7 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute } taosFprintfFile(pFile, "%s", fields[col].name); } - taosFprintfFile(pFile, "\r\n"); + taosFprintfFile(pFile, "\r\n"); do { uint8_t ty; uint32_t len; @@ -161,7 +191,8 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute taosFprintfFile(pFile, ","); } const void *value = ws_get_value_in_block(wres, i, j, &ty, &len); - shellDumpFieldToFile(pFile, (const char*)value, fields + j, len, precision); + shellDumpFieldToFile(pFile, (const char*)value, + fields + j, len, precision); } taosFprintfFile(pFile, "\r\n"); } @@ -171,7 +202,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute return numOfRows; } -static int shellDumpWebsocket(WS_RES *wres, char *fname, int *error_no, bool vertical, double* pexecute_time) { +static int shellDumpWebsocket(WS_RES *wres, char *fname, + int *error_no, bool vertical, + double* pexecute_time) { int numOfRows = 0; if (fname != NULL) { numOfRows = dumpWebsocketToFile(fname, wres, pexecute_time); @@ -227,13 +260,16 @@ void shellRunSingleCommandWebsocketImp(char *command) { // if it's not a ws connection error if (TSDB_CODE_WS_DSN_ERROR != (code&TSDB_CODE_WS_DSN_ERROR)) { et = taosGetTimestampUs(); - fprintf(stderr, "\nDB: error: %s (%.6fs)\n", ws_errstr(res), (et - st)/1E6); + fprintf(stderr, "\nDB: error: %s (%.6fs)\n", + ws_errstr(res), (et - st)/1E6); ws_free_result(res); return; } - if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { + if (code == TSDB_CODE_WS_SEND_TIMEOUT + || code == TSDB_CODE_WS_RECV_TIMEOUT) { fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); - } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { + } else if (code == TSDB_CODE_WS_INTERNAL_ERRO + || code == TSDB_CODE_WS_CLOSED) { shell.ws_conn = NULL; } ws_free_result(res); @@ -252,7 +288,8 @@ void shellRunSingleCommandWebsocketImp(char *command) { execute_time = ws_take_timing(res)/1E6; } - if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", + REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\r\n\r\n"); fflush(stdout); ws_free_result(res); @@ -266,10 +303,12 @@ void shellRunSingleCommandWebsocketImp(char *command) { double total_time = (et - st)/1E3; double net_time = total_time - (double)execute_time; printf("Query Ok, %d of %d row(s) in database\n", numOfRows, numOfRows); - printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); + printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", + execute_time, net_time, total_time); } else { int error_no = 0; - numOfRows = shellDumpWebsocket(res, fname, &error_no, printMode, &execute_time); + numOfRows = shellDumpWebsocket(res, fname, &error_no, + printMode, &execute_time); if (numOfRows < 0) { ws_free_result(res); return; @@ -279,11 +318,13 @@ void shellRunSingleCommandWebsocketImp(char *command) { double net_time = total_time - execute_time; if (error_no == 0 && !shell.stop_query) { printf("Query OK, %d row(s) in set\n", numOfRows); - printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); + printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", + execute_time, net_time, total_time); } else { printf("Query interrupted, %d row(s) in set (%.6fs)\n", numOfRows, (et - st)/1E6); - printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", execute_time, net_time, total_time); + printf("Execute: %.2f ms Network: %.2f ms Total: %.2f ms\n", + execute_time, net_time, total_time); } } printf("\n"); diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index 6ca266c55533b644c47ad61eb3bebc273ae7be7f..87b0d11d1c00f631c7c09081952af82174e2432f 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -3,8 +3,17 @@ add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) +add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) +add_executable(tmq_offset tmqOffset.c) +target_link_libraries( + tmq_offset + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( create_table PUBLIC taos_static @@ -34,6 +43,14 @@ target_link_libraries( PUBLIC os ) +target_link_libraries( + write_raw_block_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + target_link_libraries( sml_test PUBLIC taos_static diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 73c445417888ba690ebdb9a2886f9f961bb27efd..755ab556251d5f42be27622354f4e950384259fa 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -245,8 +245,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" - }; + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]"}; char *sql1[1] = {0}; for (int i = 0; i < 1; i++) { sql1[i] = taosMemoryCalloc(1, 1024); @@ -774,26 +773,26 @@ int sml_dup_time_Test() { taos_free_result(pRes); const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -851,7 +850,7 @@ int smlProcess_18784_Test() { taos_free_result(pRes); const char *sql[] = { - "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_used=176059i,total=1076048383523889174i 1661943960000000000", "disk,device=sdc inodes_free=66932805i 1661943960000000000", }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); @@ -876,7 +875,7 @@ int smlProcess_18784_Test() { if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); - ASSERT(total == 1081101176832); + ASSERT(total == 1076048383523889174); ASSERT(freed == 66932805); // ASSERT_EQ(latitude, 24.5208); // ASSERT_EQ(longitude, 28.09377); @@ -940,6 +939,11 @@ int sml_ts2164_Test() { // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "ts3038,location=l2a,groupid=ca current=L\"11.8\"", + "ts3038,location=l2a,groupid=ca voltage=L\"221\"", + "ts3038,location=l2a,groupid=ca phase=L\"221\"", +// "qgyltizmkq,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", +// "qgyltizmkq,id=sub_table_0123456,t=3,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64" // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index d105b505793eae17068af9e9db02eff5071f5bd4..ce069c2b05878cb6806f2784be896942cb701ae4 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -597,7 +597,7 @@ void printParaIntoFile() { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm); + taosLocalTime(&tTime, &tm, NULL); taosFprintfFile(pFile, "###################################################################\n"); taosFprintfFile(pFile, "# configDir: %s\n", configDir); diff --git a/utils/test/c/tmqOffset.c b/utils/test/c/tmqOffset.c new file mode 100644 index 0000000000000000000000000000000000000000..7225cb87bdcbf77807c1861d22cef72f955d49bd --- /dev/null +++ b/utils/test/c/tmqOffset.c @@ -0,0 +1,64 @@ +// +// Created by mingming wanng on 2023/4/7. +// +#include +#include +#include "taoserror.h" +#include "tlog.h" +#include "tmsg.h" + +typedef struct { + int32_t size; +} STqOffsetHead; + +int32_t tqOffsetRestoreFromFile(const char* fname) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile != NULL) { + STqOffsetHead head = {0}; + int32_t code; + + while (1) { + if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if (code == 0) { + break; + } else { + printf("code:%d != 0\n", code); + return -1; + } + } + int32_t size = htonl(head.size); + void* memBuf = taosMemoryCalloc(1, size); + if (memBuf == NULL) { + printf("memBuf == NULL\n"); + return -1; + } + if ((code = taosReadFile(pFile, memBuf, size)) != size) { + taosMemoryFree(memBuf); + printf("code:%d != size:%d\n", code, size); + return -1; + } + STqOffset offset; + SDecoder decoder; + tDecoderInit(&decoder, memBuf, size); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + taosMemoryFree(memBuf); + tDecoderClear(&decoder); + printf("tDecodeSTqOffset error\n"); + return -1; + } + + tDecoderClear(&decoder); + printf("subkey:%s, type:%d, uid/version:%"PRId64", ts:%"PRId64"\n", + offset.subKey, offset.val.type, offset.val.uid, offset.val.ts); + taosMemoryFree(memBuf); + } + + taosCloseFile(&pFile); + } + return 0; +} + +int main(int argc, char *argv[]) { + tqOffsetRestoreFromFile("offset-ver0"); + return 0; +} diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index 026067e7f8daca2ec479fcdc729a3f4b0cd4362b..f2de219f4e740b090a87ebbb7e9ceb9cb1c6dcc2 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -166,7 +166,7 @@ static void printHelp() { char* getCurrentTimeString(char* timeString) { time_t tTime = taosGetTimestampSec(); struct tm tm; - taosLocalTime(&tTime, &tm); + taosLocalTime(&tTime, &tm, NULL); sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); @@ -386,7 +386,7 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; pInfo->numOfVgroups++; - taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); + taosFprintfFile(g_fp, "consume id %d, add new vgroupId:%d\n", pInfo->consumerId, vgroupId); if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, pInfo->numOfVgroups, vgroupId); @@ -472,7 +472,9 @@ static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { } struct tm ptm; - taosLocalTime(&tt, &ptm); + if (taosLocalTime(&tt, &ptm, buf) == NULL) { + return buf; + } size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { @@ -578,18 +580,25 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn char buf[1024]; int32_t totalRows = 0; - // printf("topic: %s\n", tmq_get_topic_name(msg)); int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); - taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", - tmq_get_topic_name(msg), vgroupId); + int32_t index = 0; + for (index = 0; index < pInfo->numOfVgroups; index++) { + if (vgroupId == pInfo->rowsOfPerVgroups[index][0]) { + break; + } + } + + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId:%d, currentRows:%d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId, pInfo->rowsOfPerVgroups[index][1]); while (1) { TAOS_ROW row = taos_fetch_row(msg); - - if (row == NULL) break; + if (row == NULL) { + break; + } TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); @@ -607,7 +616,6 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn #endif dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); - taos_print_row(buf, row, fields, numOfFields); if (0 != g_stConfInfo.showRowFlag) { @@ -621,7 +629,6 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn } addRowsToVgroupId(pInfo, vgroupId, totalRows); - return totalRows; } @@ -683,16 +690,17 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { } static int32_t g_once_commit_flag = 0; -static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); - if (0 == g_once_commit_flag) { - g_once_commit_flag = 1; - notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); +static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); + + if (0 == g_once_commit_flag) { + g_once_commit_flag = 1; + notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } - char tmpString[128]; - taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); } void build_consumer(SThreadInfo* pInfo) { @@ -730,9 +738,7 @@ void build_consumer(SThreadInfo* pInfo) { } pInfo->tmq = tmq_consumer_new(conf, NULL, 0); - tmq_conf_destroy(conf); - return; } @@ -817,7 +823,6 @@ void loop_consume(SThreadInfo* pInfo) { } taos_free_result(tmqMsg); - totalMsgs++; int64_t currentPrintTime = taosGetTimestampMs(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 1f25eae366a31b585ca420cc5f289d1cc22c0ecd..c4becdd38196727b0ac3417c028fb4f9d698ae80 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -441,7 +441,7 @@ int32_t init_env() { taos_free_result(pRes); char sql[128] = {0}; - snprintf(sql, 128, "create database if not exists db_taosx vgroups %d", g_conf.dstVgroups); + snprintf(sql, 128, "create database if not exists db_taosx vgroups %d wal_retention_period 3600", g_conf.dstVgroups); pRes = taos_query(pConn, sql); if (taos_errno(pRes) != 0) { printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); @@ -470,7 +470,7 @@ int32_t init_env() { } taos_free_result(pRes); - snprintf(sql, 128, "create database if not exists abc1 vgroups %d", g_conf.srcVgroups); + snprintf(sql, 128, "create database if not exists abc1 vgroups %d wal_retention_period 3600", g_conf.srcVgroups); pRes = taos_query(pConn, sql); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); @@ -542,7 +542,6 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "enable.heartbeat.background", "true"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c new file mode 100644 index 0000000000000000000000000000000000000000..8e5dd6275217b194fa4154fdd1235fdb27e397f1 --- /dev/null +++ b/utils/test/c/write_raw_block_test.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include "taos.h" +#include "types.h" + +int buildStable(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` INT, `voltage` INT, `phase` FLOAT) TAGS " + "(`groupid` INT, `location` VARCHAR(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1 using meters tags(2, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_raw"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_raw vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use db_raw"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildStable(pConn, pRes); + + pRes = taos_query(pConn, "select * from d0"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + void *data = NULL; + int32_t numOfRows = 0; + int error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + ASSERT(error_code == 0); + ASSERT(numOfRows == 1); + + taos_write_raw_block(pConn, numOfRows, data, "d1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "select ts,phase from d0"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + error_code = taos_fetch_raw_block(pRes, &numOfRows, &data); + ASSERT(error_code == 0); + ASSERT(numOfRows == 1); + + int numFields = taos_num_fields(pRes); + TAOS_FIELD *fields = taos_fetch_fields(pRes); + taos_write_raw_block_with_fields(pConn, numOfRows, data, "d2", fields, numFields); + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } +} diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index d82c948cf7b774703faaa8e70ee7b49bde9f8d12..1be28635ef721167d67a33e7837f782c70e17d49 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -37,15 +37,15 @@ void simLogSql(char *sql, bool useSharp) { char *simParseHostName(char *varName) { static char hostName[140]; -#ifdef WINDOWS - hostName[0] = '\"'; - taosGetFqdn(&hostName[1]); - int strEndIndex = strlen(hostName); - hostName[strEndIndex] = '\"'; - hostName[strEndIndex + 1] = '\0'; -#else +//#ifdef WINDOWS +// hostName[0] = '\"'; +// taosGetFqdn(&hostName[1]); +// int strEndIndex = strlen(hostName); +// hostName[strEndIndex] = '\"'; +// hostName[strEndIndex + 1] = '\0'; +//#else sprintf(hostName, "%s", "localhost"); -#endif +//#endif return hostName; } @@ -376,6 +376,20 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } +void simReplaceDirSep (char *buf){ +#ifdef WINDOWS + int i=0; + while(buf[i] != '\0') + { + if(buf[i] == '/') + { + buf[i] = '\\'; + } + i++; + } +#endif +} + bool simReplaceStr(char *buf, char *src, char *dst) { bool replaced = false; char *begin = strstr(buf, src); @@ -407,9 +421,10 @@ bool simExecuteSystemCmd(SScript *script, char *option) { sprintf(buf, "cd %s; ", simScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); #else - sprintf(buf, "%s", simScriptDir); + sprintf(buf, "cd %s && ", simScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); simReplaceStr(buf, ".sh", ".bat"); + simReplaceDirSep(buf); #endif if (useValgrind) { @@ -471,6 +486,8 @@ bool simExecuteSystemContentCmd(SScript *script, char *option) { #ifdef WINDOWS sprintf(buf, "cd %s && ", simScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); + simReplaceStr(buf, ".sh", ".bat"); + simReplaceDirSep(buf); sprintf(buf1, "%s > %s 2>nul", buf, filename); #else sprintf(buf, "cd %s; ", simScriptDir); @@ -755,7 +772,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tt = (*(int64_t *)row[i]) / 1000000000; } - taosLocalTime(&tt, &tp); + if (taosLocalTime(&tt, &tp, timeStr) == NULL) { + break; + } strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); if (precision == TSDB_TIME_PRECISION_MILLI) { sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000));