diff --git a/.gitignore b/.gitignore index c25f60075cc9ce8f681cdefae308411cdb73657d..1c609d3ba36e56c7ab3c91a118ed5340577177fd 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,4 @@ contrib/* !contrib/test sql debug*/ +.env \ No newline at end of file diff --git a/cmake/cmake.install b/cmake/cmake.install index 27edd8e27a3abd89cfe2cdc23a830292b4428764..d9f759217acb35d2fb9318391aac2f15d3c8b25c 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -10,7 +10,9 @@ ELSEIF (TD_WINDOWS) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector) # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .) - INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) + INSTALL(CODE "IF (NOT EXISTS ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) + execute_process(COMMAND ${CMAKE_COMMAND} -E copy ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg ${CMAKE_INSTALL_PREFIX}/cfg/taos.cfg) + ENDIF ()") INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e71598ae5a1255c68c3945ac27c9158390e6e022..9ee0ea526c2697ffba0de90738f92a6d4fdf3efa 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG c9cc20f + GIT_TAG 3c7dafe SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index ad4c150505b8a19ad7185607bc42a76510b06594..de6409a8c693f3ebdf4ea5cff8d8d61e24e47079 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git - GIT_TAG 9fa7e2f + GIT_TAG 29424d5 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/12-taos-sql/12-interval.md b/docs/en/12-taos-sql/12-distinguished.md similarity index 98% rename from docs/en/12-taos-sql/12-interval.md rename to docs/en/12-taos-sql/12-distinguished.md index acfb0de0e1521fd8c6a068497a3df7a17941524c..d2f7cf66b63521d157a6e05f1dd8d93658d65549 100644 --- a/docs/en/12-taos-sql/12-interval.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -1,6 +1,6 @@ --- -sidebar_label: Interval -title: Aggregate by Time Window +sidebar_label: Distinguished +title: Distinguished Query for Time Series Database --- Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. diff --git a/docs/en/12-taos-sql/13-tmq.md b/docs/en/12-taos-sql/13-tmq.md new file mode 100644 index 0000000000000000000000000000000000000000..4d9c475a3829456916175d8a0518c47d67bc18ee --- /dev/null +++ b/docs/en/12-taos-sql/13-tmq.md @@ -0,0 +1,66 @@ +--- +sidebar_label: 消息队列 +title: 消息队列 +--- + +TDengine 3.0.0.0 开始对消息队列做了大幅的优化和增强以简化用户的解决方案。 + +## 创建订阅主题 + +```sql +CREATE TOPIC [IF NOT EXISTS] topic_name AS {subquery | DATABASE db_name | STABLE stb_name }; +``` + +订阅主题包括三种：列订阅、超级表订阅和数据库订阅。 + +**列订阅是**用 subquery 描述，支持过滤和标量函数和 UDF 标量函数，不支持 JOIN、GROUP BY、窗口切分子句、聚合函数和 UDF 聚合函数。列订阅规则如下： + +1. TOPIC 一旦创建则返回结果的字段确定 +2. 被订阅或用于计算的列不可被删除、修改 +3. 列可以新增，但新增的列不出现在订阅结果字段中 +4. 对于 select \*，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列） + +**超级表订阅和数据库订阅**规则如下： + +1. 被订阅主体的 schema 变更不受限 +2. 返回消息中 schema 是块级别的，每块的 schema 可能不一样 +3. 列变更后写入的数据若未落盘，将以写入时的 schema 返回 +4. 列变更后写入的数据若未已落盘，将以落盘时的 schema 返回 + +## 删除订阅主题 + +```sql +DROP TOPIC [IF EXISTS] topic_name; +``` + +此时如果该订阅主题上存在 consumer，则此 consumer 会收到一个错误。 + +## 查看订阅主题 + +## SHOW TOPICS + +```sql +SHOW TOPICS; +``` + +显示当前数据库下的所有主题的信息。 + +## 创建消费组 + +消费组的创建只能通过 TDengine 客户端驱动或者连接器所提供的 API 创建。 + +## 删除消费组 + +```sql +DROP CONSUMER GROUP [IF EXISTS] cgroup_name ON topic_name; +``` + +删除主题 topic_name 上的消费组 cgroup_name。 + +## 查看消费组 + +```sql +SHOW CONSUMERS; +``` + +显示当前数据库下所有活跃的消费者的信息。 diff --git a/docs/en/12-taos-sql/14-limit.md b/docs/en/12-taos-sql/14-limit.md deleted file mode 100644 index e8bb77fc27322e4a72845d6b4cbc68f7e01954c5..0000000000000000000000000000000000000000 --- a/docs/en/12-taos-sql/14-limit.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: Naming & Restrictions ---- - -## Naming Rules - -1. Only characters from the English alphabet, digits and underscore are allowed -2. Names cannot start with a digit -3. Case insensitive without escape character "\`" -4. Identifier with escape character "\`" - To support more flexible table or column names, a new escape character "\`" is introduced. For more details please refer to [escape](/taos-sql/escape). - -## Password Rule - -The legal character set is `[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]`. - -## General Limits - -- Maximum length of database name is 32 bytes, and it can't include "." or special characters. -- Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. -- Maximum length of column name is 64. -- Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. -- Maximum length of tag name is 64. -- Maximum number of tags is 128. There must be at least 1 tag. The total length of tag values should not exceed 16K bytes. -- Maximum length of singe SQL statement is 1048576, i.e. 1 MB. It can be configured in the parameter `maxSQLLength` in the client side, the applicable range is [65480, 1048576]. -- At most 4096 columns can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded. -- Maximum numbers of databases, STables, tables are dependent only on the system resources. -- Maximum number of replicas for a database is 3. -- Maximum length of user name is 23 bytes. -- Maximum length of password is 15 bytes. -- Maximum number of rows depends only on the storage space. -- Maximum number of vnodes for a single database is 1024. - -## Restrictions of Table/Column Names - -### Name Restrictions of Table/Column - -The name of a table or column can only be composed of ASCII characters, digits and underscore and it cannot start with a digit. The maximum length is 192 bytes. Names are case insensitive. The name mentioned in this rule doesn't include the database name prefix and the separator. - -### Name Restrictions After Escaping - -To support more flexible table or column names, new escape character "\`" is introduced in TDengine to avoid the conflict between table name and keywords and break the above restrictions for table names. The escape character is not counted in the length of table name. - -With escaping, the string inside escape characters are case sensitive, i.e. will not be converted to lower case internally. - -For example: -\`aBc\` and \`abc\` are different table or column names, but "abc" and "aBc" are same names because internally they are all "abc". - -:::note -The characters inside escape characters must be printable characters. - -::: diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md new file mode 100644 index 0000000000000000000000000000000000000000..7ff7da2bfb82e282cefb1a554283860d0e683de2 --- /dev/null +++ b/docs/en/12-taos-sql/14-stream.md @@ -0,0 +1,122 @@ +--- +sidebar_label: 流式计算 +title: 流式计算 +--- + +在时序数据的处理中，经常要对原始数据进行清洗、预处理，再使用时序数据库进行长久的储存。用户通常需要在时序数据库之外再搭建 Kafka、Flink、Spark 等流计算处理引擎，增加了用户的开发成本和维护成本。 + +使用 TDengine 3.0 的流式计算引擎能够最大限度的减少对这些额外中间件的依赖，真正将数据的写入、预处理、长期存储、复杂分析、实时计算、实时报警触发等功能融为一体，并且，所有这些任务只需要使用 SQL 完成，极大降低了用户的学习成本、使用成本。 + +## 创建流式计算 + +```sql +CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery +stream_options: { + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time +} + +``` + +其中 subquery 是 select 普通查询语法的子集: + +```sql +subquery: SELECT [DISTINCT] select_list + from_clause + [WHERE condition] + [PARTITION BY tag_list] + [window_clause] + [group_by_clause] +``` + +不支持 order_by，limit，slimit，fill 语句 + +例如，如下语句创建流式计算，同时自动创建名为 avg_vol 的超级表，此流计算以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。 + +```sql +CREATE STREAM avg_vol_s INTO avg_vol AS +SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); +``` + +## 删除流式计算 + +```sql +DROP STREAM [IF NOT EXISTS] stream_name +``` + +仅删除流式计算任务，由流式计算写入的数据不会被删除。 + +## 展示流式计算 + +```sql +SHOW STREAMS; +``` + +## 流式计算的触发模式 + +在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 + +对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式： + +1. AT_ONCE：写入立即触发 + +2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用，详见《流式计算的乱序数据容忍策略》） + +3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。 + +由于窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，则事件时间无法更新，可能导致无法得到最新的计算结果。 + +因此，流式计算提供了以事件时间结合处理时间计算的 MAX_DELAY 触发模式。 + +MAX_DELAY 模式在窗口关闭时会立即触发计算。此外，当数据写入后，计算触发的时间超过 max delay 指定的时间，则立即触发计算 + +## 流式计算的乱序数据容忍策略 + +在创建流时，可以在 stream_option 中指定 watermark。 + +流式计算通过 watermark 来度量对乱序数据的容忍程度，watermark 默认为 0。 + +T = 最新事件时间 - watermark + +每批到来的数据都会以上述公式更新窗口关闭时间，并将窗口结束时间 < T 的所有打开的窗口关闭，若触发模式为 WINDOW_CLOSE 或 MAX_DELAY，则推送窗口聚合结果。 + +流式计算的过期数据处理策略 +对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据，对于过期数据，流式计算提供两种处理方式： + +1. 直接丢弃：这是常见流式计算引擎提供的默认（甚至是唯一）计算模式 + +2. 重新计算：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 + +无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 + +## 流式计算的数据填充策略 + +TODO + +## 流式计算与会话窗口（session window） + +```sql +window_clause: { + SESSION(ts_col, tol_val) + | STATE_WINDOW(col) + | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)] +} +``` + +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 + +## 流式计算的监控与流任务分布查询 + +TODO + +## 流式计算的内存控制与存算分离 + +TODO + +## 流式计算的暂停与恢复 + +```sql +STOP STREAM stream_name; + +RESUME STREAM stream_name; +``` diff --git a/docs/en/12-taos-sql/13-operators.md b/docs/en/12-taos-sql/16-operators.md similarity index 100% rename from docs/en/12-taos-sql/13-operators.md rename to docs/en/12-taos-sql/16-operators.md diff --git a/docs/en/12-taos-sql/16-json.md b/docs/en/12-taos-sql/17-json.md similarity index 100% rename from docs/en/12-taos-sql/16-json.md rename to docs/en/12-taos-sql/17-json.md diff --git a/docs/en/12-taos-sql/19-limit.md b/docs/en/12-taos-sql/19-limit.md new file mode 100644 index 0000000000000000000000000000000000000000..ff552fc9771f5b428554acc62e9aeac03a305ecc --- /dev/null +++ b/docs/en/12-taos-sql/19-limit.md @@ -0,0 +1,59 @@ +--- +sidebar_label: 命名与边界限制 +title: 命名与边界限制 +--- + +## 名称命名规则 + +1. 合法字符：英文字符、数字和下划线 +2. 允许英文字符或下划线开头，不允许以数字开头 +3. 不区分大小写 +4. 转义后表（列）名规则： + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查 + 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一 + + 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 + 需要注意的是转义字符中的内容必须是可打印字符。 + +## 密码合法字符集 + +`[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]` + +去掉了 `` ‘“`\ `` (单双引号、撇号、反斜杠、空格) + +## 一般限制 + +- 数据库名最大长度为 32 +- 表名最大长度为 192，不包括数据库名前缀和分隔符 +- 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置） +- 列名最大长度为 64 +- 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 +- 标签名最大长度为 64 +- 最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB +- SQL 语句最大长度 1048576 个字符，也可通过客户端配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576 +- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错 +- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 +- 数据库的副本数只能设置为 1 或 3 +- 用户名的最大长度是 23 个字节 +- 用户密码的最大长度是 15 个字节 +- 总数据行数取决于可用资源 +- 单个数据库的虚拟结点数上限为 1024 + +## 表(列)名合法性说明 + +### TDengine 中的表（列）名命名规则如下： + +只能由字母、数字、下划线构成，数字不能在首位，长度不能超过 192 字节，不区分大小写。这里表名称不包括数据库名的前缀和分隔符。 + +### 转义后表（列）名规则： + +为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"，可以避免表名与关键词的冲突，同时不受限于上述表名合法性约束检查，转义符不计入表名的长度。 +转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 + +例如： +\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 + +:::note +转义字符中的内容必须是可打印字符。 + +::: diff --git a/docs/en/12-taos-sql/21-node.md b/docs/en/12-taos-sql/21-node.md new file mode 100644 index 0000000000000000000000000000000000000000..4816daf42042c0607aebf37c8b57961e5b1927fe --- /dev/null +++ b/docs/en/12-taos-sql/21-node.md @@ -0,0 +1,154 @@ +--- +sidebar_label: 集群管理 +title: 集群管理 +--- + +组成 TDengine 集群的物理实体是 dnode (data node 的缩写)，它是一个运行在操作系统之上的进程。在 dnode 中可以建立负责时序数据存储的 vnode (virtual node)，在多节点集群环境下当某个数据库的 replica 为 3 时，该数据库中的每个 vgroup 由 3 个 vnode 组成；当数据库的 replica 为 1 时，该数据库中的每个 vgroup 由 1 个 vnode 组成。如果要想配置某个数据库为多副本，则集群中的 dnode 数量至少为 3。在 dnode 还可以创建 mnode (management node)，单个集群中最多可以创建三个 mnode。在 TDengine 3.0.0.0 中为了支持存算分离，引入了一种新的逻辑节点 qnode (query node)，qnode 和 vnode 既可以共存在一个 dnode 中，也可以完全分离在不同的 dnode 上。 + +## 创建数据节点 + +```sql +CREATE DNODE {dnode_endpoint | dnode_host_name PORT port_val} +``` + +其中 `dnode_endpoint` 是形成 `hostname:port`的格式。也可以分开指定 hostname 和 port。 + +实际操作中推荐先创建 dnode，再启动相应的 dnode 进程，这样该 dnode 就可以立即根据其配置文件中的 firstEP 加入集群。每个 dnode 在加入成功后都会被分配一个 ID。 + +## 查看数据节点 + +```sql +SHOW DNODES; +``` + +可以列出集群中所有的数据节点，所列出的字段有 dnode 的 ID, endpoint, status。 + +## 删除数据节点 + +```sql +DROP DNODE {dnode_id | dnode_endpoint} +``` + +可以用 dnoe_id 或 endpoint 两种方式从集群中删除一个 dnode。注意删除 dnode 不等于停止相应的进程。实际中推荐先将一个 dnode 删除之后再停止其所对应的进程。 + +## 修改数据节点配置 + +```sql +ALTER DNODE dnode_id dnode_option + +ALTER ALL DNODES dnode_option + +dnode_option: { + 'resetLog' + | 'balance' value + | 'monitor' value + | 'debugFlag' value + | 'monDebugFlag' value + | 'vDebugFlag' value + | 'mDebugFlag' value + | 'cDebugFlag' value + | 'httpDebugFlag' value + | 'qDebugflag' value + | 'sdbDebugFlag' value + | 'uDebugFlag' value + | 'tsdbDebugFlag' value + | 'sDebugflag' value + | 'rpcDebugFlag' value + | 'dDebugFlag' value + | 'mqttDebugFlag' value + | 'wDebugFlag' value + | 'tmrDebugFlag' value + | 'cqDebugFlag' value +} +``` + +上面语法中的这些可修改配置项其配置方式与 dnode 配置文件中的配置方式相同，区别是修改是动态的立即生效，且不需要重启 dnode。 + +## 添加管理节点 + +```sql +CREATE MNODE ON DNODE dnode_id +``` + +系统启动默认在 firstEP 节点上创建一个 MNODE，用户可以使用此语句创建更多的 MNODE 来提高系统可用性。一个集群最多存在三个 MNODE，一个 DNODE 上只能创建一个 MNODE。 + +## 查看管理节点 + +```sql +SHOW MNODES; +``` + +列出集群中所有的管理节点，包括其 ID，所在 DNODE 以及状态。 + +## 删除管理节点 + +```sql +DROP MNODE ON DNODE dnode_id; +``` + +删除 dnode_id 所指定的 DNODE 上的 MNODE。 + +## 创建查询节点 + +```sql +CREATE QNODE ON DNODE dnode_id; +``` + +系统启动默认没有 QNODE，用户可以创建 QNODE 来实现计算和存储的分离。一个 DNODE 上只能创建一个 QNODE。一个 DNODE 的 `supportVnodes` 参数如果不为 0，同时又在其上创建上 QNODE，则在该 dnode 中既有负责存储管理的 vnode 又有负责查询计算的 qnode，如果还在该 dnode 上创建了 mnode，则一个 dnode 上最多三种逻辑节点都可以存在。但通过配置也可以使其彻底分离。将一个 dnode 的`supportVnodes`配置为 0，可以选择在其上创建 mnode 或者 qnode 中的一种，这样可以实现三种逻辑节点在物理上的彻底分离。 + +## 查看查询节点 + +```sql +SHOW QNODES; +``` + +列出集群中所有查询节点，包括 ID，及所在 DNODE。 + +## 删除查询节点 + +```sql +DROP QNODE ON DNODE dnode_id; +``` + +删除 ID 为 dnode_id 的 DNODE 上的 QNODE，但并不会影响该 dnode 的状态。 + +## 修改客户端配置 + +如果将客户端也看作广义的集群的一部分，可以通过如下命令动态修改客户端配置参数。 + +```sql +ALTER LOCAL local_option + +local_option: { + 'resetLog' + | 'rpcDebugFlag' value + | 'tmrDebugFlag' value + | 'cDebugFlag' value + | 'uDebugFlag' value + | 'debugFlag' value +} +``` + +上面语法中的参数与在配置文件中配置客户端的用法相同，但不需要重启客户端，修改后立即生效。 + +## 查看客户端配置 + +```sql +SHOW LOCAL VARIABLES; +``` + +## 合并 vgroup + +```sql +MERGE VGROUP vgroup_no1 vgroup_no2; +``` + +如果在系统实际运行一段时间后，因为不同时间线的数据特征不同导致在 vgroups 之间的数据和负载分布不均衡，可以通过合并或拆分 vgroups 的方式逐步实现负载均衡。 + +## 拆分 vgroup + +```sql +SPLIT VGROUP vgroup_no; +``` + +会创建一个新的 vgroup，并将指定 vgroup 中的数据按照一致性 HASH 迁移一部分到新的 vgroup 中。此过程中，原 vgroup 可以正常提供读写服务。 diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md new file mode 100644 index 0000000000000000000000000000000000000000..1e178706859a3e5fa5dbabc00777b92639d76617 --- /dev/null +++ b/docs/en/12-taos-sql/22-meta.md @@ -0,0 +1,247 @@ +--- +sidebar_label: 元数据库 +title: 元数据库 +--- + +TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数据库元数据、数据库系统信息和状态的访问，例如数据库或表的名称，当前执行的 SQL 语句等。该数据库存储有关 TDengine 维护的所有其他数据库的信息。它包含多个只读表。实际上，这些表都是视图，而不是基表，因此没有与它们关联的文件。所以对这些表只能查询，不能进行 INSERT 等写入操作。`INFORMATION_SCHEMA` 数据库旨在以一种更一致的方式来提供对 TDengine 支持的各种 SHOW 语句（如 SHOW TABLES、SHOW DATABASES）所提供的信息的访问。与 SHOW 语句相比，使用 SELECT ... FROM INFORMATION_SCHEMA.tablename 具有以下优点： + +1. 可以使用 USE 语句将 INFORMATION_SCHEMA 设为默认数据库 +2. 可以使用 SELECT 语句熟悉的语法，只需要学习一些表名和列名 +3. 可以对查询结果进行筛选、排序等操作。事实上，可以使用任意 TDengine 支持的 SELECT 语句对 INFORMATION_SCHEMA 中的表进行查询 +4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列，而不用担心对既有业务系统造成影响 +5. 与其他数据库系统更具互操作性。例如，Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 + +Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们仍然被保留。 + +本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 + +## INS_DNODES + +提供 dnode 的相关信息。也可以使用 SHOW DNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ------------------------- | +| 1 | vnodes | SMALLINT | dnode 中的实际 vnode 个数 | +| 2 | support_vnodes | SMALLINT | 最多支持的 vnode 个数 | +| 3 | status | BINARY(10) | 当前状态 | +| 4 | note | BINARY(256) | 离线原因等信息 | +| 5 | id | SMALLINT | dnode id | +| 6 | endpoint | BINARY(134) | dnode 的地址 | +| 7 | create | TIMESTAMP | 创建时间 | + +## INS_MNODES + +提供 mnode 的相关信息。也可以使用 SHOW MNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------ | +| 1 | id | SMALLINT | mnode id | +| 2 | endpoint | BINARY(134) | mnode 的地址 | +| 3 | role | BINARY(10) | 当前角色 | +| 4 | role_time | TIMESTAMP | 成为当前角色的时间 | +| 5 | create_time | TIMESTAMP | 创建时间 | + +## INS_MODULES + +提供组件的相关信息。也可以使用 SHOW MODULES 来查询这些信息 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ---------- | +| 1 | id | SMALLINT | module id | +| 2 | endpoint | BINARY(134) | 组件的地址 | +| 3 | module | BINARY(10) | 组件状态 | + +## INS_QNODES + +当前系统中 QNODE 的信息。也可以使用 SHOW QNODES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------ | +| 1 | id | SMALLINT | qnode id | +| 2 | endpoint | BINARY(134) | qnode 的地址 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## INS_CLUSTER + +存储集群相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ---------- | +| 1 | id | BIGINT | cluster id | +| 2 | name | BINARY(134) | 集群名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## INS_DATABASES + +提供用户创建的数据库对象的相关信息。也可以使用 SHOW DATABASES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------------: | ---------------- | ------------------------------------------------ | +| 1 | name | BINARY(32) | 数据库名 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | ntables | INT | 数据库中表的数量，包含子表和普通表但不包含超级表 | +| 4 | vgroups | INT | 数据库中有多少个 vgroup | +| 6 | replica | INT | 副本数 | +| 7 | quorum | BINARY(3) | 强一致性 | +| 8 | duration | INT | 单文件存储数据的时间跨度 | +| 9 | keep | INT | 数据保留时长 | +| 10 | buffer | INT | 每个 vnode 写缓存的内存块大小，单位 MB | +| 11 | pagesize | INT | 每个 VNODE 中元数据存储引擎的页大小，单位为 KB | +| 12 | pages | INT | 每个 vnode 元数据存储引擎的缓存页个数 | +| 13 | minrows | INT | 文件块中记录的最大条数 | +| 14 | maxrows | INT | 文件块中记录的最小条数 | +| 15 | comp | INT | 数据压缩方式 | +| 16 | precision | BINARY(2) | 时间分辨率 | +| 17 | status | BINARY(10) | 数据库状态 | +| 18 | retention | BINARY (60) | 数据的聚合周期和保存时长 | +| 19 | single_stable | BOOL | 表示此数据库中是否只可以创建一个超级表 | +| 20 | cachemodel | BINARY(60) | 表示是否在内存中缓存子表的最近数据 | +| 21 | cachesize | INT | 表示每个 vnode 中用于缓存子表最近数据的内存大小 | +| 22 | wal_level | INT | WAL 级别 | +| 23 | wal_fsync_period | INT | 数据落盘周期 | +| 24 | wal_retention_period | INT | WAL 的保存时长 | +| 25 | wal_retention_size | INT | WAL 的保存上限 | +| 26 | wal_roll_period | INT | wal 文件切换时长 | +| 27 | wal_segment_size | wal 单个文件大小 | + +## INS_FUNCTIONS + +用户创建的自定义函数的信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------- | +| 1 | name | BINARY(64) | 函数名 | +| 2 | comment | BINARY(255) | 补充说明 | +| 3 | aggregate | INT | 是否为聚合函数 | +| 4 | output_type | BINARY(31) | 输出类型 | +| 5 | create_time | TIMESTAMP | 创建时间 | +| 6 | code_len | INT | 代码长度 | +| 7 | bufsize | INT | buffer 大小 | + +## INS_INDEXES + +提供用户创建的索引的相关信息。也可以使用 SHOW INDEX 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | ---------------------------------------------------------------------------------- | +| 1 | db_name | BINARY(32) | 包含此索引的表所在的数据库名 | +| 2 | table_name | BINARY(192) | 包含此索引的表的名称 | +| 3 | index_name | BINARY(192) | 索引名 | +| 4 | column_name | BINARY(64) | 建索引的列的列名 | +| 5 | index_type | BINARY(10) | 目前有 SMA 和 FULLTEXT | +| 6 | index_extensions | BINARY(256) | 索引的额外信息。对 SMA 类型的索引，是函数名的列表。对 FULLTEXT 类型的索引为 NULL。 | + +## INS_STABLES + +提供用户创建的超级表的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ------------------------ | +| 1 | stable_name | BINARY(192) | 超级表表名 | +| 2 | db_name | BINARY(64) | 超级表所在的数据库的名称 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | tags | INT | 标签数目 | +| 6 | last_update | TIMESTAMP | 最后更新时间 | +| 7 | table_comment | BINARY(1024) | 表注释 | +| 8 | watermark | BINARY(64) | 窗口的关闭时间 | +| 9 | max_delay | BINARY(64) | 推送计算结果的最大延迟 | +| 10 | rollup | BINARY(128) | rollup 聚合函数 | + +## INS_TABLES + +提供用户创建的普通表和子表的相关信息 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-----------: | ------------ | ---------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 数据库名 | +| 3 | create_time | TIMESTAMP | 创建时间 | +| 4 | columns | INT | 列数目 | +| 5 | stable_name | BINARY(192) | 所属的超级表表名 | +| 6 | uid | BIGINT | 表 id | +| 7 | vgroup_id | INT | vgroup id | +| 8 | ttl | INT | 表的生命周期 | +| 9 | table_comment | BINARY(1024) | 表注释 | +| 10 | type | BINARY(20) | 表类型 | + +## INS_TAGS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------- | ---------------------- | +| 1 | table_name | BINARY(192) | 表名 | +| 2 | db_name | BINARY(64) | 该表所在的数据库的名称 | +| 3 | stable_name | BINARY(192) | 所属的超级表表名 | +| 4 | tag_name | BINARY(64) | tag 的名称 | +| 5 | tag_type | BINARY(64) | tag 的类型 | +| 6 | tag_value | BINARY(16384) | tag 的值 | + +## INS_USERS + +提供系统中创建的用户的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------- | +| 1 | user_name | BINARY(23) | 用户名 | +| 2 | privilege | BINARY(256) | 权限 | +| 3 | create_time | TIMESTAMP | 创建时间 | + +## INS_GRANTS + +提供企业版授权的相关信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------------------------------------------- | +| 1 | version | BINARY(9) | 企业版授权说明：official(官方授权的)/trial(试用的) | +| 2 | cpu_cores | BINARY(9) | 授权使用的 CPU 核心数量 | +| 3 | dnodes | BINARY(10) | 授权使用的 dnode 节点数量 | +| 4 | streams | BINARY(10) | 授权创建的流数量 | +| 5 | users | BINARY(10) | 授权创建的用户数量 | +| 6 | accounts | BINARY(10) | 授权创建的帐户数量 | +| 7 | storage | BINARY(21) | 授权使用的存储空间大小 | +| 8 | connections | BINARY(21) | 授权使用的客户端连接数量 | +| 9 | databases | BINARY(11) | 授权使用的数据库数量 | +| 10 | speed | BINARY(9) | 授权使用的数据点每秒写入数量 | +| 11 | querytime | BINARY(9) | 授权使用的查询总时长 | +| 12 | timeseries | BINARY(21) | 授权使用的测点数量 | +| 13 | expired | BINARY(5) | 是否到期，true：到期，false：未到期 | +| 14 | expire_time | BINARY(19) | 试用期到期时间 | + +## INS_VGROUPS + +系统中所有 vgroups 的信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :-------: | ------------ | ------------------------------------------------------ | +| 1 | vgroup_id | INT | vgroup id | +| 2 | db_name | BINARY(32) | 数据库名 | +| 3 | tables | INT | 此 vgroup 内有多少表 | +| 4 | status | BINARY(10) | 此 vgroup 的状态 | +| 5 | v1_dnode | INT | 第一个成员所在的 dnode 的 id | +| 6 | v1_status | BINARY(10) | 第一个成员的状态 | +| 7 | v2_dnode | INT | 第二个成员所在的 dnode 的 id | +| 8 | v2_status | BINARY(10) | 第二个成员的状态 | +| 9 | v3_dnode | INT | 第三个成员所在的 dnode 的 id | +| 10 | v3_status | BINARY(10) | 第三个成员的状态 | +| 11 | nfiles | INT | 此 vgroup 中数据/元数据文件的数量 | +| 12 | file_size | INT | 此 vgroup 中数据/元数据文件的大小 | +| 13 | tsma | TINYINT | 此 vgroup 是否专用于 Time-range-wise SMA，1: 是, 0: 否 | + +## INS_CONFIGS + +系统配置参数。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ------------ | +| 1 | name | BINARY(32) | 配置项名称 | +| 2 | value | BINARY(64) | 该配置项的值 | + +## INS_DNODE_VARIABLES + +系统中每个 dnode 的配置参数。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :------: | ------------ | ------------ | +| 1 | dnode_id | INT | dnode 的 ID | +| 2 | name | BINARY(32) | 配置项名称 | +| 3 | value | BINARY(64) | 该配置项的值 | diff --git a/docs/zh/12-taos-sql/23-show.md b/docs/en/12-taos-sql/24-show.md similarity index 100% rename from docs/zh/12-taos-sql/23-show.md rename to docs/en/12-taos-sql/24-show.md diff --git a/docs/en/12-taos-sql/25-grant.md b/docs/en/12-taos-sql/25-grant.md new file mode 100644 index 0000000000000000000000000000000000000000..0c290350cc155e975e5a817c991bebc74944cd04 --- /dev/null +++ b/docs/en/12-taos-sql/25-grant.md @@ -0,0 +1,94 @@ +--- +sidebar_label: 权限管理 +title: 权限管理 +--- + +本节讲述如何在 TDengine 中进行权限管理的相关操作。 + +## 创建用户 + +```sql +CREATE USER use_name PASS password; +``` + +创建用户。 + +use_name最长为23字节。 + +password最长为128字节，合法字符包括"a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/"，不可以出现单双引号、撇号、反斜杠和空格，且不可以为空。 + +## 删除用户 + +```sql +DROP USER user_name; +``` + +## 修改用户信息 + +```sql +ALTER USER user_name alter_user_clause + +alter_user_clause: { + PASS 'literal' + | ENABLE value + | SYSINFO value +} +``` + +- PASS：修改用户密码。 +- ENABLE：修改用户是否启用。1表示启用此用户，0表示禁用此用户。 +- SYSINFO：修改用户是否可查看系统信息。1表示可以查看系统信息，0表示不可以查看系统信息。 + + +## 授权 + +```sql +GRANT privileges ON priv_level TO user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.* + | *.* +} +``` + +对用户授权。 + +授权级别支持到DATABASE，权限有READ和WRITE两种。 + +TDengine 有超级用户和普通用户两类用户。超级用户缺省创建为root，拥有所有权限。使用超级用户创建出来的用户为普通用户。在未授权的情况下，普通用户可以创建DATABASE，并拥有自己创建的DATABASE的所有权限，包括删除数据库、修改数据库、查询时序数据和写入时序数据。超级用户可以给普通用户授予其他DATABASE的读写权限，使其可以在此DATABASE上读写数据，但不能对其进行删除和修改数据库的操作。 + +对于非DATABASE的对象，如USER、DNODE、UDF、QNODE等，普通用户只有读权限（一般为SHOW命令），不能创建和修改。 + +## 撤销授权 + +```sql +REVOKE privileges ON priv_level FROM user_name + +privileges : { + ALL + | priv_type [, priv_type] ... +} + +priv_type : { + READ + | WRITE +} + +priv_level : { + dbname.* + | *.* +} + +``` + +收回对用户的授权。 \ No newline at end of file diff --git a/docs/en/12-taos-sql/26-udf.md b/docs/en/12-taos-sql/26-udf.md new file mode 100644 index 0000000000000000000000000000000000000000..bd8d61a5844241efae9eee99a73c65afd3d0926f --- /dev/null +++ b/docs/en/12-taos-sql/26-udf.md @@ -0,0 +1,28 @@ +--- +sidebar_label: 自定义函数 +title: 用户自定义函数 +--- + +除了 TDengine 的内置函数以外，用户还可以编写自己的函数逻辑并加入TDengine系统中。 + +## 创建函数 + +```sql +CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +``` + +语法说明： + +AGGREGATE：标识此函数是标量函数还是聚集函数。 +func_name：函数名，必须与函数实现中udfNormalFunc的实际名称一致。 +library_path：包含UDF函数实现的动态链接库的绝对路径，是在客户端侧主机上的绝对路径。 +OUTPUTTYPE：标识此函数的返回类型。 +BUFSIZE：中间结果的缓冲区大小，单位是字节。不设置则默认为0。最大不可超过512字节。 + +关于如何开发自定义函数，请参考 [UDF使用说明](../../develop/udf)。 + +## 删除自定义函数 + +```sql +DROP FUNCTION func_name +``` \ No newline at end of file diff --git a/docs/en/12-taos-sql/27-index.md b/docs/en/12-taos-sql/27-index.md new file mode 100644 index 0000000000000000000000000000000000000000..2c0907723e76f304566e6a19bdef2d63225f903f --- /dev/null +++ b/docs/en/12-taos-sql/27-index.md @@ -0,0 +1,47 @@ +--- +sidebar_label: 索引 +title: 使用索引 +--- + +TDengine 从 3.0.0.0 版本开始引入了索引功能，支持 SMA 索引和 FULLTEXT 索引。 + +## 创建索引 + +```sql +CREATE FULLTEXT INDEX index_name ON tb_name (col_name [, col_name] ...) + +CREATE SMA INDEX index_name ON tb_name index_option + +index_option: + FUNCTION(functions) INTERVAL(interval_val [, interval_offset]) [SLIDING(sliding_val)] [WATERMARK(watermark_val)] [MAX_DELAY(max_delay_val)] + +functions: + function [, function] ... +``` + +### SMA 索引 + +对指定列按 INTERVAL 子句定义的时间窗口创建进行预聚合计算，预聚合计算类型由 functions_string 指定。SMA 索引能提升指定时间段的聚合查询的性能。目前，限制一个超级表只能创建一个 SMA INDEX。 + +- 支持的函数包括 MAX、MIN 和 SUM。 +- WATERMARK: 最小单位毫秒，取值范围 [0ms, 900000ms]，默认值为 5 秒，只可用于超级表。 +- MAX_DELAY: 最小单位毫秒，取值范围 [1ms, 900000ms]，默认值为 interval 的值(但不能超过最大值)，只可用于超级表。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。 + +### FULLTEXT 索引 + +对指定列建立文本索引，可以提升含有文本过滤的查询的性能。FULLTEXT 索引不支持 index_option 语法。现阶段只支持对 JSON 类型的标签列创建 FULLTEXT 索引。不支持多列联合索引，但可以为每个列分布创建 FULLTEXT 索引。 + +## 删除索引 + +```sql +DROP INDEX index_name; +``` + +## 查看索引 + +````sql +```sql +SHOW INDEXES FROM tbl_name [FROM db_name]; +```` + +显示在所指定的数据库或表上已创建的索引。 diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md new file mode 100644 index 0000000000000000000000000000000000000000..72b220b8ff44917831ac16301237702c991b9b15 --- /dev/null +++ b/docs/en/12-taos-sql/28-recovery.md @@ -0,0 +1,38 @@ +--- +sidebar_label: 异常恢复 +title: 异常恢复 +--- + +在一个复杂的应用场景中，连接和查询任务等有可能进入一种错误状态或者耗时过长迟迟无法结束，此时需要有能够终止这些连接或任务的方法。 + +## 终止连接 + +```sql +KILL CONNECTION conn_id; +``` + +conn_id 可以通过 `SHOW CONNECTIONS` 获取。 + +## 终止查询 + +```sql +SHOW QUERY query_id; +``` + +query_id 可以通过 `SHOW QUERIES` 获取。 + +## 终止事务 + +```sql +KILL TRANSACTION trans_id +``` + +trans_id 可以通过 `SHOW TRANSACTIONS` 获取。 + +## 重置客户端缓存 + +```sql +RESET QUERY CACHE; +``` + +如果在多客户端情况下出现元数据不同步的情况，可以用这条命令强制清空客户端缓存，随后客户端会从服务端拉取最新的元数据。 diff --git a/docs/examples/csharp/AsyncQueryExample.cs b/docs/examples/csharp/AsyncQueryExample.cs index 3dabbebd1630a207af2e1b1b11cc4ba15bdd94a9..0d47325932e2f01fec8d55cfdb64c636258f4a03 100644 --- a/docs/examples/csharp/AsyncQueryExample.cs +++ b/docs/examples/csharp/AsyncQueryExample.cs @@ -1,4 +1,7 @@ +using System; +using System.Collections.Generic; using TDengineDriver; +using TDengineDriver.Impl; using System.Runtime.InteropServices; namespace TDengineExample @@ -19,8 +22,8 @@ namespace TDengineExample { if (code == 0 && taosRes != IntPtr.Zero) { - FetchRowAsyncCallback fetchRowAsyncCallback = new FetchRowAsyncCallback(FetchRowCallback); - TDengine.FetchRowAsync(taosRes, fetchRowAsyncCallback, param); + FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback); + TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param); } else { @@ -28,179 +31,44 @@ namespace TDengineExample } } - static void FetchRowCallback(IntPtr param, IntPtr taosRes, int numOfRows) + // Iteratively call this interface until "numOfRows" is no greater than 0. + static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows) { if (numOfRows > 0) { Console.WriteLine($"{numOfRows} rows async retrieved"); - DisplayRes(taosRes); - TDengine.FetchRowAsync(taosRes, FetchRowCallback, param); + IntPtr pdata = TDengine.GetRawBlock(taosRes); + List metaList = TDengine.FetchFields(taosRes); + List dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows); + + for (int i = 0; i < dataList.Count; i++) + { + if (i != 0 && (i+1) % metaList.Count == 0) + { + Console.WriteLine("{0}\t|", dataList[i]); + } + else + { + Console.Write("{0}\t|", dataList[i]); + } + } + Console.WriteLine(""); + TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param); } else { if (numOfRows == 0) { Console.WriteLine("async retrieve complete."); - } else { - Console.WriteLine($"FetchRowAsync callback error, error code {numOfRows}"); + Console.WriteLine($"FetchRawBlockCallback callback error, error code {numOfRows}"); } TDengine.FreeResult(taosRes); } } - public static void DisplayRes(IntPtr res) - { - if (!IsValidResult(res)) - { - TDengine.Cleanup(); - System.Environment.Exit(1); - } - - List metaList = TDengine.FetchFields(res); - int fieldCount = metaList.Count; - // metaList.ForEach((item) => { Console.Write("{0} ({1}) \t|\t", item.name, item.size); }); - - List dataList = QueryRes(res, metaList); - for (int index = 0; index < dataList.Count; index++) - { - if (index % fieldCount == 0 && index != 0) - { - Console.WriteLine(""); - } - Console.Write("{0} \t|\t", dataList[index].ToString()); - - } - Console.WriteLine(""); - } - - public static bool IsValidResult(IntPtr res) - { - if ((res == IntPtr.Zero) || (TDengine.ErrorNo(res) != 0)) - { - if (res != IntPtr.Zero) - { - Console.Write("reason: " + TDengine.Error(res)); - return false; - } - Console.WriteLine(""); - return false; - } - return true; - } - - private static List QueryRes(IntPtr res, List meta) - { - IntPtr taosRow; - List dataRaw = new(); - while ((taosRow = TDengine.FetchRows(res)) != IntPtr.Zero) - { - dataRaw.AddRange(FetchRow(taosRow, res)); - } - if (TDengine.ErrorNo(res) != 0) - { - Console.Write("Query is not complete, Error {0} {1}", TDengine.ErrorNo(res), TDengine.Error(res)); - } - TDengine.FreeResult(res); - Console.WriteLine(""); - return dataRaw; - } - - public static List FetchRow(IntPtr taosRow, IntPtr taosRes)//, List metaList, int numOfFiled - { - List metaList = TDengine.FetchFields(taosRes); - int numOfFiled = TDengine.FieldCount(taosRes); - - - List dataRaw = new(); - - IntPtr colLengthPrt = TDengine.FetchLengths(taosRes); - int[] colLengthArr = new int[numOfFiled]; - Marshal.Copy(colLengthPrt, colLengthArr, 0, numOfFiled); - - for (int i = 0; i < numOfFiled; i++) - { - TDengineMeta meta = metaList[i]; - IntPtr data = Marshal.ReadIntPtr(taosRow, IntPtr.Size * i); - - if (data == IntPtr.Zero) - { - dataRaw.Add("NULL"); - continue; - } - switch ((TDengineDataType)meta.type) - { - case TDengineDataType.TSDB_DATA_TYPE_BOOL: - bool v1 = Marshal.ReadByte(data) != 0; - dataRaw.Add(v1); - break; - case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - sbyte v2 = (sbyte)Marshal.ReadByte(data); - dataRaw.Add(v2); - break; - case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - short v3 = Marshal.ReadInt16(data); - dataRaw.Add(v3); - break; - case TDengineDataType.TSDB_DATA_TYPE_INT: - int v4 = Marshal.ReadInt32(data); - dataRaw.Add(v4); - break; - case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - long v5 = Marshal.ReadInt64(data); - dataRaw.Add(v5); - break; - case TDengineDataType.TSDB_DATA_TYPE_FLOAT: - float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); - dataRaw.Add(v6); - break; - case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: - double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); - dataRaw.Add(v7); - break; - case TDengineDataType.TSDB_DATA_TYPE_BINARY: - string v8 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - dataRaw.Add(v8); - break; - case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: - long v9 = Marshal.ReadInt64(data); - dataRaw.Add(v9); - break; - case TDengineDataType.TSDB_DATA_TYPE_NCHAR: - string v10 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - dataRaw.Add(v10); - break; - case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: - byte v12 = Marshal.ReadByte(data); - dataRaw.Add(v12.ToString()); - break; - case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: - ushort v13 = (ushort)Marshal.ReadInt16(data); - dataRaw.Add(v13); - break; - case TDengineDataType.TSDB_DATA_TYPE_UINT: - uint v14 = (uint)Marshal.ReadInt32(data); - dataRaw.Add(v14); - break; - case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: - ulong v15 = (ulong)Marshal.ReadInt64(data); - dataRaw.Add(v15); - break; - case TDengineDataType.TSDB_DATA_TYPE_JSONTAG: - string v16 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - dataRaw.Add(v16); - break; - default: - dataRaw.Add("nonsupport data type"); - break; - } - - } - return dataRaw; - } - static IntPtr GetConnection() { string host = "localhost"; @@ -223,16 +91,16 @@ namespace TDengineExample } } -//output: -// Connect to TDengine success -// 8 rows async retrieved - -// 1538548685500 | 11.8 | 221 | 0.28 | california.losangeles | 2 | -// 1538548696600 | 13.4 | 223 | 0.29 | california.losangeles | 2 | -// 1538548685000 | 10.8 | 223 | 0.29 | california.losangeles | 3 | -// 1538548686500 | 11.5 | 221 | 0.35 | california.losangeles | 3 | -// 1538548685000 | 10.3 | 219 | 0.31 | california.sanfrancisco | 2 | -// 1538548695000 | 12.6 | 218 | 0.33 | california.sanfrancisco | 2 | -// 1538548696800 | 12.3 | 221 | 0.31 | california.sanfrancisco | 2 | -// 1538548696650 | 10.3 | 218 | 0.25 | california.sanfrancisco | 3 | -// async retrieve complete. \ No newline at end of file +// //output: +// // Connect to TDengine success +// // 8 rows async retrieved + +// // 1538548685500 | 11.8 | 221 | 0.28 | california.losangeles | 2 | +// // 1538548696600 | 13.4 | 223 | 0.29 | california.losangeles | 2 | +// // 1538548685000 | 10.8 | 223 | 0.29 | california.losangeles | 3 | +// // 1538548686500 | 11.5 | 221 | 0.35 | california.losangeles | 3 | +// // 1538548685000 | 10.3 | 219 | 0.31 | california.sanfrancisco | 2 | +// // 1538548695000 | 12.6 | 218 | 0.33 | california.sanfrancisco | 2 | +// // 1538548696800 | 12.3 | 221 | 0.31 | california.sanfrancisco | 2 | +// // 1538548696650 | 10.3 | 218 | 0.25 | california.sanfrancisco | 3 | +// // async retrieve complete. \ No newline at end of file diff --git a/docs/examples/csharp/QueryExample.cs b/docs/examples/csharp/QueryExample.cs index 97f0c456d412e2ed608c345ba87469d3f5ccfc15..c90a8cd0b7a3fd3e4a797188aec9fa50ba704717 100644 --- a/docs/examples/csharp/QueryExample.cs +++ b/docs/examples/csharp/QueryExample.cs @@ -1,4 +1,5 @@ using TDengineDriver; +using TDengineDriver.Impl; using System.Runtime.InteropServices; namespace TDengineExample @@ -23,7 +24,7 @@ namespace TDengineExample Console.WriteLine("fieldCount=" + fieldCount); // print column names - List metas = TDengine.FetchFields(res); + List metas = LibTaos.GetMeta(res); for (int i = 0; i < metas.Count; i++) { Console.Write(metas[i].name + "\t"); @@ -31,98 +32,17 @@ namespace TDengineExample Console.WriteLine(); // print values - IntPtr row; - while ((row = TDengine.FetchRows(res)) != IntPtr.Zero) + List resData = LibTaos.GetData(res); + for (int i = 0; i < resData.Count; i++) { - List metaList = TDengine.FetchFields(res); - int numOfFiled = TDengine.FieldCount(res); - - List dataRaw = new List(); - - IntPtr colLengthPrt = TDengine.FetchLengths(res); - int[] colLengthArr = new int[numOfFiled]; - Marshal.Copy(colLengthPrt, colLengthArr, 0, numOfFiled); - - for (int i = 0; i < numOfFiled; i++) + Console.Write($"|{resData[i].ToString()} \t"); + if (((i + 1) % metas.Count == 0)) { - TDengineMeta meta = metaList[i]; - IntPtr data = Marshal.ReadIntPtr(row, IntPtr.Size * i); - - if (data == IntPtr.Zero) - { - Console.Write("NULL\t"); - continue; - } - switch ((TDengineDataType)meta.type) - { - case TDengineDataType.TSDB_DATA_TYPE_BOOL: - bool v1 = Marshal.ReadByte(data) == 0 ? false : true; - Console.Write(v1.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_TINYINT: - sbyte v2 = (sbyte)Marshal.ReadByte(data); - Console.Write(v2.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: - short v3 = Marshal.ReadInt16(data); - Console.Write(v3.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_INT: - int v4 = Marshal.ReadInt32(data); - Console.Write(v4.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_BIGINT: - long v5 = Marshal.ReadInt64(data); - Console.Write(v5.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_FLOAT: - float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); - Console.Write(v6.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: - double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); - Console.Write(v7.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_BINARY: - string v8 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - Console.Write(v8 + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: - long v9 = Marshal.ReadInt64(data); - Console.Write(v9.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_NCHAR: - string v10 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - Console.Write(v10 + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_UTINYINT: - byte v12 = Marshal.ReadByte(data); - Console.Write(v12.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_USMALLINT: - ushort v13 = (ushort)Marshal.ReadInt16(data); - Console.Write(v13.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_UINT: - uint v14 = (uint)Marshal.ReadInt32(data); - Console.Write(v14.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_UBIGINT: - ulong v15 = (ulong)Marshal.ReadInt64(data); - Console.Write(v15.ToString() + "\t"); - break; - case TDengineDataType.TSDB_DATA_TYPE_JSONTAG: - string v16 = Marshal.PtrToStringUTF8(data, colLengthArr[i]); - Console.Write(v16 + "\t"); - break; - default: - Console.Write("nonsupport data type value"); - break; - } - + Console.WriteLine(""); } - Console.WriteLine(); } + Console.WriteLine(); + if (TDengine.ErrorNo(res) != 0) { Console.WriteLine($"Query is not complete, Error {TDengine.ErrorNo(res)} {TDengine.Error(res)}"); diff --git a/docs/examples/csharp/SQLInsertExample.cs b/docs/examples/csharp/SQLInsertExample.cs index d5462c1062e01fd5c93bac983696d0350117ad92..192ea96d5713bbf7f37f2208687c41e3e66d473b 100644 --- a/docs/examples/csharp/SQLInsertExample.cs +++ b/docs/examples/csharp/SQLInsertExample.cs @@ -15,10 +15,10 @@ namespace TDengineExample CheckRes(conn, res, "failed to change database"); res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)"); CheckRes(conn, res, "failed to create stable"); - var sql = "INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + - "d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + - "d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + - "d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; + var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " + + "d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " + + "d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " + + "d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)"; res = TDengine.Query(conn, sql); CheckRes(conn, res, "failed to insert data"); int affectedRows = TDengine.AffectRows(res); diff --git a/docs/examples/csharp/StmtInsertExample.cs b/docs/examples/csharp/StmtInsertExample.cs index 6ade424b95d64529b7a40a782de13e3106d0c78a..0a4098091f6371a674eee6f158e1c57bff2b6862 100644 --- a/docs/examples/csharp/StmtInsertExample.cs +++ b/docs/examples/csharp/StmtInsertExample.cs @@ -21,7 +21,7 @@ namespace TDengineExample CheckStmtRes(res, "failed to prepare stmt"); // 2. bind table name and tags - TAOS_BIND[] tags = new TAOS_BIND[2] { TaosBind.BindBinary("California.SanFrancisco"), TaosBind.BindInt(2) }; + TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[]{"California.SanFrancisco"}), TaosMultiBind.MultiBindInt(new int?[] {2}) }; res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags); CheckStmtRes(res, "failed to bind table name and tags"); @@ -44,7 +44,7 @@ namespace TDengineExample CheckStmtRes(res, "faild to execute"); // 6. free - TaosBind.FreeTaosBind(tags); + TaosMultiBind.FreeTaosBind(tags); TaosMultiBind.FreeTaosBind(values); TDengine.Close(conn); TDengine.Cleanup(); diff --git a/docs/examples/csharp/SubscribeDemo.cs b/docs/examples/csharp/SubscribeDemo.cs index 34509215da73ea6369eb95f458d622cd95a97932..b62ff12e5ea38eb27ae5de8e8027aa41b1873d23 100644 --- a/docs/examples/csharp/SubscribeDemo.cs +++ b/docs/examples/csharp/SubscribeDemo.cs @@ -1,12 +1,100 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using TDengineTMQ; +using TDengineDriver; +using System.Runtime.InteropServices; -namespace csharp +namespace TMQExample { internal class SubscribeDemo { + static void Main(string[] args) + { + IntPtr conn = GetConnection(); + string topic = "topic_example"; + Console.WriteLine($"create topic if not exist {topic} as select * from meters"); + //create topic + IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters"); + + if (res == IntPtr.Zero) + { + throw new Exception($"create topic failed, reason:{TDengine.Error(res)}"); + } + + var cfg = new ConsumerConfig + { + GourpId = "group_1", + TDConnectUser = "root", + TDConnectPasswd = "taosdata", + MsgWithTableName = "true", + TDConnectIp = "127.0.0.1", + }; + + // create consumer + var consumer = new ConsumerBuilder(cfg) + .Build(); + + // subscribe + consumer.Subscribe(topic); + + // consume + for (int i = 0; i < 5; i++) + { + var consumeRes = consumer.Consume(300); + // print consumeResult + foreach (KeyValuePair kv in consumeRes.Message) + { + Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString()); + + kv.Value.Metas.ForEach(meta => + { + Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size); + }); + Console.WriteLine(""); + kv.Value.Datas.ForEach(data => + { + Console.WriteLine(data.ToString()); + }); + } + + consumer.Commit(consumeRes); + Console.WriteLine("\n================ {0} done ", i); + + } + + // retrieve topic list + List topics = consumer.Subscription(); + topics.ForEach(t => Console.WriteLine("topic name:{0}", t)); + + + // unsubscribe + consumer.Unsubscribe(); + + // close consumer after use.Otherwise will lead memory leak. + consumer.Close(); + TDengine.Close(conn); + + + } + + static IntPtr GetConnection() + { + string host = "localhost"; + short port = 6030; + string username = "root"; + string password = "taosdata"; + string dbname = "power"; + var conn = TDengine.Connect(host, username, password, dbname, port); + if (conn == IntPtr.Zero) + { + Console.WriteLine("Connect to TDengine failed"); + System.Environment.Exit(0); + } + else + { + Console.WriteLine("Connect to TDengine success"); + } + return conn; + } } + } diff --git a/docs/examples/csharp/asyncquery.csproj b/docs/examples/csharp/asyncquery.csproj index 7a952fe7abd15798bc704de52aa72624e59061eb..045969edd7febbd11cc6577c8ba958669a5a7e3b 100644 --- a/docs/examples/csharp/asyncquery.csproj +++ b/docs/examples/csharp/asyncquery.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/connect.csproj b/docs/examples/csharp/connect.csproj index 27cffa30ae5a98277b74c63cf3d2c749ec67ce8d..3a912f8987ace6ae540726886d901c8d32a7b81b 100644 --- a/docs/examples/csharp/connect.csproj +++ b/docs/examples/csharp/connect.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/influxdbline.csproj b/docs/examples/csharp/influxdbline.csproj index a8b197dc7167f229b903bfd010a9ff7282c9173b..58bca485088e409fe1d387c6020418bbc2bf871b 100644 --- a/docs/examples/csharp/influxdbline.csproj +++ b/docs/examples/csharp/influxdbline.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/optsjson.csproj b/docs/examples/csharp/optsjson.csproj index b1bd83405efb1086b23134a9c875912a9c09e3f8..da16025dcd45f8e5c4ba6e242524c2e56191e93c 100644 --- a/docs/examples/csharp/optsjson.csproj +++ b/docs/examples/csharp/optsjson.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/optstelnet.csproj b/docs/examples/csharp/optstelnet.csproj index 1ab41067715f8cb070e78aec2be59d686bac0e86..194de21bcc74653a2267b29681ece6243fd401fc 100644 --- a/docs/examples/csharp/optstelnet.csproj +++ b/docs/examples/csharp/optstelnet.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/query.csproj b/docs/examples/csharp/query.csproj index 63f13c3ddbf13be9ffe5c39054cc6e4cae6001e7..39fc135d5ab9f5a8397b412e2307a2306abd4f2a 100644 --- a/docs/examples/csharp/query.csproj +++ b/docs/examples/csharp/query.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/sqlinsert.csproj b/docs/examples/csharp/sqlinsert.csproj index 0380395a5ad00d96418410b2f7974b294bf2bb07..ab0e5e717a78faad07c949b434b0d0b8a26c7211 100644 --- a/docs/examples/csharp/sqlinsert.csproj +++ b/docs/examples/csharp/sqlinsert.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/stmtinsert.csproj b/docs/examples/csharp/stmtinsert.csproj index 8defb895eb2894c680fe1b9dd7346223f0f92df1..3d459fbeda02ab03dc40dac2ecae290724cccbcc 100644 --- a/docs/examples/csharp/stmtinsert.csproj +++ b/docs/examples/csharp/stmtinsert.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/subscribe.csproj b/docs/examples/csharp/subscribe.csproj index 8286922c6f926b166f14983525e604ce9e13d74a..eff29b3bf42bde521aae70bfd1ed555ac72bfce9 100644 --- a/docs/examples/csharp/subscribe.csproj +++ b/docs/examples/csharp/subscribe.csproj @@ -5,11 +5,11 @@ net6.0 enable enable - TDengineExample.SubscribeDemo + TMQExample.SubscribeDemo - + diff --git a/docs/examples/go/connect/afconn/main.go b/docs/examples/go/connect/afconn/main.go index 3c16212a453eef9482159f5f32c406e07f657ef5..bb2574a01b0f16f75f6ae42ad7b1085a28cb525e 100644 --- a/docs/examples/go/connect/afconn/main.go +++ b/docs/examples/go/connect/afconn/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -10,7 +11,7 @@ func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) defer conn.Close() if err != nil { - fmt.Println("failed to connect, err:", err) + log.Fatalln("failed to connect, err:", err) } else { fmt.Println("connected") } diff --git a/docs/examples/go/connect/cgoexample/main.go b/docs/examples/go/connect/cgoexample/main.go index 57b8d5d91f0d8f5e5b3d5890906f388f96fd1126..881cf15ee3098d0582efffcb0cb9168224126c04 100644 --- a/docs/examples/go/connect/cgoexample/main.go +++ b/docs/examples/go/connect/cgoexample/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosSql" ) @@ -11,7 +12,7 @@ func main() { var taosDSN = "root:taosdata@tcp(localhost:6030)/" taos, err := sql.Open("taosSql", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) + log.Fatalln("failed to connect TDengine, err:", err) return } fmt.Println("Connected") diff --git a/docs/examples/go/connect/restexample/main.go b/docs/examples/go/connect/restexample/main.go index f10151f04fc0b4b213a235be1bef17a8fab49e4c..67a129bf9c6d934a5fee3666d3f180a5efe56fc9 100644 --- a/docs/examples/go/connect/restexample/main.go +++ b/docs/examples/go/connect/restexample/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosRestful" ) @@ -11,7 +12,7 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) + log.Fatalln("failed to connect TDengine, err:", err) return } fmt.Println("Connected") diff --git a/docs/examples/go/connect/wrapper/main.go b/docs/examples/go/connect/wrapper/main.go deleted file mode 100644 index a459539db34ae7e37f619712cd8e142580116d8f..0000000000000000000000000000000000000000 --- a/docs/examples/go/connect/wrapper/main.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/taosdata/driver-go/v3/wrapper" -) - -func main() { - conn, err := wrapper.TaosConnect("localhost", "root", "taosdata", "", 6030) - defer wrapper.TaosClose(conn) - if err != nil { - fmt.Println("fail to connect, err:", err) - } else { - fmt.Println("connected") - } -} diff --git a/docs/examples/go/insert/json/main.go b/docs/examples/go/insert/json/main.go index 805c123342ef6564239d5e7122bb9a7960f5c95e..e470cc6bd463327f9805caaa996e968c29af48a7 100644 --- a/docs/examples/go/insert/json/main.go +++ b/docs/examples/go/insert/json/main.go @@ -1,7 +1,7 @@ package main import ( - "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -20,7 +20,7 @@ func prepareDatabase(conn *af.Connector) { func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) if err != nil { - fmt.Println("fail to connect, err:", err) + log.Fatalln("fail to connect, err:", err) } defer conn.Close() prepareDatabase(conn) @@ -32,6 +32,6 @@ func main() { err = conn.OpenTSDBInsertJsonPayload(payload) if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/insert/line/main.go b/docs/examples/go/insert/line/main.go index 1a09edc40cc6e600190c781d7df2c1c6f4fad5cf..a270ab2633d1afc41c62d0f6adb74fd879fc9c7d 100644 --- a/docs/examples/go/insert/line/main.go +++ b/docs/examples/go/insert/line/main.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -33,6 +34,6 @@ func main() { err = conn.InfluxDBInsertLines(lines, "ms") if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/insert/sql/main.go b/docs/examples/go/insert/sql/main.go index a493ef5fb0a6418b30aa55911a713cf64ce75def..57f881314c87de1e16106ec707106b1159b2b9a0 100644 --- a/docs/examples/go/insert/sql/main.go +++ b/docs/examples/go/insert/sql/main.go @@ -3,6 +3,7 @@ package main import ( "database/sql" "fmt" + "log" _ "github.com/taosdata/driver-go/v3/taosRestful" ) @@ -10,28 +11,26 @@ import ( func createStable(taos *sql.DB) { _, err := taos.Exec("CREATE DATABASE power") if err != nil { - fmt.Println("failed to create database, err:", err) + log.Fatalln("failed to create database, err:", err) } _, err = taos.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)") if err != nil { - fmt.Println("failed to create stable, err:", err) + log.Fatalln("failed to create stable, err:", err) } } func insertData(taos *sql.DB) { - sql := `INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)` + sql := `INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)` result, err := taos.Exec(sql) if err != nil { - fmt.Println("failed to insert, err:", err) - return + log.Fatalln("failed to insert, err:", err) } rowsAffected, err := result.RowsAffected() if err != nil { - fmt.Println("failed to get affected rows, err:", err) - return + log.Fatalln("failed to get affected rows, err:", err) } fmt.Println("RowsAffected", rowsAffected) } @@ -40,8 +39,7 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + log.Fatalln("failed to connect TDengine, err:", err) } defer taos.Close() createStable(taos) diff --git a/docs/examples/go/insert/stmt/main.go b/docs/examples/go/insert/stmt/main.go index d23a6e345c6c14f6cfeb8a21a81be296fb19f70e..bcbb8e632bd476a45e869d22d8e818273b214139 100644 --- a/docs/examples/go/insert/stmt/main.go +++ b/docs/examples/go/insert/stmt/main.go @@ -5,8 +5,8 @@ import ( "time" "github.com/taosdata/driver-go/v3/af" - "github.com/taosdata/driver-go/v3/af/param" "github.com/taosdata/driver-go/v3/common" + "github.com/taosdata/driver-go/v3/common/param" ) func checkErr(err error, prompt string) { diff --git a/docs/examples/go/insert/telnet/main.go b/docs/examples/go/insert/telnet/main.go index 4cf4375db5be6c4cca53bbcce877bf598ac326e7..63e9a56dbf1c9546199fa91a477303b934c5ac19 100644 --- a/docs/examples/go/insert/telnet/main.go +++ b/docs/examples/go/insert/telnet/main.go @@ -1,7 +1,7 @@ package main import ( - "fmt" + "log" "github.com/taosdata/driver-go/v3/af" ) @@ -20,7 +20,7 @@ func prepareDatabase(conn *af.Connector) { func main() { conn, err := af.Open("localhost", "root", "taosdata", "", 6030) if err != nil { - fmt.Println("fail to connect, err:", err) + log.Fatalln("fail to connect, err:", err) } defer conn.Close() prepareDatabase(conn) @@ -37,6 +37,6 @@ func main() { err = conn.OpenTSDBInsertTelnetLines(lines) if err != nil { - fmt.Println("insert error:", err) + log.Fatalln("insert error:", err) } } diff --git a/docs/examples/go/query/sync/main.go b/docs/examples/go/query/sync/main.go index add422e3859bee88febef83ad5666550e67adf5b..e37164f47fc0d877c420cea812ec45c124222d93 100644 --- a/docs/examples/go/query/sync/main.go +++ b/docs/examples/go/query/sync/main.go @@ -2,7 +2,7 @@ package main import ( "database/sql" - "fmt" + "log" "time" _ "github.com/taosdata/driver-go/v3/taosRestful" @@ -12,14 +12,12 @@ func main() { var taosDSN = "root:taosdata@http(localhost:6041)/power" taos, err := sql.Open("taosRestful", taosDSN) if err != nil { - fmt.Println("failed to connect TDengine, err:", err) - return + log.Fatalln("failed to connect TDengine, err:", err) } defer taos.Close() rows, err := taos.Query("SELECT ts, current FROM meters LIMIT 2") if err != nil { - fmt.Println("failed to select from table, err:", err) - return + log.Fatalln("failed to select from table, err:", err) } defer rows.Close() @@ -30,9 +28,9 @@ func main() { } err := rows.Scan(&r.ts, &r.current) if err != nil { - fmt.Println("scan error:\n", err) + log.Fatalln("scan error:\n", err) return } - fmt.Println(r.ts, r.current) + log.Fatalln(r.ts, r.current) } } diff --git a/docs/examples/node/nativeexample/async_query_example.js b/docs/examples/node/nativeexample/async_query_example.js index 25b78bc48a82454cc0f9db69b77323a00f3852b4..432d8b8f6c1e89ebe35adac55d4f97376d205cb6 100644 --- a/docs/examples/node/nativeexample/async_query_example.js +++ b/docs/examples/node/nativeexample/async_query_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); const cursor = conn.cursor(); @@ -18,4 +18,3 @@ try { conn.close(); }, 2000); } -// bug here: jira 14506 diff --git a/docs/examples/node/nativeexample/connect.js b/docs/examples/node/nativeexample/connect.js index da791c566e3aa40315b6f35cfeca704040cd8107..bb027d4fe85f35e5a7047b1517caee00314e2153 100644 --- a/docs/examples/node/nativeexample/connect.js +++ b/docs/examples/node/nativeexample/connect.js @@ -1,13 +1,20 @@ -const taos = require("td2.0-connector"); +const { options, connect } = require("@tdengine/rest"); -var conn = taos.connect({ - host: "localhost", - port: 6030, - user: "root", - password: "taosdata", -}); -conn.close(); +async function test() { + options.path = "/rest/sql"; + options.host = "localhost"; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query("SELECT server_version()"); + res.toString(); + } catch (err) { + console.log(err); + } +} +test(); -// run with: node connect.js // output: -// Successfully connected to TDengine +// server_version() | +// =================== +// 3.0.0.0 | diff --git a/docs/examples/node/nativeexample/influxdb_line_example.js b/docs/examples/node/nativeexample/influxdb_line_example.js index 2050bee54506a3ee6fe7d89de97b3b41334dd4a6..57170770d8f128b034236b71d8ff7d46f31b3199 100644 --- a/docs/examples/node/nativeexample/influxdb_line_example.js +++ b/docs/examples/node/nativeexample/influxdb_line_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/insert_example.js b/docs/examples/node/nativeexample/insert_example.js index ade9d83158362cbf00a856b43a973de31def7601..42050fa251f3da5e5a4a28aed232aef418ff8f7e 100644 --- a/docs/examples/node/nativeexample/insert_example.js +++ b/docs/examples/node/nativeexample/insert_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -11,11 +11,11 @@ try { cursor.execute( "CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)" ); - var sql = `INSERT INTO power.d1001 USING power.meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -power.d1002 USING power.meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -power.d1003 USING power.meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -power.d1004 USING power.meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)`; - cursor.execute(sql); + var sql = `INSERT INTO power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)`; + cursor.execute(sql,{'quiet':false}); } finally { cursor.close(); conn.close(); diff --git a/docs/examples/node/nativeexample/multi_bind_example.js b/docs/examples/node/nativeexample/multi_bind_example.js index 6ef8b30c097393fef8c6a2837f8683c736b363f1..6cc3993397990af9bb9c01e1b5bf844f8a242201 100644 --- a/docs/examples/node/nativeexample/multi_bind_example.js +++ b/docs/examples/node/nativeexample/multi_bind_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -24,10 +24,10 @@ function insertData() { ); // bind table name and tags - let tagBind = new taos.TaosBind(2); - tagBind.bindBinary("California.SanFrancisco"); - tagBind.bindInt(2); - cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + let tagBind = new taos.TaosMultiBindArr(2); + tagBind.multiBindBinary(["California.SanFrancisco"]); + tagBind.multiBindInt([2]); + cursor.stmtSetTbnameTags("d1001", tagBind.getMultiBindArr()); // bind values let valueBind = new taos.TaosMultiBindArr(4); diff --git a/docs/examples/node/nativeexample/opentsdb_json_example.js b/docs/examples/node/nativeexample/opentsdb_json_example.js index 2d78444a3f805bc77ab5e11925a28dd18fe221fe..4ea36ca68be042bd6c114d44b70f390bf4e503e2 100644 --- a/docs/examples/node/nativeexample/opentsdb_json_example.js +++ b/docs/examples/node/nativeexample/opentsdb_json_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/opentsdb_telnet_example.js b/docs/examples/node/nativeexample/opentsdb_telnet_example.js index 7f80f558838e18f07ad79e580e7d08638b74e940..8f5d17822b0feb9da06fbd7f419dbf1d8369e6a0 100644 --- a/docs/examples/node/nativeexample/opentsdb_telnet_example.js +++ b/docs/examples/node/nativeexample/opentsdb_telnet_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", diff --git a/docs/examples/node/nativeexample/param_bind_example.js b/docs/examples/node/nativeexample/param_bind_example.js index c7e04c71a0d19ff8666f3d43fe09109009741266..efe422586fb75734073311d24f03e6c50d18ab28 100644 --- a/docs/examples/node/nativeexample/param_bind_example.js +++ b/docs/examples/node/nativeexample/param_bind_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", @@ -23,25 +23,22 @@ function insertData() { ); // bind table name and tags - let tagBind = new taos.TaosBind(2); - tagBind.bindBinary("California.SanFrancisco"); - tagBind.bindInt(2); - cursor.stmtSetTbnameTags("d1001", tagBind.getBind()); + let tagBind = new taos.TaosMultiBindArr(2); + tagBind.multiBindBinary(["California.SanFrancisco"]); + tagBind.multiBindInt([2]); + cursor.stmtSetTbnameTags("d1001", tagBind.getMultiBindArr()); // bind values - let rows = [ - [1648432611249, 10.3, 219, 0.31], - [1648432611749, 12.6, 218, 0.33], - ]; - for (let row of rows) { - let valueBind = new taos.TaosBind(4); - valueBind.bindTimestamp(row[0]); - valueBind.bindFloat(row[1]); - valueBind.bindInt(row[2]); - valueBind.bindFloat(row[3]); - cursor.stmtBindParam(valueBind.getBind()); - cursor.stmtAddBatch(); - } + let rows = [[1648432611249, 1648432611749], [10.3, 12.6], [219, 218], [0.31, 0.33]]; + + let valueBind = new taos.TaosMultiBindArr(4); + valueBind.multiBindTimestamp(rows[0]); + valueBind.multiBindFloat(rows[1]); + valueBind.multiBindInt(rows[2]); + valueBind.multiBindFloat(rows[3]); + cursor.stmtBindParamBatch(valueBind.getMultiBindArr()); + cursor.stmtAddBatch(); + // execute cursor.stmtExecute(); diff --git a/docs/examples/node/nativeexample/query_example.js b/docs/examples/node/nativeexample/query_example.js index a51bc939ae8dc458f019ac3bf006d9320dfdee81..11cfd9a6057eb8d645282a4f0cab945ba46f4a9b 100644 --- a/docs/examples/node/nativeexample/query_example.js +++ b/docs/examples/node/nativeexample/query_example.js @@ -1,4 +1,4 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); const cursor = conn.cursor(); @@ -9,8 +9,6 @@ query.execute().then(function (result) { // output: // Successfully connected to TDengine -// Query OK, 2 row(s) in set (0.00317767s) - // ts | current | // ======================================================= // 2018-10-03 14:38:05.000 | 10.3 | diff --git a/docs/examples/node/nativeexample/subscribe_demo.js b/docs/examples/node/nativeexample/subscribe_demo.js index bdf0c98687c359ba20664e869a25690dffbabd29..c4f7e6df84933f8f8541814cabd231fcf5c2db68 100644 --- a/docs/examples/node/nativeexample/subscribe_demo.js +++ b/docs/examples/node/nativeexample/subscribe_demo.js @@ -1,4 +1,51 @@ -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); const conn = taos.connect({ host: "localhost", database: "power" }); -// 未完成 \ No newline at end of file +var cursor = conn.cursor(); + +function runConsumer() { + + // create topic + cursor.execute("create topic topic_name_example as select * from meters"); + + let consumer = taos.consumer({ + 'group.id': 'tg2', + 'td.connect.user': 'root', + 'td.connect.pass': 'taosdata', + 'msg.with.table.name': 'true', + 'enable.auto.commit': 'true' + }); + + // subscribe the topic just created. + consumer.subscribe("topic_name_example"); + + // get subscribe topic list + let topicList = consumer.subscription(); + console.log(topicList); + + for (let i = 0; i < 5; i++) { + let msg = consumer.consume(100); + console.log(msg.topicPartition); + console.log(msg.block); + console.log(msg.fields) + consumer.commit(msg); + console.log(`=======consumer ${i} done`) + } + + consumer.unsubscribe(); + consumer.close(); + + // drop topic + cursor.execute("drop topic topic_name_example"); +} + + +try { + runConsumer(); +} finally { + + setTimeout(() => { + cursor.close(); + conn.close(); + }, 2000); +} \ No newline at end of file diff --git a/docs/examples/node/package.json b/docs/examples/node/package.json index f56196d2e5011b1c56e59d78d2ce5fdda98205e9..36d3f016b5262472d5c63a2c98cc9704e57a59fe 100644 --- a/docs/examples/node/package.json +++ b/docs/examples/node/package.json @@ -4,7 +4,7 @@ "main": "index.js", "license": "MIT", "dependencies": { - "td2.0-connector": "^2.0.12", - "td2.0-rest-connector": "^1.0.0" + "@tdengine/client": "^3.0.0", + "@tdengine/rest": "^3.0.0" } } diff --git a/docs/examples/node/restexample/connect.js b/docs/examples/node/restexample/connect.js index b84ce2fadf9177983701582b05200986ea9d5ba7..132e284ce9fe6e684a18f1c175256659ae1ebe77 100644 --- a/docs/examples/node/restexample/connect.js +++ b/docs/examples/node/restexample/connect.js @@ -1,4 +1,4 @@ -const { options, connect } = require("td2.0-rest-connector"); +const { options, connect } = require("@tdengine/rest"); async function test() { options.path = "/rest/sqlt"; @@ -17,4 +17,4 @@ test(); // output: // server_version() | // =================== -// 2.4.0.12 | +// 3.0.0.0 | diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 6dbf74f8bcc3442018d259c9781999182fdc579d..1cdfe20c8d8cd5b2d6c39fec68ebf7acabb06266 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -11,7 +11,7 @@ import TabItem from "@theme/TabItem"; ::: -TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。 +TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包。也支持通过 `apt-get` 工具从线上进行安装。 ## 安装 @@ -293,4 +293,4 @@ taos> select avg(current), max(voltage), min(phase) from test.meters where group ```sql taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); -``` \ No newline at end of file +``` diff --git a/docs/zh/07-develop/03-insert-data/_go_stmt.mdx b/docs/zh/07-develop/03-insert-data/_go_stmt.mdx index d65a96549fbef3cd14b47853f765d557447dde1d..db4e227c438180637c0b17850a9a80e96edf91ed 100644 --- a/docs/zh/07-develop/03-insert-data/_go_stmt.mdx +++ b/docs/zh/07-develop/03-insert-data/_go_stmt.mdx @@ -3,6 +3,6 @@ ``` :::tip -driver-go 的模块 `github.com/taosdata/driver-go/v2/wrapper` 是 C 接口的底层封装。使用这个模块也可以实现参数绑定写入。 +driver-go 的模块 `github.com/taosdata/driver-go/v3/wrapper` 是 C 接口的底层封装。使用这个模块也可以实现参数绑定写入。 ::: diff --git a/docs/zh/07-develop/04-query-data/index.mdx b/docs/zh/07-develop/04-query-data/index.mdx index 824f36ef2f98aac227bdcaf2016d7be0a2e59328..eecda92744e12934f09ea8c66218f78da9a66146 100644 --- a/docs/zh/07-develop/04-query-data/index.mdx +++ b/docs/zh/07-develop/04-query-data/index.mdx @@ -25,6 +25,7 @@ TDengine 采用 SQL 作为查询语言。应用程序可以通过 REST API 或 - 单列、多列数据查询 - 标签和数值的多种过滤条件：>, <, =, <\>, like 等 - 聚合结果的分组（Group by）、排序（Order by）、约束输出（Limit/Offset） +- 时间窗口（Interval）、会话窗口（Session）和状态窗口（State_window）等窗口切分聚合查询 - 数值列及聚合结果的四则运算 - 时间戳对齐的连接查询（Join Query: 隐式连接）操作 - 多种聚合/计算函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff 等 @@ -40,7 +41,7 @@ taos> select * from d1001 where voltage > 215 order by ts desc limit 2; Query OK, 2 row(s) in set (0.001100s) ``` -为满足物联网场景的需求，TDengine 支持几个特殊的函数，比如 twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。TDengine 还支持连续查询。 +为满足物联网场景的需求，TDengine 支持几个特殊的函数，比如 twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。 具体的查询语法请看 [TAOS SQL 的数据查询](/taos-sql/select) 章节。 @@ -73,7 +74,7 @@ taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - Query OK, 1 row(s) in set (0.002136s) ``` -TDengine 仅容许对属于同一个超级表的表之间进行聚合查询，不同超级表之间的聚合查询不支持。在 [TAOS SQL 的数据查询](/taos-sql/select) 一章，查询类操作都会注明是否支持超级表。 +在 [TAOS SQL 的数据查询](/taos-sql/select) 一章，查询类操作都会注明是否支持超级表。 ## 降采样查询、插值 diff --git a/docs/zh/07-develop/07-tmq.md b/docs/zh/07-develop/07-tmq.md index 0f531e07c9dce7dbb03bacebf8e5cbefae82671f..459b88085f9910bca4840192223595a47a0bf2b9 100644 --- a/docs/zh/07-develop/07-tmq.md +++ b/docs/zh/07-develop/07-tmq.md @@ -1,254 +1,246 @@ ---- -sidebar_label: 数据订阅 -description: "轻量级的数据订阅与推送服务。连续写入到 TDengine 中的时序数据能够被自动推送到订阅客户端。" -title: 数据订阅 ---- - -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; -import Java from "./_sub_java.mdx"; -import Python from "./_sub_python.mdx"; -import Go from "./_sub_go.mdx"; -import Rust from "./_sub_rust.mdx"; -import Node from "./_sub_node.mdx"; -import CSharp from "./_sub_cs.mdx"; -import CDemo from "./_sub_c.mdx"; - -基于数据天然的时间序列特性，TDengine 的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致，均可视为系统中插入一条带时间戳的新记录。同时，TDengine 在内部严格按照数据时间序列单调递增的方式保存数据。本质上来说，TDengine 中每一张表均可视为一个标准的消息队列。 - -TDengine 内嵌支持轻量级的消息订阅与推送服务。使用系统提供的 API，用户可使用普通查询语句订阅数据库中的一张或多张表。订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达，有新的记录到达就会将结果反馈到客户。 - -TDengine 的订阅与推送服务的状态是由客户端维持，TDengine 服务端并不维持。因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。 - -TDengine 的 API 中，与订阅相关的主要有以下三个： - -```c -taos_subscribe -taos_consume -taos_unsubscribe -``` - -这些 API 的文档请见 [C/C++ Connector](/reference/connector/cpp)，下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”），完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/examples/c/subscribe.c) 找到。 - -如果我们希望当某个电表的电流超过一定限制（比如 10A）后能得到通知并进行一些处理， 有两种方法：一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： - -```sql -select * from D1001 where ts > {last_timestamp1} and current > 10; -select * from D1002 where ts > {last_timestamp2} and current > 10; -... -``` - -这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响，当电表数增长到一定的程度，系统就无法承受了。 - -另一种方法是对超级表进行查询。这样，无论有多少电表，都只需一次查询： - -```sql -select * from meters where ts > {last_timestamp} and current > 10; -``` - -但是，如何选择 `last_timestamp` 就成了一个新的问题。因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大；另一方面，不同电表的数据到达 TDengine 的时间也会有差异。所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`，就可能重复读入其它电表的数据；如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。 - -TDengine 的订阅功能为上面这个问题提供了一个彻底的解决方案。 - -首先是使用 `taos_subscribe` 创建订阅： - -```c -TAOS_SUB* tsub = NULL; -if (async) { -　　// create an asynchronized subscription, the callback function will be called every 1s -　　tsub = taos_subscribe(taos, restart, topic, sql, subscribe_callback, &blockFetch, 1000); -} else { -　　// create an synchronized subscription, need to call 'taos_consume' manually -　　tsub = taos_subscribe(taos, restart, topic, sql, NULL, NULL, 0); -} -``` - -TDengine 中的订阅既可以是同步的，也可以是异步的，上面的代码会根据从命令行获取的参数 `async` 的值来决定使用哪种方式。这里，同步的意思是用户程序要直接调用 `taos_consume` 来拉取数据，而异步则由 API 在内部的另一个线程中调用 `taos_consume`，然后把拉取到的数据交给回调函数 `subscribe_callback`去处理。（注意，`subscribe_callback` 中不宜做较为耗时的操作，否则有可能导致客户端阻塞等不可控的问题。） - -参数 `taos` 是一个已经建立好的数据库连接，在同步模式下无特殊要求。但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误，因为回调函数在 API 的内部线程中被调用，而 TDengine 的部分 API 不是线程安全的。 - -参数 `sql` 是查询语句，可以在其中使用 where 子句指定过滤条件。在我们的例子中，如果只想订阅电流超过 10A 时的数据，可以这样写： - -```sql -select * from meters where current > 10; -``` - -注意，这里没有指定起始时间，所以会读到所有时间的数据。如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件： - -```sql -select * from meters where ts > now - 1d and current > 10; -``` - -订阅的 `topic` 实际上是它的名字，因为订阅功能是在客户端 API 中实现的，所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 - -如果名为 `topic` 的订阅不存在，参数 `restart` 没有意义；但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个 `topic` 时，`restart` 就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。本例中，如果 `restart` 是 **true**（非零值），用户程序肯定会读到所有数据。但如果这个订阅之前就存在了，并且已经读取了一部分数据，且 `restart` 是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 - -`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。在同步模式下，如果前后两次调用 `taos_consume` 的时间间隔小于此时间，`taos_consume` 会阻塞，直到间隔超过此时间。异步模式下，这个时间是两次调用回调函数的最小时间间隔。 - -`taos_subscribe` 的倒数第二个参数用于用户程序向回调函数传递附加参数，订阅 API 不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。 - -订阅创建以后，就可以消费其数据了，同步模式下，示例代码是下面的 else 部分： - -```c -if (async) { -　　getchar(); -} else while(1) { -　　TAOS_RES* res = taos_consume(tsub); -　　if (res == NULL) { -　　　　printf("failed to consume data."); -　　　　break; -　　} else { -　　　　print_result(res, blockFetch); -　　　　getchar(); -　　} -} -``` - -这里是一个 **while** 循环，用户每按一次回车键就调用一次 `taos_consume`，而 `taos_consume` 的返回值是查询到的结果集，与 `taos_use_result` 完全相同，例子中使用这个结果集的代码是函数 `print_result`： - -```c -void print_result(TAOS_RES* res, int blockFetch) { -　　TAOS_ROW row = NULL; -　　int num_fields = taos_num_fields(res); -　　TAOS_FIELD* fields = taos_fetch_fields(res); -　　int nRows = 0; -　　if (blockFetch) { -　　　　nRows = taos_fetch_block(res, &row); -　　　　for (int i = 0; i < nRows; i++) { -　　　　　　char temp[256]; -　　　　　　taos_print_row(temp, row + i, fields, num_fields); -　　　　　　puts(temp); -　　　　} -　　} else { -　　　　while ((row = taos_fetch_row(res))) { -　　　　　　char temp[256]; -　　　　　　taos_print_row(temp, row, fields, num_fields); -　　　　　　puts(temp); -　　　　　　nRows++; -　　　　} -　　} -　　printf("%d rows consumed.\n", nRows); -} -``` - -其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。而异步模式下，消费订阅到的数据则显得更为简单： - -```c -void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { -　　print_result(res, *(int*)param); -} -``` - -当要结束一次数据订阅时，需要调用 `taos_unsubscribe`： - -```c -taos_unsubscribe(tsub, keep); -``` - -其第二个参数，用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**（**0**），那无论下次调用 `taos_subscribe` 时的 `restart` 参数是什么，订阅都只能重新开始。另外，进度信息的保存位置是 _{DataDir}/subscribe/_ 这个目录下（注：`taos.cfg` 配置文件中 `DataDir` 参数值默认为 **/var/lib/taos/**,但是 Windows 服务器上本身不存在该目录，所以需要在 Windows 的配置文件中修改 `DataDir` 参数值为相应的已存在目录"），每个订阅有一个与其 `topic` 同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。 - -代码介绍完毕，我们来看一下实际的运行效果。假设： - -- 示例代码已经下载到本地 -- TDengine 也已经在同一台机器上安装好 -- 示例所需的数据库、超级表、子表已经全部创建好 - -则可以在示例代码所在目录执行以下命令来编译并启动示例程序： - -```bash -make -./subscribe -sql='select * from meters where current > 10;' -``` - -示例程序启动后，打开另一个终端窗口，启动 TDengine CLI 向 **D1001** 插入一条电流为 12A 的数据： - -```sql -$ taos -> use test; -> insert into D1001 values(now, 12, 220, 1); -``` - -这时，因为电流超过了 10A，您应该可以看到示例程序将它输出到了屏幕上。您可以继续插入一些数据观察示例程序的输出。 - -## 示例程序 - -下面的示例程序展示是如何使用连接器订阅所有电流超过 10A 的记录。 - -### 准备数据 - -``` -# create database "power" -taos> create database power; -# use "power" as the database in following operations -taos> use power; -# create super table "meters" -taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupId int); -# create tabes using the schema defined by super table "meters" -taos> create table d1001 using meters tags ("California.SanFrancisco", 2); -taos> create table d1002 using meters tags ("California.LosAngeles", 2); -# insert some rows -taos> insert into d1001 values("2020-08-15 12:00:00.000", 12, 220, 1),("2020-08-15 12:10:00.000", 12.3, 220, 2),("2020-08-15 12:20:00.000", 12.2, 220, 1); -taos> insert into d1002 values("2020-08-15 12:00:00.000", 9.9, 220, 1),("2020-08-15 12:10:00.000", 10.3, 220, 1),("2020-08-15 12:20:00.000", 11.2, 220, 1); -# filter out the rows in which current is bigger than 10A -taos> select * from meters where current > 10; - ts | current | voltage | phase | location | groupid | -=========================================================================================================== - 2020-08-15 12:10:00.000 | 10.30000 | 220 | 1 | California.LosAngeles | 2 | - 2020-08-15 12:20:00.000 | 11.20000 | 220 | 1 | California.LosAngeles | 2 | - 2020-08-15 12:00:00.000 | 12.00000 | 220 | 1 | California.SanFrancisco | 2 | - 2020-08-15 12:10:00.000 | 12.30000 | 220 | 2 | California.SanFrancisco | 2 | - 2020-08-15 12:20:00.000 | 12.20000 | 220 | 1 | California.SanFrancisco | 2 | -Query OK, 5 row(s) in set (0.004896s) -``` - -### 示例代码 - - - - - - - - - {/* - - */} - - - - {/* - - - - - */} - - - - - -### 运行示例程序 - -示例程序会先消费符合查询条件的所有历史数据： - -```bash -ts: 1597464000000 current: 12.0 voltage: 220 phase: 1 location: California.SanFrancisco groupid : 2 -ts: 1597464600000 current: 12.3 voltage: 220 phase: 2 location: California.SanFrancisco groupid : 2 -ts: 1597465200000 current: 12.2 voltage: 220 phase: 1 location: California.SanFrancisco groupid : 2 -ts: 1597464600000 current: 10.3 voltage: 220 phase: 1 location: California.LosAngeles groupid : 2 -ts: 1597465200000 current: 11.2 voltage: 220 phase: 1 location: California.LosAngeles groupid : 2 -``` - -接着，使用 TDengine CLI 向表中新增一条数据： - -``` -# taos -taos> use power; -taos> insert into d1001 values(now, 12.4, 220, 1); -``` - -因为这条数据的电流大于 10A，示例程序会将其消费： - -``` -ts: 1651146662805 current: 12.4 voltage: 220 phase: 1 location: California.SanFrancisco groupid: 2 -``` +--- +sidebar_label: 数据订阅 +description: "数据订阅与推送服务。写入到 TDengine 中的时序数据能够被自动推送到订阅客户端。" +title: 数据订阅 +--- + +为了帮助应用实时获取写入 TDengine 的数据，或者以事件到达顺序处理数据，TDengine提供了类似消息队列产品的数据订阅、消费接口。这样在很多场景下，采用 TDengine 的时序数据处理系统不再需要集成消息队列产品，比如 kafka, 从而简化系统设计的复杂度，降低运营维护成本。 + +与 kafka 一样，你需要定义 topic, 但 TDengine 的 topic 是基于一个已经存在的超级表、子表或普通表的查询条件，即一个 SELECT 语句。你可以使用 SQL 对标签、表名、列、表达式等条件进行过滤，以及对数据进行标量函数与 UDF 计算（不包括数据聚合）。与其他消息队列软件相比，这是 TDengine 数据订阅功能的最大的优势，它提供了更大的灵活性，数据的颗粒度可以由应用随时调整，而且数据的过滤与预处理交给 TDengine，而不是应用完成，有效的减少传输的数据量与应用的复杂度。 + +消费者订阅 topic 后，可以实时获得最新的数据。多个消费者可以组成一个消费者组 (consumer group), 一个消费者组里的多个消费者共享消费进度，便于多线程、分布式地消费数据，提高消费速度。但不同消费者组中的消费者即使消费同一个topic, 并不共享消费进度。一个消费者可以订阅多个 topic。如果订阅的是超级表，数据可能会分布在多个不同的 vnode 上，也就是多个 shard 上，这样一个消费组里有多个消费者可以提高消费效率。TDengine 的消息队列提供了消息的ACK机制，在宕机、重启等复杂环境下确保 at least once 消费。 + +为了实现上述功能，TDengine 会为 WAL (Write-Ahead-Log) 文件自动创建索引以支持快速随机访问，并提供了灵活可配置的文件切换与保留机制：用户可以按需指定 WAL 文件保留的时间以及大小（详见 create database 语句）。通过以上方式将 WAL 改造成了一个保留事件到达顺序的、可持久化的存储引擎（但由于 TSDB 具有远比 WAL 更高的压缩率，我们不推荐保留太长时间，一般来说，不超过几天）。 对于以 topic 形式创建的查询，TDengine 将对接 WAL 而不是 TSDB 作为其存储引擎。在消费时，TDengine 根据当前消费进度从 WAL 直接读取数据，并使用统一的查询引擎实现过滤、变换等操作，将数据推送给消费者。 + +本文档不对消息队列本身的基础知识做介绍，如果需要了解，请自行搜索。 + +## 主要数据结构和API + +TMQ 的 API 中，与订阅相关的主要数据结构和API如下： + +```c +typedef struct tmq_t tmq_t; +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; + +typedef void(tmq_commit_cb(tmq_t *, int32_t code, void *param)); + +DLL_EXPORT tmq_list_t *tmq_list_new(); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); +DLL_EXPORT void tmq_list_destroy(tmq_list_t *); +DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT const char *tmq_err2str(int32_t code); + +DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); +DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); +DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); +DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); +DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); +DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); + +enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, +}; +typedef enum tmq_conf_res_t tmq_conf_res_t; + +DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); +DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); +``` + +这些 API 的文档请见 [C/C++ Connector](/reference/connector/cpp)，下面介绍一下它们的具体用法（超级表和子表结构请参考“数据建模”一节），完整的示例代码可以在 [tmq.c](https://github.com/taosdata/TDengine/blob/3.0/examples/c/tmq.c) 看到。 + +## 写入数据 + +首先完成建库、建一张超级表和多张子表操作，然后就可以写入数据了，比如： + +```sql +drop database if exists tmqdb; +create database tmqdb; +create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16) tags(t1 int, t3 varchar(16)); +create table tmqdb.ctb0 using tmqdb.stb tags(0, "subtable0"); +create table tmqdb.ctb1 using tmqdb.stb tags(1, "subtable1"); +create table tmqdb.ctb2 using tmqdb.stb tags(2, "subtable2"); +create table tmqdb.ctb3 using tmqdb.stb tags(3, "subtable3"); +insert into tmqdb.ctb0 values(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00'); +insert into tmqdb.ctb1 values(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11'); +insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22'); +insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33'); +``` + +## 创建topic： + +```sql +create topic topicName as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1; +``` + +TMQ支持多种订阅类型： + +### 列订阅 + +语法：CREATE TOPIC topic_name as subquery +通过select语句订阅（包括select *，或select ts, c1等指定列描述订阅，可以带条件过滤、标量函数计算，但不支持聚合函数、不支持时间窗口聚合） + +- TOPIC一旦创建则schema确定 +- 被订阅或用于计算的column和tag不可被删除、修改 +- 若发生schema变更，新增的column不出现在结果中 + +### 超级表订阅 +语法：CREATE TOPIC topic_name AS STABLE stbName + +与select * from stbName订阅的区别是： +- 不会限制用户的schema变更 +- 返回的是非结构化的数据：返回数据的schema会随之超级表的schema变化而变化 +- 用户对于要处理的每一个数据块都可能有不同的schema，因此，必须重新获取schema +- 返回数据不带有tag + +## 创建 consumer 以及consumer group + +对于consumer, 目前支持的config包括： + +| 参数名称 | 参数值 | 备注 | +| ---------------------------- | ------------------------------ | ------------------------------------------------------ | +| group.id | 最大长度：192 | | +| enable.auto.commit | 合法值：true, false | | +| auto.commit.interval.ms | | | +| auto.offset.reset | 合法值：earliest, latest, none | | +| td.connect.ip | 用于连接，同taos_connect的参数 | | +| td.connect.user | 用于连接，同taos_connect的参数 | | +| td.connect.pass | 用于连接，同taos_connect的参数 | | +| td.connect.port | 用于连接，同taos_connect的参数 | | +| enable.heartbeat.background | 合法值：true, false | 开启后台心跳，即consumer不会因为长时间不poll而认为离线 | +| experimental.snapshot.enable | 合法值：true, false | 从wal开始消费，还是从tsbs开始消费 | +| msg.with.table.name | 合法值：true, false | 从消息中能否解析表名 | + +```sql +/* 根据需要，设置消费组(group.id)、自动提交(enable.auto.commit)、自动提交时间间隔(auto.commit.interval.ms)、用户名(td.connect.user)、密码(td.connect.pass)等参数 */ + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + tmq_conf_set(conf, "group.id", "cgrpName"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + return tmq; +``` + +上述配置中包括consumer group ID，如果多个 consumer 指定的 consumer group ID一样，则自动形成一个consumer group，共享消费进度。 + + +## 创建 topic 列表 + +单个consumer支持同时订阅多个topic。 + +```sql + tmq_list_t* topicList = tmq_list_new(); + tmq_list_append(topicList, "topicName"); + return topicList; +``` + +## 启动订阅并开始消费 + +```sql + /* 启动订阅 */ + tmq_subscribe(tmq, topicList); + tmq_list_destroy(topicList); + + /* 循环poll消息 */ + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeOut = 5000; + while (running) { + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeOut); + if (tmqmsg) { + msgCnt++; + totalRows += msg_process(tmqmsg); + taos_free_result(tmqmsg); + } else { + break; + } + } + + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +``` + +这里是一个 **while** 循环，每调用一次tmq_consumer_poll()，获取一个消息，该消息与普通查询返回的结果集完全相同，可以使用相同的解析API完成消息内容的解析： + +```sql + static int32_t msg_process(TAOS_RES* msg) { + char buf[1024]; + int32_t rows = 0; + + const char* topicName = tmq_get_topic_name(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); + + printf("topic: %s\n", topicName); + printf("db: %s\n", dbName); + printf("vgroup id: %d\n", vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; + taos_print_row(buf, row, fields, numOfFields); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + } + + return rows; +} +``` + +## 结束消费 + +```sql + /* 取消订阅 */ + tmq_unsubscribe(tmq); + + /* 关闭消费 */ + tmq_consumer_close(tmq); +``` + +## 删除topic + +如果不再需要，可以删除创建topic，但注意：只有没有被订阅的topic才能别删除。 + +```sql + /* 删除topic */ + drop topic topicName; +``` + +## 状态查看 + +1、topics：查询已经创建的topic + +```sql + show topics; +``` + +2、consumers：查询consumer的状态及其订阅的topic + +```sql + show consumers; +``` + +3、subscriptions：查询consumer与vgroup之间的分配关系 + +```sql + show subscriptions; +``` + + diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 6071275b551d68aab51b5434a7ac07498bd27c62..b8ae61810584dd8ffc3016c0ce026ddb5b1a5ccf 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -124,52 +124,49 @@ gcc -g -O0 -fPIC -shared add_one.c -o add_one.so 用户可以通过 SQL 指令在系统中加载客户端所在主机上的 UDF 函数库（不能通过 RESTful 接口或 HTTP 管理界面来进行这一过程）。一旦创建成功，则当前 TDengine 集群的所有用户都可以在 SQL 指令中使用这些函数。UDF 存储在系统的 MNode 节点上，因此即使重启 TDengine 系统，已经创建的 UDF 也仍然可用。 -在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外， UDF 支持输入与输出类型不一致，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 +在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 - 创建标量函数 ```sql -CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ]; +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type; ``` - - ids(X)：标量函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - - B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 + - function_name：标量函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udf 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型名称； - 例如，如下语句可以把 add_one.so 创建为系统中可用的 UDF： + 例如，如下语句可以把 libbitand.so 创建为系统中可用的 UDF： ```sql - CREATE FUNCTION add_one AS "/home/taos/udf_example/add_one.so" OUTPUTTYPE INT; + CREATE FUNCTION bit_and AS "/home/taos/udf_example/libbitand.so" OUTPUTTYPE INT; ``` - 创建聚合函数： ```sql -CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ]; +CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ BUFSIZE buffer_size ]; ``` - - ids(X)：聚合函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - - B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 + - function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； + - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 - 关于中间计算结果的使用，可以参考示例程序[demo.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) - - 例如，如下语句可以把 demo.so 创建为系统中可用的 UDF： + 例如，如下语句可以把 libsqrsum.so 创建为系统中可用的 UDF： ```sql - CREATE AGGREGATE FUNCTION demo AS "/home/taos/udf_example/demo.so" OUTPUTTYPE DOUBLE bufsize 14; + CREATE AGGREGATE FUNCTION sqr_sum AS "/home/taos/udf_example/libsqrsum.so" OUTPUTTYPE DOUBLE bufsize 8; ``` ### 管理 UDF - 删除指定名称的用户定义函数： ``` -DROP FUNCTION ids(X); +DROP FUNCTION function_name; ``` -- ids(X)：此参数的含义与 CREATE 指令中的 ids(X) 参数一致，也即要删除的函数的名字，例如 +- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 ```sql -DROP FUNCTION add_one; +DROP FUNCTION bit_and; ``` - 显示系统中当前可用的所有 UDF： ```sql @@ -180,53 +177,32 @@ SHOW FUNCTIONS; 在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： ```sql -SELECT X(c) FROM table/stable; +SELECT X(c1,c2) FROM table/stable; ``` -表示对名为 c 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 - -## UDF 的一些使用限制 - -在当前版本下，使用 UDF 存在如下这些限制： +表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 -1. 在创建和调用 UDF 时，服务端和客户端都只支持 Linux 操作系统； -2. UDF 不能与系统内建的 SQL 函数混合使用，暂不支持在一条 SQL 语句中使用多个不同名的 UDF ； -3. UDF 只支持以单个数据列作为输入； -4. UDF 只要创建成功，就会被持久化存储到 MNode 节点中； -5. 无法通过 RESTful 接口来创建 UDF； -6. UDF 在 SQL 中定义的函数名，必须与 .so 库文件实现中的接口函数名前缀保持一致，也即必须是 udfNormalFunc 的名称，而且不可与 TDengine 中已有的内建 SQL 函数重名。 ## 示例代码 -### 标量函数示例 [add_one](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/add_one.c) - -
-add_one.c - -```c -{{#include tests/script/sh/add_one.c}} -``` - -
- -### 向量函数示例 [abs_max](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/abs_max.c) +### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c)
-abs_max.c +bit_and.c ```c -{{#include tests/script/sh/abs_max.c}} +{{#include tests/script/sh/bit_and.c}} ```
-### 使用中间计算结果示例 [demo](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) +### 聚合函数示例 [sqr_sum](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sqr_sum.c)
-demo.c +sqr_sum.c ```c -{{#include tests/script/sh/demo.c}} +{{#include tests/script/sh/sqr_sum.c}} ```
diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md new file mode 100644 index 0000000000000000000000000000000000000000..781f94324c78e7975abde33803cffdb914da020c --- /dev/null +++ b/docs/zh/12-taos-sql/24-show.md @@ -0,0 +1,270 @@ +--- +sidebar_label: SHOW 命令 +title: 使用 SHOW 命令查看系统元数据 +--- + +除了使用 `select` 语句查询 `INFORMATION_SCHEMA` 数据库中的表获得系统中的各种元数据、系统信息和状态之外，也可以用 `SHOW` 命令来实现同样的目的。 + +## SHOW ACCOUNTS + +```sql +SHOW ACCOUNTS; +``` + +显示当前系统中所有租户的信息。 + +注：企业版独有 + +## SHOW APPS + +```sql +SHOW APPS; +``` + +显示接入集群的应用（客户端）信息。 + +## SHOW BNODES + +```sql +SHOW BNODES; +``` + +显示当前系统中存在的 BNODE (backup node, 即备份节点）的信息。 + +## SHOW CLUSTER + +```sql +SHOW CLUSTER; +``` + +显示当前集群的信息 + +## SHOW CONNECTIONS + +```sql +SHOW CONNECTIONS; +``` + +显示当前系统中存在的连接的信息。 + +## SHOW CONSUMERS + +```sql +SHOW CONSUMERS; +``` + +显示当前数据库下所有活跃的消费者的信息。 + +## SHOW CREATE DATABASE + +```sql +SHOW CREATE DATABASE db_name; +``` + +显示 db_name 指定的数据库的创建语句。 + +## SHOW CREATE STABLE + +```sql +SHOW CREATE STABLE [db_name.]stb_name; +``` + +显示 tb_name 指定的超级表的创建语句 + +## SHOW CREATE TABLE + +```sql +SHOW CREATE TABLE [db_name.]tb_name +``` + +显示 tb_name 指定的表的创建语句。支持普通表、超级表和子表。 + +## SHOW DATABASES + +```sql +SHOW DATABASES; +``` + +显示用户定义的所有数据库。 + +## SHOW DNODES + +```sql +SHOW DNODES; +``` + +显示当前系统中 DNODE 的信息。 + +## SHOW FUNCTIONS + +```sql +SHOW FUNCTIONS; +``` + +显示用户定义的自定义函数。 + +## SHOW LICENSE + +```sql +SHOW LICENSE; +SHOW GRANTS; +``` + +显示企业版许可授权的信息。 + +注：企业版独有 + +## SHOW INDEXES + +```sql +SHOW INDEXES FROM tbl_name [FROM db_name]; +``` + +显示已创建的索引。 + +## SHOW LOCAL VARIABLES + +```sql +SHOW LOCAL VARIABLES; +``` + +显示当前客户端配置参数的运行值。 + +## SHOW MNODES + +```sql +SHOW MNODES; +``` + +显示当前系统中 MNODE 的信息。 + +## SHOW MODULES + +```sql +SHOW MODULES; +``` + +显示当前系统中所安装的组件的信息。 + +## SHOW QNODES + +```sql +SHOW QNODES; +``` + +显示当前系统中 QNODE （查询节点）的信息。 + +## SHOW SCORES + +```sql +SHOW SCORES; +``` + +显示系统被许可授权的容量的信息。 + +注：企业版独有 + +## SHOW SNODES + +```sql +SHOW SNODES; +``` + +显示当前系统中 SNODE （流计算节点）的信息。 + +## SHOW STABLES + +```sql +SHOW [db_name.]STABLES [LIKE 'pattern']; +``` + +显示当前数据库下的所有超级表的信息。可以使用 LIKE 对表名进行模糊匹配。 + +## SHOW STREAMS + +```sql +SHOW STREAMS; +``` + +显示当前系统内所有流计算的信息。 + +## SHOW SUBSCRIPTIONS + +```sql +SHOW SUBSCRIPTIONS; +``` + +显示当前数据库下的所有的订阅关系 + +## SHOW TABLES + +```sql +SHOW [db_name.]TABLES [LIKE 'pattern']; +``` + +显示当前数据库下的所有普通表和子表的信息。可以使用 LIKE 对表名进行模糊匹配。 + +## SHOW TABLE DISTRIBUTED + +```sql +SHOW TABLE DISTRIBUTED table_name; +``` + +显示表的数据分布信息。 + +## SHOW TAGS + +```sql +SHOW TAGS FROM child_table_name [FROM db_name]; +``` + +显示子表的标签信息。 + +## SHOW TOPICS + +```sql +SHOW TOPICS; +``` + +显示当前数据库下的所有主题的信息。 + +## SHOW TRANSACTIONS + +```sql +SHOW TRANSACTIONS; +``` + +显示当前系统中正在执行的事务的信息 + +## SHOW USERS + +```sql +SHOW USERS; +``` + +显示当前系统中所有用户的信息。包括用户自定义的用户和系统默认用户。 + +## SHOW VARIABLES + +```sql +SHOW VARIABLES; +SHOW DNODE dnode_id VARIABLES; +``` + +显示当前系统中各节点需要相同的配置参数的运行值，也可以指定 DNODE 来查看其的配置参数。 + +## SHOW VGROUPS + +```sql +SHOW [db_name.]VGROUPS; +``` + +显示当前系统中所有 VGROUP 或某个 db 的 VGROUPS 的信息。 + +## SHOW VNODES + +```sql +SHOW VNODES [dnode_name]; +``` + +显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。 diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index bd8d61a5844241efae9eee99a73c65afd3d0926f..12922063113f990f171347fcdb03633abef21b8e 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -8,21 +8,30 @@ title: 用户自定义函数 ## 创建函数 ```sql -CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE buffer_size] ``` 语法说明： AGGREGATE：标识此函数是标量函数还是聚集函数。 -func_name：函数名，必须与函数实现中udfNormalFunc的实际名称一致。 +func_name：函数名，必须与函数实现中 udf 的实际名称一致。 library_path：包含UDF函数实现的动态链接库的绝对路径，是在客户端侧主机上的绝对路径。 -OUTPUTTYPE：标识此函数的返回类型。 -BUFSIZE：中间结果的缓冲区大小，单位是字节。不设置则默认为0。最大不可超过512字节。 +type_name：标识此函数的返回类型。 +buffer_size：中间结果的缓冲区大小，单位是字节。不设置则默认为0。 关于如何开发自定义函数，请参考 [UDF使用说明](../../develop/udf)。 ## 删除自定义函数 +``` +DROP FUNCTION function_name; +``` + +- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 + + +## 显示 UDF + ```sql -DROP FUNCTION func_name -``` \ No newline at end of file +SHOW FUNCTION; +``` diff --git a/docs/zh/14-reference/03-connector/01-error-code.md b/docs/zh/14-reference/03-connector/01-error-code.md new file mode 100644 index 0000000000000000000000000000000000000000..53e006e108543805232c8195474f2afd793e7332 --- /dev/null +++ b/docs/zh/14-reference/03-connector/01-error-code.md @@ -0,0 +1,481 @@ +--- +sidebar_label: 错误码 +title: TDengine C/C++ 连接器错误码 +--- + +本文中详细列举了在使用 TDengine C/C++ 连接器时客户端可能得到的错误码以及所要采取的相应动作。其它语言的连接器在使用原生连接方式时也会所得到的返回码返回给连接器的调用者。 + + | **Error Code** | **说明** | 如何处理错误 | + | -------------- | ------------------ | ------------ | + | 0 | 请求处理成功 | None | + | -1 | 请求失败，未知原因 | TODO | + | 0x0003 | RPC 认证失败 | TODO | + | 0x0004 | RPC 重定向 |TODO| + | 0x000B | 无法建立连接 |TODO| + | 0x0015 | FQDN 解析失败 |检查各个dnode配置的FQDN是否正确，并且能够从其它节点 ping 到 | + | 0x0017 | 端口已经被占用 | 检查所配置的端口被哪个进程占用，关闭该进程以释放端口；或者更改配置使用另一端口号 | + | 0x0018 | 连接被断开 | 检查 dnode 进程是否还在，如果无异常检查网络情况 | + | 0x0013 | 客户端和服务端的时间未同步 | 检查客户端和服务端的时间设置是否同步| + | 0x0014 | 数据库不可用 |检查数据库是否存在，检查数据库的vgroups的状态 | + | 0x0100 | 该操作不支持 | 参考SQL手册，使用正确的操作 | + | 0x0102 | 无法分配内存，系统内存耗尽 | 检查系统中内存被耗尽的原因，采取措施释放内存。如果内存是被 dnode 耗尽的话重启该进程 | + | 0x0104 | 文件被破坏 | TODO | + | 0x0111 | 操作在进行中 | 等待该操作完成 | + | 0x0115 | 无效的消息 | TODO | + | 0x0116 | 无效的消息长度 | TODO | + | 0x0117 | 无效指针 | TODO | + | 0x0118 | 无效参数 | | + | 0x0119 | 无效配置 | 检查配置参数的值是否在合法值域范围内 | + | 0x011A | 无效选项 | 检查配置中是否有不支持的无效配置项 | + | 0x011B | 无效的JSON 格式 | 修正插入数据中的JSON值| + | 0x011C | 无效版本号 | 检查客户端版本是否匹配，更换正确的客户端 | + | 0x011D | 无效版本信息 | 检查客户端版本是否匹配，更换正确的客户端 | + | 0x011E | 版本不匹配 | TODO | + | 0x011F | 校验和错误 | TODO | + | 0x0120 | 数据压缩失败 | TODO | + | 0x0121 | 消息未处理 | TODO | + | 0x0122 | 配置未找到 | TODO| + | 0x0123 | 重复初始化 | TODO | + | 0x0124 | 无法将重复的key加入hash| TODO | + | 0x0125 | 需要重试 | 重试 | + | 0x0126 | RPC 队列内存耗尽 | TODO | + | 0x0127 | 无效的时间戳 | 修正插入数据 | + | 0x0128 | 消息解码失败 | 检查客户端版本是否匹配和兼容 | + | 0x0129 | 磁盘耗尽 | TODO | + | 0x012A | TODO | TODO | + | 0x0200 | 无效操作 | 检查SQL命令，根据SQL手册使用正确的操作| + | 0x0201 | 无效的查询句柄 | TODO | + | 0x0202 | 无效的客户端/服务端时间组合 | TODO | + | 0x0203 | 无效值 | TODO | + | 0x0204 | 无效的客户端版本 | 检查客户端版本是否匹配，更换为正确的客户端版本 | + | 0x0205 | TODO | TODO | + | 0x0206 | 无效的 FQDN | 检查服务器 fqdn 配置是否正确，是否能够从其它服务器访问到该 fqdn | + | 0x0207 | 用户名长度过长 | 检查用户名长度是否过长 | + | 0x0208 | 密码长度过长 | 检查密码长度是否过长 | + | 0x0209 | 数据库名字长度过长 | 检查数据库名是否过长 | + | 0x020A | 表名长度过长 | 检查表名是否过长 | + | 0x020B | 无效连接 | 检查该连接是否已经断开 | + | 0x020C | 系统内存耗尽 | 检查系统中内存被耗尽的原因，尝试释放出内存 | + | 0x020D | 磁盘空间耗尽 | 检查系统中磁盘耗尽的原因，尝试释放或增加磁盘空间 | + | 0x020E | 查询缓存被清除 |TODO | + | 0x020F | 查询已经被结束 | 尝试优化查询条件再重启查询 | + | 0x0210 | 结果集过大无法存储 | 尝试优化查询条件，缩小结果集，再重启查询 | + | 0x0211 | 数据库不可用 | 查看数据库是否存在，查看数据库的 vgroups 状态 | + | 0x0212 | 操作正在进行中 | 等待操作完成 | + | 0x0213 | 连接被服务端断开 | 查看服务端进程是否发生 crash 等异常终止情况 | + | 0x0214 | 没有写权限 | 只进行读操作或者尝试获取写权限 | + | 0x0215 | 连接被 kill | 重新建立连接 | + | 0x0216 | SQL 语法错误 | 参考 SQL手册，纠正语法错误再重试 | + | 0x0217 | 未指定数据库或者指定的数据库不可用 | 指定数据库，或者检查指定数据库的状态 | + | 0x0218 | 所查询的表不存在 | 确认表名后纠正查询语句 | + | 0x0219 | SQL 语句超长 | 根据 maxSQLLength 缩短SQL语句或者加大maxSQLLength （如果还未配置到上限）| + | 0x021A | 空文件 | TODO | + | 0x021B | Line 协议语法错误 | 纠正插入语句 | + | 0x021C | 元数据未被缓存 | TODO | + | 0x021D | 重复的列名 | 纠正SQL语句中的相应错误 | + | 0x021E | tag 过长 | 纠正过长的 tag | + | 0x021F | 列名过长 | 纠正过长的列名 | + | 0x0220 | 重复的表名**TODO** | TODO| + | 0x0221 | JSON 格式错误 | 纠正错误的 JSON 结构 | + | 0x0222 | JSON 中使用了无效的数据类型 | 纠正 JSON 结构中的错误 | + | 0x0224 | 超出了所支持的值域 | 纠正到值域范围内 | + | 0x0229 | 无效的 tsc 输入 **TODO** | TODO | + | 0x022A | stmt API 使用错误 | 根据参考手册正确使用 | + | 0x022B | stmt 使用时未指定表名 | 指定表名 | + | 0x022C | 不支持 stmt 子名 | 根据参考手册纠正错误用法 | + | 0x022D | 查询被 kill | 优化查询语句，尽量减小计算量和结果集，然后重新启动查询 | + | 0x022E | 在当前配置的查询策略下没有可用的计算节点 | 创建新的 qnode | + | 0x022F | 所指定的表不是超级表 | 确认下该查询场景适用于超级表还是子表/普通表，如果是前者则纠正为超级表名 | + | 0x0303 | 没有权限进行所发起的操作 | 申请权限或调整操作 | + | 0x0304 | 管理节点内部错误 | TODO | + | 0x0305 | 无效连接 | TODO | + | 0x030B | 所要展示的操作其数据已经因为超时而被删除 | 更换想要展示操作或者放弃此次操作 | + | 0x030C | 无效的查询ID| 确认正确的查询ID再重新发起 | + | 0x030D | 无效的流ID| 确认正确的流ID再重新发起| + | 0x030E | 无效的连接ID| 确认正确的连接ID再重新发起| + | 0x0310 | mnode已经在运行 | 无须采取任何动作 | + | 0x0311 | 配置同步失败 | TODO | + | 0x0312 | 无法启动同步 | TODO | + | 0x0313 | 无法创建 mnode 对应的目录 | 确认磁盘上是否有可用空间以及是否有相应的写权限 | + | 0x0314 | 启动组件失败 | TODO | + | 0x0315 | 用户帐号被禁用 | 联系管理员激活该帐号 | + | 0x0320 | 元数据中已经存在所要创建的对象 | 检查所要创建的对象，比如超级表或表，是否已经存在 | + | 0x0321 | 元数据库中非预期的一般性错误 | TODO | + | 0x0322 | 无效的表类型 | TODO | + | 0x0323 | 所要查找的对象不存在 | TODO | + | 0x0325 | 无效的 key 类型 | TODO | + | 0x0326 | 无效的动作类型 | TODO | + | 0x0327 | 无效的状态类型 | TODO | + | 0x0328 | 无效的原始数据版本 | TODO | + | 0x0329 | 无效的原始数据长度 | TODO | + | 0x032A | 无效的原始数据内容 | TODO | + | 0x032B | 无效的 wal 版本 | TODO | + | 0x032C | 对象创建中 | TODO | + | 0x032D | 对象停止中 | TODO | + | 0x0330 | dnode 已经存在 | 无需任何动作，放弃重复创建dnode的操作 | + | 0x0331 | dnode 不存在 | 确认所要查询或者操作的dnode ID 或者 end point 是否正确 | + | 0x0332 | vgroup 不存在 | 确认所要查询或者操作的vgroup ID 是否正确 | + | 0x0333 | 系统拒绝 drop 其角色是 leader 的 mnode | 放弃该操作 | + | 0x0334 | 没有足够的 dnode 创建所指定的 vgroups | 增加 dnode 或者修改现有dnode的配置参数 `supportVgroups` | + | 0x0335 | 集群中各个dnode的配置不一致 | 检查各个dnode的配置参数确保其一致 | + | 0x0338 | 所要查询或操作的vgroup不在所指定的dnode中| 检查vgroup ID 和 dnode ID是否正确 | + | 0x0339 | 所要查询或操作的 vgroup 已经在所指定的dnode中 | 检查 vgroup ID 和 dnode ID是否正确 | + | 0x033B | 集群 ID 不匹配 | TODO | + | 0x0340 | 该帐户已经存在 | 放弃重复创建帐户的操作 | + | 0x0342 | 无效的帐户选项 | 检查创建帐户时的参数选项是否正确 | + | 0x0343 | 帐户授权已经过期 | 联系管理员重新授权 | + | 0x0344 | 无效帐户 | 联系管理员确认帐户 | + | 0x0345 | 操作的帐户过多，无法支持 | 减少同时操作的帐户数 | + | 0x0350 | 用户已经存在 | 放弃重复创建用户的操作 | + | 0x0351 | 无效用户 | 检查并确认正确用户 | + | 0x0352 | 无效的用户名格式 | 查看参考手册修改用户名 | + | 0x0353 | 无效的密码格式 | 查看参考手册修改密码 | + | 0x0354 | 无法从连接中获取用户名 | 检查客户端环境初始化是否使用了正确的用户名 | + | 0x0355 | 一次尝试操作的用户过多 | 查看参考手册减少同时操作的用户数 | + | 0x0356 | 无效的修改操作 | 查看参考手册使用正确的操作 | + | 0x0357 | 认证失败 | 使用正确的用户名和密码 | + | 0x0360 | 要创建的超级表已经存在 | 放弃重复创建的操作或者删除该超级表再重新创建 | + | 0x0362 | 所使用或查询的超级表不存在 | 确认超级表名是否正确，如果正确则需要先创建超级表 | + | 0x0364 | 标签过多 | 查看参考手册减少标签数量 | + | 0x0365 | 列过多 | 查看参考手册减少列数量 | + | 0x0369 | 要添加的标签已经存在 | 修改标签名 | + | 0x036A | 要查询或修改的标签不存在 | 确认标签名是否正确 | + | 0x036B | 要添加的列已经存在 | 修改列名或者放弃该操作 | + | 0x036C | 要查询或修改的列不存在 | 确认列名是否正确 | + | 0x036E | 无效的超级表操作 | 查看参考手册进行正确的操作 | + | 0x036F | 错误的行字节数 | TODO | + | 0x0370 | 无效的函数名 | 确认函数名是否正确 | + | 0x0372 | 无效的函数代码 | 无效的函数编码 | + | 0x0373 | 该函数已经存在 | 修改函数名或者放弃该操作| + | 0x0374 | 所引用的函数不存在 | 确认函数名是否正确 | + | 0x0375 | 无效的 bufSize | 查看参考手册修改 bufSize | + | 0x0378 | 无效的函数注释 | 查看参考手册修改函数注释 | + | 0x0379 | 无效的函数检索消息 | TODO | + | 0x0380 | 未指定数据库或者指定的数据库不可用 | 指定数据库，或者检查所指定的数据库的状态 | + | 0x0381 | 数据库已经存在 | 放弃重复创建，或者修改数据库名 | + | 0x0382 | 无效的数据库参数 | 查看参考手册使用正确的参数 | + | 0x0383 | 无效的数据库名称 | 查看参考手册使用正确的数据库名 | + | 0x0385 | 该帐号下的数据库过多 | 删除旧的数据库再尝试创建新数据库 | + | 0x0388 | 数据库不存在 | 确认数据库名是否正确 | + | 0x0389 | 无效的数据库帐户 | 确认帐户是否正确 | + | 0x038A | 数据库参数未修改 | 查看参考手册确认修改的参数和值是否正确 | + | 0x038B | 索引不存在 |确认索引名称是否正确 | + | 0x039A | 无效的系统表名 | 查看参考手册确认表名是否正确 | + | 0x03A0 | mnode 已经存在 | 放弃该操作 | + | 0x03A1 | mnode 不存在 | 确认要查看或操作的 mnode ID | + | 0x03A2 | qnode 已经存在 | 放弃该操作 | + | 0x03A3 | qnode 不存在 | 确认要查看或操作的 qnode ID 是否正确 | + | 0x03A8 | mnode 的 replica 不能小于1 | 停止 drop mnode | + | 0x03A9 | mnode 的 replica 不能大于3 | 停止 create mnode | + | 0x03B0 | dnode 数量过多 | 停止添加新的 dnode | + | 0x03B1 | dnode 没有足够的可用内存 | 检查所在系统的内存使用情况 ，尝试释放出内存 | + | 0x03B2 | 无效的 dnode 配置 | 查看参考手册纠正配置 | + | 0x03B3 | 无效的 dnode 地址 | 确认 dnode 的 FQDN 和 serverPort参数是否正确 | + | 0x03B4 | 无效的 dnode ID | 确认正确的 dnode ID | + | 0x03B5 | vgroup 的分布未发生变化 | TODO | + | 0x03B6 | 存在状态为 offline 的 dnode | drop 这些 dnode 或者启动相应的 dnode 使其状态为 ready | + | 0x03B7 | 无效的 vgroup 副本 | TODO | + | 0x03C0 | topic 与超级表冲突 | TODO | + | 0x03C1 | 订阅了过多的超级表 | 查看参考手册减少超级表数量 | + | 0x03C2 | 无效的 超级表修改参数 | 查看参考手册进行纠正 | + | 0x03C3 | 超级表参数未被修改 | 查看参考手册确认参数是否正确 | + | 0x03C4 | 该字段被某个主题所使用 | TODO | + | 0x03C5 | 该数据库是单超级表模式 | 修改数据库为多超级表模式或者放弃创建新的超级表 | + | 0x03C6 | 修改超级表使用了无效的 schema 版本 | TODO | + | 0x03C7 | 修改超级表使用了无效的超级表 ID | 确认超级表使用是否正确 | + | 0x03C8 | 该字段被 tsma 所使用 | TODO | + | 0x03D0 | 该事务已经存在 | TODO | + | 0x03D1 | 该事务不存在 | TODO | + | 0x03D2 | 要 kill 的 stage 不存在 | TODO | + | 0x03D3 | 冲突的事务没有完成 | TODO | + | 0x03D4 | 未知的事务错误 | TODO | + | 0x03D5 | 事务提交日志已满 | TODO | + | 0x03DF | 在执行事务时无法建立连接 | 等待事务完成尝试重新建立连接 | + | 0x03E0 | Topic 已经存在 | 修改 topic 名字或者放弃创建重复的 topic | + | 0x03E1 | Topic 不存在 | 确认 Topic 名字是否正确 | + | 0x03E2 | Topic 过多 | 尝试删除不用的 topic 再建立新的，或者放弃此次操作 | + | 0x03E3 | 无效的 Topic | 确认 Topic 是否正确 | + | 0x03E4 | 建立 Topic 的查询子名无效 | 查看参考手册纠正查询子名 | + | 0x03E5 | 建立 Topic 的参数无效 | 查看参考手册使用正确的参数 | + | 0x03E6 | 消费者不存在 | 确认正确的消费者 ID | + | 0x03E7 | 消费者未修改 | TODO | + | 0x03E8 | 订阅不存在 | 确认正确的订阅 ID | + | 0x03E9 | 偏移量不存在 | 纠正偏移量 | + | 0x03EA | 消费者不可用 | TODO | + | 0x03EB | 无法删除已经被订阅的 Topic | 先取消订阅再尝试删除 | + | 0x03EC | Consumer group正在被某些消费者使用 | TODO | + | 0x03F0 | 流已经存在 | 修改流名称或者放弃创建该流 | + | 0x03F1 | 要查询或操作的流不存在 | 确认正确的流 ID | + | 0x03F2 | 无效的流参数 | 查看参考手册纠正错误的参数 | + | 0x0480 | SMA 已经存在 | 修改 SMA 名称或者放弃创建 | + | 0x0481 | SMA 不存在 | 确认正确的 SMA 名称或者 ID | + | 0x0482 | SMA 参数错误 | 查看参考手册纠正参数 | + | 0x0408 | 节点不在线 | TODO | + | 0x0409 | 节点已经部署 | TODO | + | 0x040A | 节点未部署 | TODO | + | 0x0500 | 该动作作在进行中| TODO | + | 0x0501 | 消息未被处理 | TODO | + | 0x0502 | 该动作需要被重新处理 | TODO | + | 0x0503 | 无效的 vgroup ID | 检查确认正确的 vgroups ID | + | 0x0504 | vnode 初始化失败 | TODO | + | 0x0505 | 系统磁盘空间耗尽 | 尝试释放或者增加磁盘空间 | + | 0x0506 | 对磁盘文件没有写权限 | 检查启动 TDengine 的系统帐号的写权限 | + | 0x0507 | 数据文件缺失 | TODO | + | 0x0508 | vnode 没有可用内存 | TODO | + | 0x0509 | vnode 中未预期的一般性错误 | TODO | + | 0x050C | 数据库无空闲内存 | TODO | + | 0x050D | 数据库正在删除中 | TODO | + | 0x050E | 数据库正在更新中 | TODO | + | 0x0510 | 数据库正在关闭中 | TODO | + | 0x0511 | 数据库被暂停操作 | TODO | + | 0x0512 | 数据库写操作被拒绝 | 检查用户权限，申请写操作授权 | + | 0x0513 | 数据库正在同步中 | TODO | + | 0x0514 | 无效的 tsdb 状态 | TODO | + | 0x0520 | 指定的表不存在 | 检查确认正确的表名 | + | 0x0521 | 指定的SMA 不存在 | 检查确认正确的 SMA名称 | + | 0x0522 | Hash 值不匹配 | TODO | + | 0x0523 | 指定的表不存在 | 检查确认正确的表名 | + | 0x0524 | 无效的表动作 | TODO | + | 0x0525 | 列名已经存在 | 修改列名或放弃操作 | + | 0x0526 | 列名不存在 | 确认正确的列名 | + | 0x0527 | 该列已经被订阅 | 先取消订阅再操作或者放弃操作 | + | 0x0528 | 无效的配置文件 | 检查配置文件的路径和访问权限 | + | 0x0529 | 无效的 term 文件 | TODO | + | 0x0600 | 无效的表 ID | 确认表名是否正确 | + | 0x0601 | 无效的表 类型 | TODO | + | 0x0602 | 无效的 schema 版本 | TODO | + | 0x0603 | 表已经存在 | 修改表名或放弃操作 | + | 0x0604 | 配置无效 | 查看参考手册纠正配置 | + | 0x0605 | TSDB 初始化失败 | TODO | + | 0x0606 | 磁盘空间耗尽 | 查看磁盘空间耗尽的原因，尝试释放或增加磁盘空间 | + | 0x0607 | 磁盘文件没有访问权限 | 确认启动集群的系统帐户是否有相应的写权限 | + | 0x0608 | 数据文件被破坏 | TODO | + | 0x0609 | 内存耗尽 | 检查内存被耗尽的原因，尝试释放内存 | + | 0x060A | 标签版本过老 | TODO | + | 0x060B | 时间戳不在允许范围内 | 查看参考手册了解允许写入的时间戳规则 | + | 0x060C | 提交消息被破坏 | TODO | + | 0x060D | 无效操作 | TODO | + | 0x060E | 建表消息无效 | TODO | + | 0x060F | 内存跳表中没有表的数据 | TODO | + | 0x0610 | 文件已经存在 | TODO | + | 0x0611 | 需要重新配置该表 | TODO | + | 0x0612 | 建表的信息无效 | TODO | + | 0x0613 | 磁盘空间耗尽 | 尝试释放或增加磁盘空间 | + | 0x0614 | 消息被破坏 |TODO | + | 0x0615 | 无效的标签值 | 修正标签值 | + | 0x0616 | 未缓存最后一行的原始数据 | 修改数据库的 cacheModel 参数 | + | 0x0618 | 该表不存在 | 检查表名是否正确 | + | 0x0619 | 超级表已经存在 | 修改超级表名再次尝试 | + | 0x061A | 超级表不存在 | 检查超级表名是否正确 | + | 0x061B | 表被重新创建 | TODO | + | 0x061C | TDB 环境打开错误 | N/A | + | 0x0700 | 无效的查询句柄 | N/A | + | 0x0701 | 无效的消息 | TODO | + | 0x0702 | 磁盘空间耗尽 | 尝试释放或增加磁盘空间 | + | 0x0703 | 系统内存耗尽 | 尝试释放内存 | + | 0x0704 | 未知错误 | TODO | + | 0x0705 | 重复的 Join Key | 修正查询语句中的 Join Key | + | 0x0706 | 标签过滤条件过多 | 减小查询语句中的标签过滤条件 | + | 0x0707 | 查询不可用 | TODO | + | 0x0708 | TODO | TODO | + | 0x0709 | TODO | TODO | + | 0x070A | 查询中的时间窗口过多 | 修改查询语句以减小时间窗口的数量 | + | 0x070B | 查询缓冲区达到上限 | TODO | + | 0x070C | 多副本数据不一致 | TODO | + | 0x070D | 系统错误 | TODO | + | 0x070E | 无效的时间范围 | 修正查询语句中的时间范围 | + | 0x070F | 无效输入 | 修正查询语句 | + | 0x0720 | 调度器不存在 | TODO | + | 0x0721 | 任务不存在 | TODO | + | 0x0722 | 任务已经存在 | TODO | + | 0x0723 | 任务上下文不存在 | TODO | + | 0x0724 | 任务被取消 | TODO | + | 0x0725 | 任务被停止 | TODO | + | 0x0726 | 任务正在取消中 | TODO | + | 0x0727 | 任务正在停止中 | TODO | + | 0x0728 | 重复操作 | TODO | + | 0x0729 | 任务消息错误 | TODO | + | 0x072A | 作业已经被释放 | TODO | + | 0x072B | 任务状态错误 | TODO | + | 0x072C | in 和 not in 操作符不支持 JSON 类型 | 修正查询语句 | + | 0x072D | 此处不支持 JSON |修正查询语句 | + | 0x072E | group 和 partition by 不支持 JSON 类型 | + | 0x072F | 查询作业不存在 | TODO | + | 0x0800 | License 已经过期 | 重新激活或获取 License | + | 0x0801 | 受限于 License 无法创建 dnode | 获取新的License | + | 0x0802 | 受限于 License 无法创建帐户 | 获取新的 License | + | 0x0803 | 受限于 License 无法创建表 | 获取新的 License | + | 0x0804 | 受限于 License 无法创建数据库 | 获取新的 License | + | 0x0805 | 受限于 License 无法创建用户 | 获取新的 License | + | 0x0806 | 受限于 License 无法创建连接 | 获取新的 License | + | 0x0807 | 受限于 License 无法创建流 | 获取新的 License | + | 0x0808 | 写入速度受限于 License | 获取新的 License | + | 0x0809 | 存储容量受限于 License | 获取新的 License | + | 0x080A | 查询时间受限于 License | 获取新的 License | + | 0x080B | CPU 核数受限于 License | 获取新的 License | + | 0x080C | 受限于 License 无法创建超级表 | 获取新的 License | + | 0x080D | 受限于 License 无法创建表 | 获取新的 License | + | 0x0A00 | TQ 无效配置 | TODO | + | 0x0A01 | TQ 初始化失败 | TODO | + | 0x0A02 | TQ 磁盘空间耗尽 | 尝试释放或增加磁盘空间 | + | 0x0A03 | TQ 没有写磁盘权限 | 确认启动集群的系统帐号是否具有写磁盘权限 | + | 0x0A04 | TQ 文件被破坏 | TODO | + | 0x0A05 | TQ 内存耗尽 | 尝试释放内存 | + | 0x0A06 | TQ 文件已经存在 | TODO | + | 0x0A07 | TQ 创建目录失败 | TODO | + | 0x0A08 | TQ meta 中不存在该 key | TODO | + | 0x0A09 | meta key在事务中不存在 | TODO | + | 0x0A0A | meta key在事务中重复 | TODO | + | 0x0A0B | 消费组不存在 | 指定正确的消费组 | + | 0x0A0C | 该表的 schema 不存在 | 确认表名是否正确 | + | 0x0A0D | 没有已经提交的 offset | TODO | + | 0x1000 | WAL 未知错误 | TODO | + | 0x1001 | WAL 文件被破坏 | TODO | + | 0x1002 | WAL 大小超出上限 | TODO | + | 0x1003 | WAL 使用了错误的版本号 | TODO | + | 0x1004 | 系统内存耗尽 | 尝试释放内存 | + | 0x1005 | WAL 日志不存在 | TODO | + | 0x2201 | 无效的 mount 配置 | 修正 mount 配置参数 | + | 0x2202 | mount 点过多 | TODO | + | 0x2203 | 重复的 primary mount | TODO | + | 0x2204 | primary mount 缺失 | TODO | + | 0x2205 | no mount at tier: TODO | TODO | + | 0x2206 | 文件已经存在 | 更改文件名或者删除该文件 | + | 0x2207 | 无效的级别 | TODO | + | 0x2208 | 没有可用磁盘 | TODO | + | 0x220F | 系统内存耗尽 | TODO | + | 0x2400 | catalog 内部错误 | TODO | + | 0x2401 | 无效的 catalog 输入参数 | TODO | + | 0x2402 | catalog 不可用 | TODO | + | 0x2403 | catalog 系统错误 | TODO | + | 0x2404 | 数据库被删除 | TODO | + | 0x2405 | catalog 不可用 | TODO | + | 0x2406 | 表元数据和 vgroup 不匹配 | TODO | + | 0x2407 | catalog 不存在 | TODO | + | 0x2550 | 无效的消息顺序 | TODO | + | 0x2501 | 调度器状态错误 | TODO | + | 0x2502 | 调度器内部错误 | TODO | + | 0x2504 | 任务超时 | TODO | + | 0x2505 | 作业正在停止中 | TODO | + | 0x2600 | 语法错误 | 参考 SQL 手册纠正 | + | 0x2601 | 不完整的 SQL 语句 | 参考 SQL 手册纠正 | + | 0x2602 | 无效列名 | 使用正确的列名 | + | 0x2603 | 表不存在 | 使用正确的表名 | + | 0x2604 | 表名定义有二义性 | 参考 SQL 手册纠正 | + | 0x2605 | 无效的值类型 | 参考 SQL 手册纠正 | + | 0x2608 | 此处不能使用聚合查询 | 参考 SQL 手册纠正 | + | 0x2609 | ORDER BY 只能用于查询语句中的结果列 | 参考 SQL 手册纠正 | + | 0x260A | GROUP BY 缺失表达式 (TODO) | 参考 SQL 手册纠正 | + | 0x260B | 不是 SELECT 表达式 | 参考 SQL 手册纠正 | + | 0x260C | 不是单一分组的分组函数 (TODO) | 参考 SQL 手册纠正 | + | 0x260D | 标签数量不匹配 | 参考 SQL 手册纠正 | + | 0x260E | 无效的标签名 | 改用正确的标签名 | + | 0x2610 | 名字或密码过长 | 参考 SQL 手册纠正 | + | 0x2611 | 密码不能为空 | 提供非空密码 | + | 0x2612 | 端口无效 | 端口号必须在 (0,65535) 范围内 | + | 0x2613 | 地址格式错误 | 正确格式是 "fqdn: port" | + | 0x2614 | 该语句不再支持 | 参考 SQL 手册纠正 | + | 0x2615 | 时间窗口过小 | 参考 SQL 手册纠正 | + | 0x2616 | 未指定数据库 | 在表名或超级表名前添加 "." 指定数据库 | + | 0x2617 | 标识符无效 | 参考 SQL 手册纠正 | + | 0x2618 | 该数据库中不存在对应的超级表 | 使用正确的数据库名或者超级表名 | + | 0x2619 | 数据库参数无效 | 参考 SQL 手册纠正 | + | 0x261A | 建表参数无效 | 参考 SQL 手册纠正 | + | 0x2624 | GROUP BY 和 窗口子句不能共用 | 参考 SQL 手册纠正 | + | 0x2627 | 聚合函数不支持嵌套 | 参考 SQL 手册纠正 | + | 0x2628 | 在 integer/bool/varchar 类型的列上只支持 状态窗口 | 参考 SQL 手册纠正 | + | 0x2629 | 标签列上不支持状态窗口 | 参考 SQL 手册纠正 | + | 0x262A | 状态窗口查询不支持超级表 | 参考 SQL 手册纠正 | + | 0x262B | 会话之间的 gap 应该是大于 0 的固定大小的窗口 | 参考 SQL 手册纠正 | + | 0x262C | 只在主键时间戳列上支持会话 | 参考 SQL 手册纠正 | + | 0x262D | 窗口偏移量不能是负值 | 参考 SQL 手册纠正 | + | 0x262E | 当 interval 的单位是 "year" 时 offset 的单位不能是 "month" | 参考 SQL 手册纠正 | + | 0x262F | offset 所指定的时间长度应该小于 interval 所指定的时间长度 | 参考 SQL 手册纠正 | + | 0x2630 | 当 interval 是自然年/月时不能使用 slidig | 参考 SQL 手册纠正 | + | 0x2631 | sliding 所指定的时间长度不能大于 interval 所指定的时间长度 | 参考 SQL 手册纠正 | + | 0x2632 | sliding 不能小于 interval 的 1%% | 参考 SQL 手册纠正 | + | 0x2633 | 当使用 JSON 类型的 tag 时只允许这一个 tag 的存在 | 去除其它 tag | + | 0x2634 | 查询块中包含的结果列的数量不正确 | TODO | + | 0x2635 | 时间戳不正确 | TODO | + | 0x2637 | offset/soffset 不能小于 0 | 纠正 offset/soffset | + | 0x2638 | offset/soffset 只能用于 partition by | 参考 SQL 手册纠正 | + | 0x2639 | 无效的 topic 查询 | TODO | + | 0x263A | 不能批量删除超级表 | 请逐个删除 | + | 0x263B | 查询时间范围未指定起止时间或者时间范围过大 | 参考 SQL 手册纠正 | + | 0x263C | 重复的列表 | 参考 SQL 手册纠正 | + | 0x263D | 标签长度超过上限 | 参考 SQL 手册纠正 | + | 0x263E | 行长度超过上限 | 参考 SQL 手册纠正 | + | 0x263F | 不合法的列数量 | 参考 SQL 手册纠正 | + | 0x2640 | 列数过多 | 参考 SQL 手册纠正 | + | 0x2641 | 首列必须是时间戳 | 参考 SQL 手册纠正 | + | 0x2642 | binary/nchar 类型的列长度无效 | 参考 SQL 手册纠正 | + | 0x2643 | 标签列数量无效 | 参考 SQL 手册纠正 | + | 0x2644 | 无权进行该操作 | 参考 SQL 手册纠正 | + | 0x2645 | 无效的流查询 | 参考 SQL 手册纠正 | + | 0x2646 | 无效的 _c0 或 _rowts 表达式 | 参考 SQL 手册纠正 | + | 0x2647 | 无效的时间线函数 | 参考 SQL 手册纠正 | + | 0x2648 | 无效的密码 | 参考 SQL 手册纠正 | + | 0x2649 | 无效的 alter table 语句 | 参考 SQL 手册纠正 | + | 0x264A | 不能删除时间戳主列 | 参考 SQL 手册纠正 | + | 0x264B | 只有 binary/nchar 类型的列能够修改长度 | 参考 SQL 手册纠正 | + | 0x264C | 无效的 tbname 伪列 | 参考 SQL 手册纠正 | + | 0x264D | 无效的函数名 | 参考 SQL 手册纠正 | + | 0x264E | 注释过长 | 参考 SQL 手册纠正 | + | 0x264F | 有些函数只能用在查询的 SELECT 列表中，且不能与其它非标量函数或列混用 | 参考 SQL 手册纠正 | + | 0x2650 | 不支持窗口查询，因为子查询的结果不包含时间戳列 | 参考 SQL 手册纠正 | + | 0x2651 | 任何列都不能被删除 | 参考 SQL 手册纠正 | + | 0x2652 | 只有 标签列可以是 JSON 类型 | 参考 SQL 手册纠正 | + | 0x2653 | 列或标签的值过长 | 参考 SQL 手册纠正 | + | 0x2655 | DELETE 语句必须有一个确定的时间范围 | 参考 SQL 手册纠正 | + | 0x2656 | REDISTRIBUTE VGROUP 语句只支持 1 到 3 个 vgroup | 参考 SQL 手册纠正 | + | 0x2657 | 不支持 Fill | 参考 SQL 手册纠正 | + | 0x2658 | 无效的窗口伪列 | 参考 SQL 手册纠正 | + | 0x2659 | 不允许做窗口查询: TODO | 参考 SQL 手册纠正 | + | 0x265A | 不允许做流计算: TODO | 参考 SQL 手册纠正 | + | 0x265B | 不允许做 Group By | 参考 SQL 手册纠正 | + | 0x265D | interp 子句错误 | 参考 SQL 手册纠正 | + | 0x265E | 窗口查询中不支持该函数 | 参考 SQL 手册纠正 | + | 0x265F | 只支持单表 | 参考 SQL 手册纠正 | + | 0x2660 | 无效的 SMA 索引 | 参考 SQL 手册纠正 | + | 0x2661 | 无效的 SELECT 表达式 | 参考 SQL 手册纠正 | + | 0x2662 | 获取表的元数据失败 | TODO | + | 0x2663 | 表名/表名不唯一 | 参考 SQL 手册纠正 | + | 0x266F | 解析器内部错误 | TODO | + | 0x2700 | 计划器内部错误 | TODO | + | 0x2701 | TODO | TODO | + | 0x2702 | 不支持 cross join | 参考 SQL 手册纠正 | + | 0x2800 | 函数内部错误 | 参考 SQL 手册纠正 | + | 0x2801 | 函数参数个数错误 | 参考 SQL 手册纠正 | + | 0x2802 | 函数参数类型错误 | 参考 SQL 手册纠正 | + | 0x2803 | 函数参数值错误 | 参考 SQL 手册纠正 | + | 0x2804 | 非内置函数 | 参考 SQL 手册纠正 | + | 0x2901 | UDF 正在停止 | TODO | + | 0x2902 | UDF 管道读取错误 | TODO | + | 0x2903 | UDF 连接错误 | TODO | + | 0x2904 | UDF 管道缺失 | TODO | + | 0x2905 | UDF 加载失败 | TODO | + | 0x2906 | UDF 无效状态 | TODO | + | 0x2907 | UDF 无效输入 | TODO | + | 0x2908 | UDF 没有函数句柄 | TODO | + | 0x2909 | UDF 无效的 bufsize | TODO | + | 0x290A | UDF 无效的输出类型 | TODO | + | 0x3000 | 无效的行协议类型 | 修正数据中的协议类型 | + | 0x3001 | 无效的时间戳精度类型 | 修正时间戳精度类型 | + | 0x3002 | 无效的数据格式 | 修正数据格式 | + | 0x3003 | 无效的无模式数据库配置 | 修改配置 | + | 0x3004 | 写入类型与之前的不同 | 修正写入类型 | + | 0x3100 | TSMA 初始化失败 | TODO | + | 0x3101 | TSMA 已经存在 | 放弃重复建立 TSMA | + | 0x3102 | 元数据中没有 TSMA 索引 | TODO | + | 0x3103 | 无效的 TSMA 环境 | TODO | + | 0x3104 | 无效的 TSMA 状态 | TODO | + | 0x3105 | 无效的 TSMA 指针 | TODO | + | 0x3106 | 无效的 TSMA 参数 | 参考 SQL 手册纠正 | + | 0x3107 | cache 中没有该 TSMA 的索引 | TODO | + | 0x3150 | 无效的 RSMA 索引 | TODO | + | 0x3151 | 无效的 RSMA 状态 | TODO | + | 0x3152 | RSMA 创建 qtaskinfo 失败 | TODO | + | 0x3153 | RSMA 文件被破坏 | TODO | + | 0x3200 | 索引正在重建中 |TODO | + | 0x3201 | 无效的索引文件 | TODO | + | 0x4000 | 无效消息 | TODO | \ No newline at end of file diff --git a/docs/zh/14-reference/03-connector/csharp.mdx b/docs/zh/14-reference/03-connector/csharp.mdx index 1e23df9286bf0cb3bf1db95e334301c04d01ad04..723c12932b410e9f85a0f35cd0c0b8273f4f7723 100644 --- a/docs/zh/14-reference/03-connector/csharp.mdx +++ b/docs/zh/14-reference/03-connector/csharp.mdx @@ -22,7 +22,9 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`，并通过 `TDengine.Connector` 连接 TDengine 集群，进行数据写入、查询等基本操作。 -`TDengine.Connector` 的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-dotnet)。 +注意：`TDengine.Connector` 3.x 不兼容 TDengine 2.x，如果在运行 TDengine 2.x 版本的环境下需要使用 C# 连接器请使用 TDengine.Connector 的 1.x 版本 。 + +`TDengine.Connector` 的源码托管在 [GitHub](https://github.com/taosdata/taos-connector-dotnet/tree/3.0)。 ## 支持的平台 @@ -63,15 +65,15 @@ dotnet add package TDengine.Connector -可以下载 TDengine 的源码，直接引用最新版本的 TDengine.Connector 库 +也可以[下载源码](https://github.com/taosdata/taos-connector-dotnet/tree/3.0)，直接引用 TDengine.Connector 库 ```bash -git clone https://github.com/taosdata/TDengine.git -cd TDengine/src/connector/C#/src/ -cp -r TDengineDriver/ myProject +git clone -b 3.0 https://github.com/taosdata/taos-connector-dotnet.git +cd taos-connector-dotnet +cp -r src/ myProject cd myProject -dotnet add TDengineDriver/TDengineDriver.csproj +dotnet add exmaple.csproj reference src/TDengine.csproj ``` @@ -145,20 +147,19 @@ namespace TDengineExample |示例程序 | 示例程序描述 | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [C#checker](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/C%23checker) | 使用 TDengine.Connector 可以通过 help 命令中提供的参数，测试C# Driver的同步写入和查询 | -| [TDengineTest](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/TDengineTest) | 使用 TDengine.Connector 实现的简单写入和查询的示例 | -| [insertCn](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/insertCn) | 使用 TDengine.Connector 实现的写入和查询中文字符的示例 | -| [jsonTag](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/jsonTag) | 使用 TDengine.Connector 实现的写入和查询 json tag 类型数据的示例 | -| [stmt](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/stmt) | 使用 TDengine.Connector 实现的参数绑定的示例 | -| [schemaless](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | -| [benchmark](https://github.com/taosdata/TDengine/tree/develop/examples/C%23/taosdemo) | 使用 TDengine.Connector 实现的简易 Benchmark | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/develop/examples/QueryAsyncSample.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | -| [subscribe](https://github.com/taosdata/taos-connector-dotnet/blob/develop/examples/SubscribeSample.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | +| [TMQ](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | ## 重要更新记录 | TDengine.Connector | 说明 | |--------------------|--------------------------------| +| 3.0.0 | 支持 TDengine 3.0.0.0，不兼容 2.x。新增接口TDengine.Impl.GetData()，解析查询结果。 | +| 1.0.7 | 修复 TDengine.Query()内存泄露。 | | 1.0.6 | 修复 schemaless 在 1.0.4 和 1.0.5 中失效 bug。 | | 1.0.5 | 修复 Windows 同步查询中文报错 bug。 | | 1.0.4 | 新增异步查询，订阅等功能。修复绑定参数 bug。 | diff --git a/docs/zh/14-reference/03-connector/node.mdx b/docs/zh/14-reference/03-connector/node.mdx index 9f2bed9e97cb33aeabfce3d69dc3774931b426c0..b089da99d26d0d671641fd0b50119853a04000a9 100644 --- a/docs/zh/14-reference/03-connector/node.mdx +++ b/docs/zh/14-reference/03-connector/node.mdx @@ -15,11 +15,11 @@ import NodeOpenTSDBTelnet from "../../07-develop/03-insert-data/_js_opts_telnet. import NodeOpenTSDBJson from "../../07-develop/03-insert-data/_js_opts_json.mdx"; import NodeQuery from "../../07-develop/04-query-data/_js.mdx"; -`td2.0-connector` 和 `td2.0-rest-connector` 是 TDengine 的官方 Node.js 语言连接器。Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。 +`@tdengine/client` 和 `@tdengine/rest` 是 TDengine 的官方 Node.js 语言连接器。 Node.js 开发人员可以通过它开发可以存取 TDengine 集群数据的应用软件。注意：从 TDengine 3.0 开始 Node.js 原生连接器的包名由 `td2.0-connector` 改名为 `@tdengine/client` 而 rest 连接器的包名由 `td2.0-rest-connector` 改为 `@tdengine/rest`。并且不与 TDengine 2.x 兼容。 -`td2.0-connector` 是**原生连接器**，它通过 TDengine 客户端驱动程序（taosc）连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。`td2.0-rest-connector` 是 **REST 连接器**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 的运行实例。REST 连接器可以在任何平台运行，但性能略为下降，接口实现的功能特性集合和原生接口有少量不同。 +`@tdengine/client` 是**原生连接器**，它通过 TDengine 客户端驱动程序（taosc）连接 TDengine 运行实例，支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。`@tdengine/rest` 是 **REST 连接器**，它通过 taosAdapter 提供的 REST 接口连接 TDengine 的运行实例。REST 连接器可以在任何平台运行，但性能略为下降，接口实现的功能特性集合和原生接口有少量不同。 -Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-connector-node)。 +Node.js 连接器源码托管在 [GitHub](https://github.com/taosdata/taos-connector-node/tree/3.0)。 ## 支持的平台 @@ -58,7 +58,7 @@ REST 连接器支持所有能运行 Node.js 的平台。 - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) -- `td2.0-connector` 2.0.6 支持 Node.js LTS v10.9.0 或更高版本, Node.js LTS v12.8.0 或更高版本；2.0.5 及更早版本支持 Node.js LTS v10.x 版本。其他版本可能存在包兼容性的问题 +- `@tdengine/client` 3.0.0 支持 Node.js LTS v10.9.0 或更高版本, Node.js LTS v12.8.0 或更高版本；其他版本可能存在包兼容性的问题 - `make` - C 语言编译器，[GCC](https://gcc.gnu.org) v4.8.5 或更高版本 @@ -90,14 +90,14 @@ REST 连接器支持所有能运行 Node.js 的平台。 ```bash -npm install td2.0-connector +npm install @tdengine/client ``` ```bash -npm i td2.0-rest-connector +npm install @tdengine/rest ``` @@ -109,13 +109,13 @@ npm i td2.0-rest-connector 验证方法： -- 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/TDengine/tree/develop/examples/nodejs/nodejsChecker.js)到本地。 +- 新建安装验证目录，例如：`~/tdengine-test`，下载 GitHub 上 [nodejsChecker.js 源代码](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/nodejsChecker.js)到本地。 - 在命令行中执行以下命令。 ```bash npm init -y -npm install td2.0-connector +npm install @tdengine/client node nodejsChecker.js host=localhost ``` @@ -128,11 +128,11 @@ node nodejsChecker.js host=localhost -安装并引用 `td2.0-connector` 包。 +安装并引用 `@tdengine/client` 包。 ```javascript //A cursor also needs to be initialized in order to interact with TDengine from Node.js. -const taos = require("td2.0-connector"); +const taos = require("@tdengine/client"); var conn = taos.connect({ host: "127.0.0.1", user: "root", @@ -149,12 +149,12 @@ conn.close(); -安装并引用 `td2.0-rest-connector` 包。 +安装并引用 `@tdengine/rest` 包。 ```javascript //A cursor also needs to be initialized in order to interact with TDengine from Node.js. -import { options, connect } from "td2.0-rest-connector"; -options.path = "/rest/sqlt"; +import { options, connect } from "@tdengine/rest"; +options.path = "/rest/sql"; // set host options.host = "localhost"; // set other options like user/passwd @@ -190,26 +190,23 @@ let cursor = conn.cursor(); + ## 更多示例程序 | 示例程序 | 示例程序描述 | | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------- | -| [connection](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/cursorClose.js) | 建立连接的示例。 | -| [stmtBindBatch](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindParamBatchSample.js) | 绑定多行参数插入的示例。 | -| [stmtBind](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindParamSample.js) | 一行一行绑定参数插入的示例。 | -| [stmtBindSingleParamBatch](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtBindSingleParamBatchSample.js) | 按列绑定参数插入的示例。 | -| [stmtUseResult](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/stmtUseResultSample.js) | 绑定参数查询的示例。 | -| [json tag](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testJsonTag.js) | Json tag 的使用示例。 | -| [Nanosecond](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testNanoseconds.js) | 时间戳为纳秒精度的使用的示例。 | -| [Microsecond](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testMicroseconds.js) | 时间戳为微秒精度的使用的示例。 | -| [schemless insert](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testSchemalessInsert.js) | schemless 插入的示例。 | -| [subscribe](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/testSubscribe.js) | 订阅的使用示例。 | -| [asyncQuery](https://github.com/taosdata/taos-connector-node/tree/develop/nodejs/examples/tset.js) | 异步查询的使用示例。 | -| [REST](https://github.com/taosdata/taos-connector-node/blob/develop/typescript-rest/example/example.ts) | 使用 REST 连接的 TypeScript 使用示例。 | +| [basicUse](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/queryExample.js) | 基本的使用如如建立连接，执行 SQL 等操作。 | +| [stmtBindBatch](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/bindParamBatch.js) | 绑定多行参数插入的示例。 | | +| [stmtBindSingleParamBatch](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/bindSingleParamBatch.js) | 按列绑定参数插入的示例。 | +| [stmtQuery](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/stmtQuery.js) | 绑定参数查询的示例。 | +| [schemless insert](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/schemaless.js) | schemless 插入的示例。 | +| [TMQ](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/tmq.js) | 订阅的使用示例。 | +| [asyncQuery](https://github.com/taosdata/taos-connector-node/blob/3.0/nodejs/examples/asyncQueryExample.js) | 异步查询的使用示例。 | +| [REST](https://github.com/taosdata/taos-connector-node/blob/3.0/typescript-rest/example/example.ts) | 使用 REST 连接的 TypeScript 使用示例。 | ## 使用限制 -Node.js 连接器 >= v2.0.6 目前支持 node 的版本为：支持 >=v12.8.0 <= v12.9.1 || >=v10.20.0 <= v10.9.0 ；2.0.5 及更早版本支持 v10.x 版本，其他版本可能存在包兼容性的问题。 +native 连接器（`@tdengine/client`） >= v3.0.0 目前支持 node 的版本为：支持 >=v12.8.0 <= v12.9.1 || >=v10.20.0 <= v10.9.0 ；2.0.5 及更早版本支持 v10.x 版本，其他版本可能存在包兼容性的问题。 ## 其他说明 @@ -225,7 +222,7 @@ Node.js 连接器的使用参见[视频教程](https://www.taosdata.com/blog/202 2. Node.js 版本 - 连接器 >v2.0.6 目前兼容的 Node.js 版本为：>=v10.20.0 <= v10.9.0 || >=v12.8.0 <= v12.9.1 + 原生连接器 `@tdengine/client` 目前兼容的 Node.js 版本为：>=v10.20.0 <= v10.9.0 || >=v12.8.0 <= v12.9.1 3. "Unable to establish connection"，"Unable to resolve FQDN" @@ -235,18 +232,22 @@ Node.js 连接器的使用参见[视频教程](https://www.taosdata.com/blog/202 ### 原生连接器 -| td2.0-connector 版本 | 说明 | -| -------------------- | ---------------------------------------------------------------- | -| 2.0.12 | 修复 cursor.close() 报错的 bug。 | -| 2.0.11 | 支持绑定参数、json tag、schemaless 接口等功能。 | -| 2.0.10 | 支持连接管理，普通查询、连续查询、获取系统信息、订阅功能等功能。 | - +| package name | version | TDengine version | 说明 | +|------------------|---------|---------------------|------------------------------------------------------------------| +| @tdengine/client | 3.0.0 | 3.0.0 | 支持TDengine 3.0 且不与2.x 兼容。 | +| td2.0-connector | 2.0.12 | 2.4.x；2.5.x；2.6.x | 修复 cursor.close() 报错的 bug。 | +| td2.0-connector | 2.0.11 | 2.4.x；2.5.x；2.6.x | 支持绑定参数、json tag、schemaless 接口等功能。 | +| td2.0-connector | 2.0.10 | 2.4.x；2.5.x；2.6.x | 支持连接管理，普通查询、连续查询、获取系统信息、订阅功能等功能。 | ### REST 连接器 -| td2.0-rest-connector 版本 | 说明 | -| ------------------------- | ---------------------------------------------------------------- | -| 1.0.3 | 支持连接管理、普通查询、获取系统信息、错误信息、连续查询等功能。 | +| package name | version | TDengine version | 说明 | +|----------------------|---------|---------------------|---------------------------------------------------------------------------| +| @tdengine/rest | 3.0.0 | 3.0.0 | 支持 TDegnine 3.0，且不与2.x 兼容。 | +| td2.0-rest-connector | 1.0.7 | 2.4.x；2.5.x；2.6.x | 移除默认端口 6041。 | +| td2.0-rest-connector | 1.0.6 | 2.4.x；2.5.x；2.6.x | 修复create，insert，update，alter 等SQL 执行返回的 affectRows 错误的bug。 | +| td2.0-rest-connector | 1.0.5 | 2.4.x；2.5.x；2.6.x | 支持云服务 cloud Token； | +| td2.0-rest-connector | 1.0.3 | 2.4.x；2.5.x；2.6.x | 支持连接管理、普通查询、获取系统信息、错误信息、连续查询等功能。 | ## API 参考 -[API 参考](https://docs.taosdata.com/api/td2.0-connector/) +[API 参考](https://docs.taosdata.com/api/td2.0-connector/) \ No newline at end of file diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 6b694543b1db435f507b5e2fb325cebe76261b48..f84ec65b4c8574c0812567a65213d7605b306c99 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -227,45 +227,34 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) #### 数据库相关配置参数 -创建数据库时的相关参数在 json 配置文件中的 `dbinfo` 中配置，具体参数如下。这些参数与 TDengine 中 `create database` 时所指定的数据库参数相对应。 +创建数据库时的相关参数在 json 配置文件中的 `dbinfo` 中配置，个别具体参数如下。其余参数均与 TDengine 中 `create database` 时所指定的数据库参数相对应，详见[../../taos-sql/database] - **name** : 数据库名。 - **drop** : 插入前是否删除数据库，默认为 true。 -- **replica** : 创建数据库时指定的副本数。 +#### 流式计算相关配置参数 -- **days** : 单个数据文件中存储数据的时间跨度，默认值为 10。 +创建流式计算的相关参数在 json 配置文件中的 `stream` 中配置，具体参数如下。 -- **cache** : 缓存块的大小，单位是 MB，默认值是 16。 +- **stream_name** : 流式计算的名称，必填项。 -- **blocks** : 每个 vnode 中缓存块的数量，默认为 6。 +- **stream_stb** : 流式计算对应的超级表名称，必填项。 -- **precision** : 数据库时间精度，默认值为 "ms"。 +- **stream_sql** : 流式计算的sql语句，必填项。 -- **keep** : 保留数据的天数，默认值为 3650。 +- **trigger_mode** : 流式计算的触发模式，可选项。 -- **minRows** : 文件块中的最小记录数，默认值为 100。 +- **watermark** : 流式计算的水印，可选项。 -- **maxRows** : 文件块中的最大记录数，默认值为 4096。 - -- **comp** : 文件压缩标志，默认值为 2。 - -- **walLevel** : WAL 级别，默认为 1。 - -- **cacheLast** : 是否允许将每个表的最后一条记录保留在内存中，默认值为 0，可选值为 0，1，2，3。 - -- **quorum** : 多副本模式下的写确认数量，默认值为 1。 - -- **fsync** : 当 wal 设置为 2 时，fsync 的间隔时间，单位为 ms，默认值为 3000。 - -- **update** : 是否支持数据更新，默认值为 0， 可选值为 0， 1， 2。 +- **drop** : 是否创建流式计算，可选项为 "yes" 或者 "no", 为 "no" 时不创建。 #### 超级表相关配置参数 -创建超级表时的相关参数在 json 配置文件中的 `super_tables` 中配置，具体参数如下表。 +创建超级表时的相关参数在 json 配置文件中的 `super_tables` 中配置，具体参数如下。 - **name**: 超级表名，必须配置，没有默认值。 + - **child_table_exists** : 子表是否已经存在，默认值为 "no"，可选值为 "yes" 或 "no"。 - **child_table_count** : 子表的数量，默认值为 10。 @@ -316,6 +305,22 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **tags_file** : 仅当 insert_mode 为 taosc, rest 的模式下生效。 最终的 tag 的数值与 childtable_count 有关，如果 csv 文件内的 tag 数据行小于给定的子表数量，那么会循环读取 csv 文件数据直到生成 childtable_count 指定的子表数量；否则则只会读取 childtable_count 行 tag 数据。也即最终生成的子表数量为二者取小。 +#### tsma配置参数 + +指定tsma的配置参数在 `super_tables` 中的 `tsmas` 中，具体参数如下。 + +- **name** : 指定 tsma 的名字，必选项。 + +- **function** : 指定 tsma 的函数，必选项。 + +- **interval** : 指定 tsma 的时间间隔，必选项。 + +- **sliding** : 指定 tsma 的窗口时间位移，必选项。 + +- **custom** : 指定 tsma 的创建语句结尾追加的自定义配置，可选项。 + +- **start_when_inserted** : 指定当插入多少行时创建 tsma，可选项，默认为 0。 + #### 标签列与数据列配置参数 指定超级表标签列与数据列的配置参数分别在 `super_tables` 中的 `columns` 和 `tag` 中。 @@ -335,6 +340,8 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **values** : nchar/binary 列/标签的值域，将从值中随机选择。 +- **sma**: 将该列加入bsma中，值为 "yes" 或者 "no"，默认为 "no"。 + #### 插入行为配置参数 - **thread_count** : 插入数据的线程数量，默认为 8。 diff --git a/docs/zh/14-reference/11-docker/index.md b/docs/zh/14-reference/11-docker/index.md index d0b7f1420420655909f6ceec5b249d2dad16dd3c..e40f0dbd359c64ba36e841cdfa92841211180837 100644 --- a/docs/zh/14-reference/11-docker/index.md +++ b/docs/zh/14-reference/11-docker/index.md @@ -147,7 +147,7 @@ import ( "fmt" "time" - _ "github.com/taosdata/driver-go/v2/taosSql" + _ "github.com/taosdata/driver-go/v3/taosSql" ) type config struct { diff --git a/docs/zh/17-operation/17-diagnose.md b/docs/zh/17-operation/17-diagnose.md index e2a2ef035a33a295b206c77ec08edf8f7842671f..e6e9be7153dee855867c4ba4fcd1d3258c9d788f 100644 --- a/docs/zh/17-operation/17-diagnose.md +++ b/docs/zh/17-operation/17-diagnose.md @@ -1,131 +1,71 @@ ---- -title: 诊断及其他 ---- - -## 网络连接诊断 - -当出现客户端应用无法访问服务端时，需要确认客户端与服务端之间网络的各端口连通情况，以便有针对性地排除故障。 - -目前网络连接诊断支持在：Linux 与 Linux，Linux 与 Windows 之间进行诊断测试。 - -诊断步骤： - -1. 如拟诊断的端口范围与服务器 taosd 实例的端口范围相同，须先停掉 taosd 实例 -2. 服务端命令行输入：`taos -n server -P -l ` 以服务端身份启动对端口 port 为基准端口的监听 -3. 客户端命令行输入：`taos -n client -h -P -l ` 以客户端身份启动对指定的服务器、指定的端口发送测试包 - --l ： 测试网络包的大小（单位：字节）。最小值是 11、最大值是 64000，默认值为 1000。 -注：两端命令行中指定的测试包长度必须一致，否则测试显示失败。 - -服务端运行正常的话会输出以下信息： - -```bash -# taos -n server -P 6000 -12/21 14:50:13.522509 0x7f536f455200 UTL work as server, host:172.27.0.7 startPort:6000 endPort:6011 pkgLen:1000 - -12/21 14:50:13.522659 0x7f5352242700 UTL TCP server at port:6000 is listening -12/21 14:50:13.522727 0x7f5351240700 UTL TCP server at port:6001 is listening -... -... -... -12/21 14:50:13.523954 0x7f5342fed700 UTL TCP server at port:6011 is listening -12/21 14:50:13.523989 0x7f53437ee700 UTL UDP server at port:6010 is listening -12/21 14:50:13.524019 0x7f53427ec700 UTL UDP server at port:6011 is listening -12/21 14:50:22.192849 0x7f5352242700 UTL TCP: read:1000 bytes from 172.27.0.8 at 6000 -12/21 14:50:22.192993 0x7f5352242700 UTL TCP: write:1000 bytes to 172.27.0.8 at 6000 -12/21 14:50:22.237082 0x7f5351a41700 UTL UDP: recv:1000 bytes from 172.27.0.8 at 6000 -12/21 14:50:22.237203 0x7f5351a41700 UTL UDP: send:1000 bytes to 172.27.0.8 at 6000 -12/21 14:50:22.237450 0x7f5351240700 UTL TCP: read:1000 bytes from 172.27.0.8 at 6001 -12/21 14:50:22.237576 0x7f5351240700 UTL TCP: write:1000 bytes to 172.27.0.8 at 6001 -12/21 14:50:22.281038 0x7f5350a3f700 UTL UDP: recv:1000 bytes from 172.27.0.8 at 6001 -12/21 14:50:22.281141 0x7f5350a3f700 UTL UDP: send:1000 bytes to 172.27.0.8 at 6001 -... -... -... -12/21 14:50:22.677443 0x7f5342fed700 UTL TCP: read:1000 bytes from 172.27.0.8 at 6011 -12/21 14:50:22.677576 0x7f5342fed700 UTL TCP: write:1000 bytes to 172.27.0.8 at 6011 -12/21 14:50:22.721144 0x7f53427ec700 UTL UDP: recv:1000 bytes from 172.27.0.8 at 6011 -12/21 14:50:22.721261 0x7f53427ec700 UTL UDP: send:1000 bytes to 172.27.0.8 at 6011 -``` - -客户端运行正常会输出以下信息： - -```bash -# taos -n client -h 172.27.0.7 -P 6000 -12/21 14:50:22.192434 0x7fc95d859200 UTL work as client, host:172.27.0.7 startPort:6000 endPort:6011 pkgLen:1000 - -12/21 14:50:22.192472 0x7fc95d859200 UTL server ip:172.27.0.7 is resolved from host:172.27.0.7 -12/21 14:50:22.236869 0x7fc95d859200 UTL successed to test TCP port:6000 -12/21 14:50:22.237215 0x7fc95d859200 UTL successed to test UDP port:6000 -... -... -... -12/21 14:50:22.676891 0x7fc95d859200 UTL successed to test TCP port:6010 -12/21 14:50:22.677240 0x7fc95d859200 UTL successed to test UDP port:6010 -12/21 14:50:22.720893 0x7fc95d859200 UTL successed to test TCP port:6011 -12/21 14:50:22.721274 0x7fc95d859200 UTL successed to test UDP port:6011 -``` - -仔细阅读打印出来的错误信息，可以帮助管理员找到原因，以解决问题。 - -## 启动状态及 RPC 诊断 - -`taos -n startup -h ` - -判断 taosd 服务端是否成功启动，是数据库管理员经常遇到的一种情形。特别当若干台服务器组成集群时，判断每个服务端实例是否成功启动就会是一个重要问题。除检索 taosd 服务端日志文件进行问题定位、分析外，还可以通过 `taos -n startup -h ` 来诊断一个 taosd 进程的启动状态。 - -针对多台服务器组成的集群，当服务启动过程耗时较长时，可通过该命令行来诊断每台服务器的 taosd 实例的启动状态，以准确定位问题。 - -`taos -n rpc -h ` - -该命令用来诊断已经启动的 taosd 实例的端口是否可正常访问。如果 taosd 程序异常或者失去响应，可以通过 `taos -n rpc -h ` 来发起一个与指定 fqdn 的 rpc 通信，看看 taosd 是否能收到，以此来判定是网络问题还是 taosd 程序异常问题。 - -## sync 及 arbitrator 诊断 - -``` -taos -n sync -P 6040 -h -taos -n sync -P 6042 -h -``` - -用来诊断 sync 端口是否工作正常，判断服务端 sync 模块是否成功工作。另外，-P 6042 用来诊断 arbitrator 是否配置正常，判断指定服务器的 arbitrator 是否能正常工作。 - -## 网络速度诊断 - -`taos -n speed -h -P 6030 -N 10 -l 10000000 -S TCP` - -从 2.2.0.0 版本开始，taos 工具新提供了一个网络速度诊断的模式，可以对一个正在运行中的 taosd 实例或者 `taos -n server` 方式模拟的一个服务端实例，以非压缩传输的方式进行网络测速。这个模式下可供调整的参数如下： - --n：设为“speed”时，表示对网络速度进行诊断。 --h：所要连接的服务端的 FQDN 或 ip 地址。如果不设置这一项，会使用本机 taos.cfg 文件中 FQDN 参数的设置作为默认值。 --P：所连接服务端的网络端口。默认值为 6030。 --N：诊断过程中使用的网络包总数。最小值是 1、最大值是 10000，默认值为 100。 --l：单个网络包的大小（单位：字节）。最小值是 1024、最大值是 1024 `*` 1024 `*` 1024，默认值为 1024。 --S：网络封包的类型。可以是 TCP 或 UDP，默认值为 TCP。 - -## FQDN 解析速度诊断 - -`taos -n fqdn -h ` - -从 2.2.0.0 版本开始，taos 工具新提供了一个 FQDN 解析速度的诊断模式，可以对一个目标 FQDN 地址尝试解析，并记录解析过程中所消耗的时间。这个模式下可供调整的参数如下： - --n：设为“fqdn”时，表示对 FQDN 解析进行诊断。 --h：所要解析的目标 FQDN 地址。如果不设置这一项，会使用本机 taos.cfg 文件中 FQDN 参数的设置作为默认值。 - -## 服务端日志 - -taosd 服务端日志文件标志位 debugflag 默认为 131，在 debug 时往往需要将其提升到 135 或 143 。 - -一旦设定为 135 或 143，日志文件增长很快，特别是写入、查询请求量较大时，增长速度惊人。如合并保存日志，很容易把日志内的关键信息（如配置信息、错误信息等）冲掉。为此，服务端将重要信息日志与其他日志分开存放： - -- taosinfo 存放重要信息日志, 包括：INFO/ERROR/WARNING 级别的日志信息。不记录 DEBUG、TRACE 级别的日志。 -- taosdlog 服务器端生成的日志，记录 taosinfo 中全部信息外，还根据设置的日志输出级别，记录 DEBUG（日志级别 135）、TRACE（日志级别是 143）。 - -## 客户端日志 - -每个独立运行的客户端（一个进程）生成一个独立的客户端日志，其命名方式采用 taoslog+<序号> 的方式命名。文件标志位 debugflag 默认为 131，在 debug 时往往需要将其提升到 135 或 143 。 - -- taoslog 客户端（driver）生成的日志，默认记录客户端 INFO/ERROR/WARNING 级别日志，还根据设置的日志输出级别，记录 DEBUG（日志级别 135）、TRACE（日志级别是 143）。 - -其中，日志文件最大长度由 numOfLogLines 来进行配置，一个 taosd 实例最多保留两个文件。 - -taosd 服务端日志采用异步落盘写入机制，优点是可以避免硬盘写入压力太大，对性能造成很大影响。缺点是，在极端情况下，存在少量日志行数丢失的可能。 +--- +title: 诊断及其他 +--- + +## 网络连接诊断 + +当出现客户端应用无法访问服务端时，需要确认客户端与服务端之间网络的各端口连通情况，以便有针对性地排除故障。 + +目前网络连接诊断支持在：Linux 与 Linux，Linux 与 Windows 之间进行诊断测试。 + +诊断步骤： + +1. 如拟诊断的端口范围与服务器 taosd 实例的端口范围相同，须先停掉 taosd 实例 +2. 服务端命令行输入：`taos -n server -P -l ` 以服务端身份启动对端口 port 为基准端口的监听 +3. 客户端命令行输入：`taos -n client -h -P -l ` 以客户端身份启动对指定的服务器、指定的端口发送测试包 + +-l ： 测试网络包的大小（单位：字节）。最小值是 11、最大值是 64000，默认值为 1000。 +注：两端命令行中指定的测试包长度必须一致，否则测试显示失败。 + +服务端运行正常的话会输出以下信息： + +```bash +# taos -n server -P 6030 -l 1000 +network test server is initialized, port:6030 +request is received, size:1000 +request is received, size:1000 +... +... +... +request is received, size:1000 +request is received, size:1000 +``` + +客户端运行正常会输出以下信息： + +```bash +# taos -n client -h 172.27.0.7 -P 6000 +taos -n client -h v3s2 -P 6030 -l 1000 +network test client is initialized, the server is v3s2:6030 +request is sent, size:1000 +response is received, size:1000 +request is sent, size:1000 +response is received, size:1000 +... +... +... +request is sent, size:1000 +response is received, size:1000 +request is sent, size:1000 +response is received, size:1000 + +total succ: 100/100 cost: 16.23 ms speed: 5.87 MB/s +``` + +仔细阅读打印出来的错误信息，可以帮助管理员找到原因，以解决问题。 + +## 服务端日志 + +taosd 服务端日志文件标志位 debugflag 默认为 131，在 debug 时往往需要将其提升到 135 或 143 。 + +一旦设定为 135 或 143，日志文件增长很快，特别是写入、查询请求量较大时，增长速度惊人。请注意日志文件目录所在磁盘的空间大小。 + +## 客户端日志 + +每个独立运行的客户端（一个进程）生成一个独立的客户端日志，其命名方式采用 taoslog+<序号> 的方式命名。文件标志位 debugflag 默认为 131，在 debug 时往往需要将其提升到 135 或 143 。 + +- taoslog 客户端（driver）生成的日志，默认记录客户端 INFO/ERROR/WARNING 级别日志，还根据设置的日志输出级别，记录 DEBUG（日志级别 135）、TRACE（日志级别是 143）。 + +其中，日志文件最大长度由 numOfLogLines 来进行配置，一个 taosd 实例最多保留两个文件。 + +taosd 服务端日志采用异步落盘写入机制，优点是可以避免硬盘写入压力太大，对性能造成很大影响。缺点是，在极端情况下，存在少量日志行数丢失的可能。当问题分析需要的时候，可以考虑将 参数 asynclog 设置成 0，修改为同步落盘写入机制，保证日志不会丢失。 diff --git a/docs/zh/28-releases.md b/docs/zh/28-releases.md new file mode 100644 index 0000000000000000000000000000000000000000..5f30325829bda75d466118f69f8516908c4c99ab --- /dev/null +++ b/docs/zh/28-releases.md @@ -0,0 +1,9 @@ +--- +sidebar_label: 发布历史 +title: 发布历史 +--- + +import Release from "/components/Release"; + + + diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 3686251b4b17dcef0553e912a7babb04404461e9..1cdd4c02daf0e1158745ff0d51a0a35d9934041c 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -1,473 +1,287 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include "taos.h" - -static int running = 1; -static void msg_process(TAOS_RES* msg) { - char buf[1024]; - /*memset(buf, 0, 1024);*/ - printf("topic: %s\n", tmq_get_topic_name(msg)); - printf("db: %s\n", tmq_get_db_name(msg)); - printf("vg: %d\n", tmq_get_vgroup_id(msg)); - if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data raw = {0}; - int32_t code = tmq_get_raw(msg, &raw); - if (code == 0) { - TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return; - } - - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 5"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - return; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return; - } - taos_free_result(pRes); - - int32_t ret = tmq_write_raw(pConn, raw); - printf("write raw data: %s\n", tmq_err2str(ret)); - taos_close(pConn); - } - char* result = tmq_get_json_meta(msg); - if (result) { - printf("meta result: %s\n", result); - } - tmq_free_json_meta(result); - return; - } - while (1) { - TAOS_ROW row = taos_fetch_row(msg); - if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - taos_print_row(buf, row, fields, numOfFields); - printf("%s\n", buf); - - const char* tbName = tmq_get_table_name(msg); - if (tbName) { - printf("from tb: %s\n", tbName); - } - } -} - -int32_t init_env() { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 5"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct0 values(now, 1, 2, 'a')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into ct3 values(now, 5, 6, 'c')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - -#if 0 - pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); - if (taos_errno(pRes) != 0) { - printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table ct3 ct1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); - if (taos_errno(pRes) != 0) { - printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 comment 'hello'"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "alter table n1 drop column c1"); - if (taos_errno(pRes) != 0) { - printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table n1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table jt2 using jt tags('')"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, - "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " - "nchar(8), t4 bool)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop table st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); -#endif - - return 0; -} - -int32_t create_topic() { - printf("create topic\n"); - TAOS_RES* pRes; - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - if (pConn == NULL) { - return -1; - } - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); - if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create topic topic2 as select ts, c1, c2, c3 from st1"); - if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - -#if 0 - pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); - pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')"); - if (taos_errno(pRes) != 0) { - printf("failed to insert, reason:%s\n", taos_errstr(pRes)); - return -1; - } - taos_free_result(pRes); -#endif - - taos_close(pConn); - return 0; -} - -void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - printf("commit %d tmq %p param %p\n", code, tmq, param); -} - -tmq_t* build_consumer() { -#if 0 - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); -#endif - - tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "group.id", "tg2"); - tmq_conf_set(conf, "client.id", "my app 1"); - tmq_conf_set(conf, "td.connect.user", "root"); - tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "msg.with.table.name", "true"); - tmq_conf_set(conf, "enable.auto.commit", "true"); - - /*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/ - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); - assert(tmq); - tmq_conf_destroy(conf); - return tmq; -} - -tmq_list_t* build_topic_list() { - tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "topic_ctb_column"); - /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/ - return topic_list; -} - -void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - int32_t code; - - if ((code = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); - printf("subscribe err\n"); - return; - } - int32_t cnt = 0; - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, -1); - if (tmqmessage) { - cnt++; - msg_process(tmqmessage); - /*if (cnt >= 2) break;*/ - /*printf("get data\n");*/ - taos_free_result(tmqmessage); - /*} else {*/ - /*break;*/ - /*tmq_commit_sync(tmq, NULL);*/ - } - } - - code = tmq_consumer_close(tmq); - if (code) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - static const int MIN_COMMIT_COUNT = 1; - - int msg_count = 0; - int32_t code; - - if ((code = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); - return; - } - - tmq_list_t* subList = NULL; - tmq_subscription(tmq, &subList); - char** subTopics = tmq_list_to_c_array(subList); - int32_t sz = tmq_list_get_size(subList); - printf("subscribed topics: "); - for (int32_t i = 0; i < sz; i++) { - printf("%s, ", subTopics[i]); - } - printf("\n"); - tmq_list_destroy(subList); - - while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); - if (tmqmessage) { - msg_process(tmqmessage); - taos_free_result(tmqmessage); - - /*tmq_commit_sync(tmq, NULL);*/ - /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ - } - } - - code = tmq_consumer_close(tmq); - if (code) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - else - fprintf(stderr, "%% Consumer closed\n"); -} - -int main(int argc, char* argv[]) { - if (argc > 1) { - printf("env init\n"); - if (init_env() < 0) { - return -1; - } - create_topic(); - } - tmq_t* tmq = build_consumer(); - tmq_list_t* topic_list = build_topic_list(); - basic_consume_loop(tmq, topic_list); - /*sync_consume_loop(tmq, topic_list);*/ -} +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" + +static int running = 1; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; +static char topicName[64] = "topicname"; + +static int32_t msg_process(TAOS_RES* msg) { + char buf[1024]; + int32_t rows = 0; + + const char* topicName = tmq_get_topic_name(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); + + printf("topic: %s\n", topicName); + printf("db: %s\n", dbName); + printf("vgroup id: %d\n", vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; + taos_print_row(buf, row, fields, numOfFields); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + } + + return rows; +} + +static int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes; + // drop database if exists + printf("create database\n"); + pRes = taos_query(pConn, "drop database if exists tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create database + pRes = taos_query(pConn, "create database tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create super table + printf("create super table\n"); + pRes = taos_query(pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create sub tables + printf("create sub tables\n"); + pRes = taos_query(pConn, "create table tmqdb.ctb0 using tmqdb.stb tags(0, 'subtable0')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb1 using tmqdb.stb tags(1, 'subtable1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb2 using tmqdb.stb tags(2, 'subtable2')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb3 using tmqdb.stb tags(3, 'subtable3')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // insert data + printf("insert data into sub tables\n"); + pRes = taos_query(pConn, "insert into tmqdb.ctb0 values(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb1 values(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in use tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); + pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("tmq_commit_cb_print() code: %d, tmq: %p, param: %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_res_t code; + tmq_conf_t* conf = tmq_conf_new(); + code = tmq_conf_set(conf, "enable.auto.commit", "true"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "group.id", "cgrpName"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "td.connect.user", "root"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "msg.with.table.name", "true"); + if (TMQ_CONF_OK != code) return NULL; + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topicList = tmq_list_new(); + int32_t code = tmq_list_append(topicList, "topicname"); + if (code) { + return NULL; + } + return topicList; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topicList))) { + fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); + return; + } + + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t consumeDelay = 5000; + while (running) { + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + if (tmqmsg) { + msgCnt++; + totalRows += msg_process(tmqmsg); + taos_free_result(tmqmsg); + } else { + break; + } + } + + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + +int main(int argc, char* argv[]) { + int32_t code; + + if (init_env() < 0) { + return -1; + } + + if (create_topic() < 0) { + return -1; + } + + tmq_t* tmq = build_consumer(); + if (NULL == tmq) { + fprintf(stderr, "%% build_consumer() fail!\n"); + return -1; + } + + tmq_list_t* topic_list = build_topic_list(); + if (NULL == topic_list) { + return -1; + } + + basic_consume_loop(tmq, topic_list); + + code = tmq_unsubscribe(tmq); + if (code) { + fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); + } + else { + fprintf(stderr, "%% unsubscribe\n"); + } + + code = tmq_consumer_close(tmq); + if (code) { + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + } + else { + fprintf(stderr, "%% Consumer closed\n"); + } + + return 0; +} diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 4eea744be19b51ebb321629c92c8d7e1f515fcda..be18ef1fc0d9516db76067e3fbc3a48c2ea7d581 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -104,9 +104,11 @@ typedef struct SDataBlockInfo { uint32_t capacity; // TODO: optimize and remove following int64_t version; // used for stream, and need serialization + int64_t ts; // used for stream, and need serialization int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize STimeWindow calWin; // used for stream, do not serialize + TSKEY watermark;// used for stream } SDataBlockInfo; typedef struct SSDataBlock { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5ca6a337a8aecbc03e04e45673b4b689aa872289..ca57e33a669227f4db5559671c9bd7ea05fdc7cd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1370,6 +1370,7 @@ typedef struct { int64_t skey; int64_t ekey; int64_t version; // for stream + TSKEY watermark;// for stream char data[]; } SRetrieveTableRsp; @@ -2657,6 +2658,34 @@ typedef struct { SEpSet epSet; } SVgEpSet; +typedef struct { + int64_t refId; + int64_t suid; + int8_t level; +} SRSmaFetchMsg; + +static FORCE_INLINE int32_t tEncodeSRSmaFetchMsg(SEncoder* pCoder, const SRSmaFetchMsg* pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->refId) < 0) return -1; + if (tEncodeI64(pCoder, pReq->suid) < 0) return -1; + if (tEncodeI8(pCoder, pReq->level) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static FORCE_INLINE int32_t tDecodeSRSmaFetchMsg(SDecoder* pCoder, SRSmaFetchMsg* pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->refId) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->level) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX @@ -3055,6 +3084,7 @@ int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); typedef struct { + int32_t msgIdx; int32_t msgType; int32_t msgLen; void* msg; @@ -3068,12 +3098,13 @@ typedef struct { typedef struct { int32_t reqType; + int32_t msgIdx; int32_t msgLen; int32_t rspCode; void* msg; } SBatchRsp; -static FORCE_INLINE void tFreeSBatchRsp(void *p) { +static FORCE_INLINE void tFreeSBatchRsp(void* p) { if (NULL == p) { return; } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 1d1849f24a72b33e2eff3ec23c8076b74f785206..ee566d759a980cf42cc8a77e81a7cc2e639b707c 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -58,12 +58,17 @@ typedef struct SDbInfo { int64_t dbId; } SDbInfo; +typedef struct STablesReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SArray* pTables; +} STablesReq; + typedef struct SCatalogReq { SArray* pDbVgroup; // element is db full name SArray* pDbCfg; // element is db full name SArray* pDbInfo; // element is db full name - SArray* pTableMeta; // element is SNAME - SArray* pTableHash; // element is SNAME + SArray* pTableMeta; // element is STablesReq + SArray* pTableHash; // element is STablesReq SArray* pUdf; // element is udf name SArray* pIndex; // element is index name SArray* pUser; // element is SUserAuthInfo diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index e15708e357973d707a4312a6b21911c0f586654e..7bc2ede4fcdf2430db2e74aa74aaed46a65d124f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -76,6 +76,13 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n */ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type); +/** + * @brief Cleanup SSDataBlock for StreamScanInfo + * + * @param tinfo + */ +void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo); + /** * Update the table id list, add or remove. * @@ -96,7 +103,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo * @return */ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model); + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model); /** * diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f6f2ceb9579b3f5f9e5d3355b87d6b17aa0a66bd..479432eebed40d75bdd8e260bd5a44d7d64ded62 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -202,6 +202,7 @@ bool fmIsForbidStreamFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); bool fmIsLastRowFunc(int32_t funcId); +bool fmIsNotNullOutputFunc(int32_t funcId); bool fmIsSelectValueFunc(int32_t funcId); bool fmIsSystemInfoFunc(int32_t funcId); bool fmIsImplicitTsFunc(int32_t funcId); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9572157fbbf5174007fbe62183dc263c7afec72f..842e656b9ded148534e1cc488dff53071eb95ae1 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -17,6 +17,7 @@ #include "os.h" #include "query.h" #include "tdatablock.h" +#include "tdbInt.h" #include "tmsg.h" #include "tmsgcb.h" #include "tqueue.h" @@ -46,9 +47,10 @@ enum { }; enum { - TASK_EXEC_STATUS__IDLE = 1, - TASK_EXEC_STATUS__EXECUTING, - TASK_EXEC_STATUS__CLOSING, + TASK_SCHED_STATUS__INACTIVE = 1, + TASK_SCHED_STATUS__WAITING, + TASK_SCHED_STATUS__ACTIVE, + TASK_SCHED_STATUS__FAILED, }; enum { @@ -65,6 +67,31 @@ enum { TASK_OUTPUT_STATUS__BLOCKED, }; +enum { + TASK_TRIGGER_STATUS__INACTIVE = 1, + TASK_TRIGGER_STATUS__ACTIVE, +}; + +enum { + TASK_LEVEL__SOURCE = 1, + TASK_LEVEL__AGG, + TASK_LEVEL__SINK, +}; + +enum { + TASK_OUTPUT__FIXED_DISPATCH = 1, + TASK_OUTPUT__SHUFFLE_DISPATCH, + TASK_OUTPUT__TABLE, + TASK_OUTPUT__SMA, + TASK_OUTPUT__FETCH, +}; + +enum { + STREAM_QUEUE__SUCESS = 1, + STREAM_QUEUE__FAILED, + STREAM_QUEUE__PROCESSING, +}; + typedef struct { int8_t type; } SStreamQueueItem; @@ -103,12 +130,6 @@ typedef struct { SSDataBlock* pBlock; } SStreamTrigger; -enum { - STREAM_QUEUE__SUCESS = 1, - STREAM_QUEUE__FAILED, - STREAM_QUEUE__PROCESSING, -}; - typedef struct { STaosQueue* queue; STaosQall* qall; @@ -201,41 +222,6 @@ typedef struct { int8_t reserved; } STaskSinkFetch; -enum { - TASK_SOURCE__SCAN = 1, - TASK_SOURCE__PIPE, - TASK_SOURCE__MERGE, -}; - -enum { - TASK_EXEC__NONE = 1, - TASK_EXEC__PIPE, - TASK_EXEC__MERGE, -}; - -enum { - TASK_DISPATCH__NONE = 1, - TASK_DISPATCH__FIXED, - TASK_DISPATCH__SHUFFLE, -}; - -enum { - TASK_SINK__NONE = 1, - TASK_SINK__TABLE, - TASK_SINK__SMA, - TASK_SINK__FETCH, -}; - -enum { - TASK_INPUT_TYPE__SUMBIT_BLOCK = 1, - TASK_INPUT_TYPE__DATA_BLOCK, -}; - -enum { - TASK_TRIGGER_STATUS__IN_ACTIVE = 1, - TASK_TRIGGER_STATUS__ACTIVE, -}; - typedef struct { int32_t nodeId; int32_t childId; @@ -248,28 +234,25 @@ typedef struct { typedef struct SStreamTask { int64_t streamId; int32_t taskId; - int8_t isDataScan; - int8_t execType; - int8_t sinkType; - int8_t dispatchType; - int8_t isStreamDistributed; + int32_t totalLevel; + int8_t taskLevel; + int8_t outputType; int16_t dispatchMsgType; int8_t taskStatus; - int8_t execStatus; + int8_t schedStatus; // node info int32_t selfChildId; int32_t nodeId; SEpSet epSet; - // used for semi or single task, - // while final task should have processedVer for each child + // used for task source and sink, + // while task agg should have processedVer for each child int64_t recoverSnapVer; int64_t startVer; int64_t checkpointVer; int64_t processedVer; - // int32_t numOfVgroups; // children info SArray* childEpInfo; // SArray @@ -277,19 +260,13 @@ typedef struct SStreamTask { // exec STaskExec exec; - // TODO: unify sink and dispatch - - // local sink - union { - STaskSinkTb tbSink; - STaskSinkSma smaSink; - STaskSinkFetch fetchSink; - }; - - // remote dispatcher + // output union { STaskDispatcherFixedEp fixedEpDispatcher; STaskDispatcherShuffle shuffleDispatcher; + STaskSinkTb tbSink; + STaskSinkSma smaSink; + STaskSinkFetch fetchSink; }; int8_t inputStatus; @@ -303,9 +280,6 @@ typedef struct SStreamTask { int64_t triggerParam; void* timer; - // application storage - // void* ahandle; - // msg handle SMsgCb* pMsgCb; } SStreamTask; @@ -342,7 +316,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem } if (pItem->type != STREAM_INPUT__GET_RES && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { - atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } #if 0 @@ -357,18 +331,15 @@ static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { } static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { - if (pTask->sinkType == TASK_SINK__TABLE) { - ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks); taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); - } else if (pTask->sinkType == TASK_SINK__SMA) { - ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE); + } else if (pTask->outputType == TASK_OUTPUT__SMA) { pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks); taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); } else { - ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); taosWriteQitem(pTask->outputQueue->queue, pBlock); } return 0; @@ -475,7 +446,6 @@ typedef struct { int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); -int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId); int32_t streamSetupTrigger(SStreamTask* pTask); int32_t streamProcessRunReq(SStreamTask* pTask); @@ -487,13 +457,29 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); -typedef struct SStreamMeta SStreamMeta; +int32_t streamTryExec(SStreamTask* pTask); +int32_t streamSchedExec(SStreamTask* pTask); + +typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask); + +typedef struct SStreamMeta { + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pStateDb; + SHashObj* pTasks; + void* ahandle; + TXN txn; + FTaskExpand* expandFunc; +} SStreamMeta; -SStreamMeta* streamMetaOpen(); +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc); void streamMetaClose(SStreamMeta* streamMeta); -int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, char* msg, int32_t msgLen); +int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); +SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); int32_t streamMetaBegin(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 50f995917708204bde3b442832a07f02d7157ab3..467f0a9ff02b14842306d4b5b2e6a201babeff40 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -41,12 +41,13 @@ typedef struct { typedef struct SRpcHandleInfo { // rpc info - void *handle; // rpc handle returned to app - int64_t refId; // refid, used by server - int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp); - int32_t persistHandle; // persist handle or not + void *handle; // rpc handle returned to app + int64_t refId; // refid, used by server + int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) + int8_t persistHandle; // persist handle or not + int8_t hasEpSet; + STraceId traceId; - int8_t hasEpSet; // app info void *ahandle; // app handle set by client @@ -69,8 +70,9 @@ typedef struct SRpcMsg { SRpcHandleInfo info; } SRpcMsg; -typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; @@ -84,12 +86,15 @@ typedef struct SRpcInit { // the following is for client app ecurity only char *user; // user name - // call back to process incoming msg, code shall be ignored by server app + // call back to process incoming msg RpcCfp cfp; - // user defined retry func + // retry not not for particular msg RpcRfp rfp; + // set up timeout for particular msg + RpcTfp tfp; + void *parent; } SRpcInit; diff --git a/include/os/osDef.h b/include/os/osDef.h index 14f38eb7ffbe15dbf07ca5e57f6963f1624a7352..be8689398dacc44a262de53c597f8ec103b49923 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -57,7 +57,7 @@ extern "C" { #if defined(WINDOWS) char *stpcpy (char *dest, const char *src); - char *stpncpy (char *dest, const char *src, size_t n); + char *stpncpy (char *dest, const char *src, int n); // specific #ifndef __COMPAR_FN_T @@ -77,7 +77,7 @@ extern "C" { char * strsep(char **stringp, const char *delim); char * getpass(const char *prefix); - char * strndup(const char *s, size_t n); + char * strndup(const char *s, int n); // for send function in tsocket.c #define MSG_NOSIGNAL 0 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 97a664d776790a0d3391f99ae213ac6ad74cf90a..27fb057b44371032bda52de7be3f6f35da2ea917 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -46,6 +46,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) #define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) +#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) //common & util #define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index 8db6be88609baff3d95b63410ad694ca833a4418..82dd5a0e8bcdc440b0b44f920f5ede02de7f07f2 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -25,9 +25,9 @@ extern "C" { // reference counting typedef void (*_ref_fn_t)(const void *pObj); -#define T_REF_DECLARE() \ - struct { \ - int32_t val; \ +#define T_REF_DECLARE() \ + struct { \ + volatile int32_t val; \ } _ref; #define T_REF_REGISTER_FUNC(s, e) \ diff --git a/packaging/release.bat b/packaging/release.bat index c1cf7875a505852ce3f8c0b78029fedf481aed8f..d58e19cece44dddb8466edeb301a5ed8d636e4c2 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -2,61 +2,78 @@ set internal_dir=%~dp0\..\..\ set community_dir=%~dp0\.. -cd %community_dir% -git checkout -- . -cd %community_dir%\packaging +set package_dir=%cd% :: %1 name %2 version if !%1==! GOTO USAGE if !%2==! GOTO USAGE -if %1 == taos GOTO TAOS -if %1 == power GOTO POWER -if %1 == tq GOTO TQ -if %1 == pro GOTO PRO -if %1 == kh GOTO KH -if %1 == jh GOTO JH -GOTO USAGE - -:TAOS -goto RELEASE - -:POWER -call sed_power.bat %community_dir% -goto RELEASE - -:TQ -call sed_tq.bat %community_dir% -goto RELEASE - -:PRO -call sed_pro.bat %community_dir% -goto RELEASE - -:KH -call sed_kh.bat %community_dir% -goto RELEASE - -:JH -call sed_jh.bat %community_dir% -goto RELEASE - -:RELEASE -echo release windows-client-64 for %1, version: %2 -if not exist %internal_dir%\debug\ver-%2-64bit-%1 ( - md %internal_dir%\debug\ver-%2-64bit-%1 + +if "%1" == "cluster" ( + set work_dir=%internal_dir% + set packagServerName_x64=TDengine-enterprise-server-%2-beta-Windows-x64 + set packagServerName_x86=TDengine-enterprise-server-%2-beta-Windows-x86 + set packagClientName_x64=TDengine-enterprise-client-%2-beta-Windows-x64 + set packagClientName_x86=TDengine-enterprise-client-%2-beta-Windows-x86 +) else ( + set work_dir=%community_dir% + set packagServerName_x64=TDengine-server-%2-Windows-x64 + set packagServerName_x86=TDengine-server-%2-Windows-x86 + set packagClientName_x64=TDengine-client-%2-Windows-x64 + set packagClientName_x86=TDengine-client-%2-Windows-x86 +) + +echo release windows-client for %1, version: %2 +if not exist %work_dir%\debug ( + md %work_dir%\debug +) +if not exist %work_dir%\debug\ver-%2-x64 ( + md %work_dir%\debug\ver-%2-x64 ) else ( - rd /S /Q %internal_dir%\debug\ver-%2-64bit-%1 - md %internal_dir%\debug\ver-%2-64bit-%1 + rd /S /Q %work_dir%\debug\ver-%2-x64 + md %work_dir%\debug\ver-%2-x64 ) -cd %internal_dir%\debug\ver-%2-64bit-%1 -call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 -cmake ../../ -G "NMake Makefiles" -DVERNUMBER=%2 -DCPUTYPE=x64 -set CL=/MP4 -nmake install +if not exist %work_dir%\debug\ver-%2-x86 ( + md %work_dir%\debug\ver-%2-x86 +) else ( + rd /S /Q %work_dir%\debug\ver-%2-x86 + md %work_dir%\debug\ver-%2-x86 +) +cd %work_dir%\debug\ver-%2-x64 +call vcvarsall.bat x64 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DVERNUMBER=%2 -DCPUTYPE=x64 +cmake --build . +rd /s /Q C:\TDengine +cmake --install . +if not %errorlevel% == 0 ( call :RUNFAILED build x64 failed & exit /b 1) +cd %package_dir% +iscc /DMyAppInstallName="%packagServerName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x64% failed & exit /b 1) +iscc /DMyAppInstallName="%packagClientName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x64% failed & exit /b 1) + +cd %work_dir%\debug\ver-%2-x86 +call vcvarsall.bat x86 +cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DVERNUMBER=%2 -DCPUTYPE=x86 +cmake --build . +rd /s /Q C:\TDengine +cmake --install . +if not %errorlevel% == 0 ( call :RUNFAILED build x86 failed & exit /b 1) +cd %package_dir% +iscc /DMyAppInstallName="%packagServerName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x86% failed & exit /b 1) +iscc /DMyAppInstallName="%packagClientName_x86%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release +if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x86% failed & exit /b 1) + goto EXIT0 :USAGE -echo Usage: release.bat $productName $version +echo Usage: release.bat $verMode $version goto EXIT0 -:EXIT0 \ No newline at end of file +:EXIT0 +exit /b + +:RUNFAILED +echo %* +cd %package_dir% +goto :eof \ No newline at end of file diff --git a/packaging/release.sh b/packaging/release.sh index 3426c2856d03573cb6ed6ec6078c38dfa6371cb0..2452ee18134027aa31ef0963294dc162c6ec82ed 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -26,7 +26,7 @@ soMode=dynamic # [static | dynamic] dbName=taos # [taos | ...] allocator=glibc # [glibc | jemalloc] verNumber="" -verNumberComp="2.0.0.0" +verNumberComp="3.0.0.0" httpdBuild=false while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do @@ -216,7 +216,7 @@ else fi # check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]]; then +if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]]; then if [ "$verMode" != "cluster" ]; then # community-version compile cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} diff --git a/packaging/tools/favicon.ico b/packaging/tools/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..20b8026d1d2efd6d77b013a1b94cee7aaed11772 Binary files /dev/null and b/packaging/tools/favicon.ico differ diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 2c3b70f5ed6ed39fd2066638db498415ff7f7dcc..beb0718987a013ea85f570f8dd864b113901b1c6 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -30,7 +30,6 @@ configDir="/etc/taos" installDir="/usr/local/taos" adapterName="taosadapter" benchmarkName="taosBenchmark" -tmqName="tmq_sim" dumpName="taosdump" demoName="taosdemo" diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a3f4c3584261582215de088bb9aef0fc24ee0fb7..6103ce170ce646665ed0d97ef85dad46b2473ed6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -60,7 +60,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/${serverName} strip ${build_dir}/bin/${clientName} # lite version doesn't include taosadapter, which will lead to no restful interface - bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark ${build_dir}/bin/tmq_sim" + bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark " taostools_bin_files="" else @@ -78,7 +78,6 @@ else taostools_bin_files=" ${build_dir}/bin/taosdump \ ${build_dir}/bin/taosBenchmark \ - ${build_dir}/bin/tmq_sim \ ${build_dir}/bin/TDinsight.sh \ $tdinsight_caches" diff --git a/packaging/tools/taos.bat b/packaging/tools/taos.bat new file mode 100644 index 0000000000000000000000000000000000000000..c3fe625e16f295d78fc3ffcb3c3af2e36d9d8839 --- /dev/null +++ b/packaging/tools/taos.bat @@ -0,0 +1,6 @@ +@echo off +cd C:\TDengine +if not "%1" == "" ( + %1 --help + @cmd /k +) \ No newline at end of file diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss new file mode 100644 index 0000000000000000000000000000000000000000..73102018159b3d37dca8531cadc2a388a2843651 --- /dev/null +++ b/packaging/tools/tdengine.iss @@ -0,0 +1,81 @@ +#define MyAppName "TDengine" +#define MyAppPublisher "taosdata" +#define MyAppURL "http://www.taosdata.com/" +#define MyAppBeforeInstallTxt "windows_before_install.txt" +#define MyAppIco "favicon.ico" +#define MyAppInstallDir "C:\TDengine" +#define MyAppOutputDir "./" +#define MyAppSourceDir "C:\TDengine" +;#define MyAppAllFile "\*" +#define MyAppCfgName "\cfg\*" +#define MyAppDriverName "\driver\*" +#define MyAppConnectorName "\connector\*" +#define MyAppExamplesName "\examples\*" +#define MyAppIncludeName "\include\*" +#define MyAppExeName "\*.exe" +#define MyAppTaosExeName "\taos.bat" +#define MyAppTaosdemoExeName "\taosBenchmark.exe" +#define MyAppDLLName "\driver\taos.dll" +;#define MyAppVersion "3.0" +;#define MyAppInstallName "TDengine" + +[Setup] +VersionInfoVersion={#MyAppVersion} +AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB} +AppName={#MyAppName} +AppVersion={#MyAppVersion} +;AppVerName={#MyAppName} {#MyAppVersion} +AppPublisher={#MyAppPublisher} +AppPublisherURL={#MyAppURL} +AppSupportURL={#MyAppURL} +AppUpdatesURL={#MyAppURL} +DefaultDirName={#MyAppInstallDir} +DefaultGroupName={#MyAppName} +DisableProgramGroupPage=yes +InfoBeforeFile={#MyAppBeforeInstallTxt} +OutputDir={#MyAppOutputDir} +OutputBaseFilename={#MyAppInstallName} +SetupIconFile={#MyAppIco} +Compression=lzma +SolidCompression=yes +DisableDirPage=yes +Uninstallable=yes + +[Languages] +Name: "chinesesimp"; MessagesFile: "compiler:Default.isl" +;Name: "english"; MessagesFile: "compiler:Languages\English.isl" + +[Files] +;Source: {#MyAppSourceDir}{#MyAppAllFile}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: taos.bat; DestDir: "{app}\include"; Flags: igNoreversion; +;Source: taosdemo.png; DestDir: "{app}\include"; Flags: igNoreversion; +;Source: taosShell.png; DestDir: "{app}\include"; Flags: igNoreversion; +Source: favicon.ico; DestDir: "{app}\include"; Flags: igNoreversion; +Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: igNoreversion; +Source: {#MyAppSourceDir}{#MyAppCfgName}; DestDir: "{app}\cfg"; Flags: igNoreversion recursesubdirs createallsubdirs onlyifdoesntexist uninsneveruninstall +Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igNoreversion recursesubdirs createallsubdirs +;Source: {#MyAppSourceDir}{#MyAppConnectorName}; DestDir: "{app}\connector"; Flags: igNoreversion recursesubdirs createallsubdirs +;Source: {#MyAppSourceDir}{#MyAppExamplesName}; DestDir: "{app}\examples"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}{#MyAppExeName}; DestDir: "{app}"; Excludes: {#MyAppExcludeSource} ; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs + +[UninstallDelete] +Name: {app}\driver; Type: filesandordirs +Name: {app}\connector; Type: filesandordirs +Name: {app}\examples; Type: filesandordirs +Name: {app}\include; Type: filesandordirs + +[Tasks] +Name: "desktopicon";Description: "{cm:CreateDesktopIcon}"; GroupDescription:"{cm:AdditionalIcons}"; Flags: checkablealone + +[Icons] +Name:"{group}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; IconFilename: "{app}\include\{#MyAppIco}" +Name:"{group}\Open TDengine Directory"; Filename: "{app}\" +Name:"{group}\Taosdemo"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taosdemo.exe" ; IconFilename: "{app}\include\{#MyAppIco}" +Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" ; IconFilename: "{app}\include\{#MyAppIco}" +Name:"{commondesktop}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; Tasks: desktopicon; WorkingDir: "{app}" ; IconFilename: "{app}\include\{#MyAppIco}" + + +[Messages] +ConfirmUninstall=Do you really want to uninstall TDengine from your computer?%n%nPress [Y] to completely delete %1 and all its components;%nPress [N] to keep the software on your computer. diff --git a/packaging/tools/windows_before_install.txt b/packaging/tools/windows_before_install.txt new file mode 100644 index 0000000000000000000000000000000000000000..b793a3e8014493b374c3b7bd79d7f535ca14f555 --- /dev/null +++ b/packaging/tools/windows_before_install.txt @@ -0,0 +1,3 @@ +TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. + +TDengine will be installed under C:\TDengine, users can modify configuration file C:\TDengine\cfg\taos.cfg, set the log file path or other parameters. \ No newline at end of file diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index d88a587f6ae49954c5e8213f2e3f4d765b32ea34..812351e2080cc031e996e415bc1c3690db11abd2 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -60,7 +60,7 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { } static void deregisterRequest(SRequestObj *pRequest) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable + const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable assert(pRequest != NULL); STscObj *pTscObj = pRequest->pTscObj; @@ -77,13 +77,13 @@ static void deregisterRequest(SRequestObj *pRequest) { if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { - atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } - + if (duration >= SLOW_QUERY_INTERVAL) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); } - + releaseTscObj(pTscObj->id); } @@ -109,6 +109,14 @@ static bool clientRpcRfp(int32_t code, tmsg_t msgType) { } } +// start timer for particular msgType +static bool clientRpcTfp(int32_t code, tmsg_t msgType) { + if (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_CREATE_TABLE) { + return true; + } + return false; +} + // TODO refactor void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; @@ -118,6 +126,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.rfp = clientRpcRfp; + rpcInit.tfp = clientRpcTfp; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -375,7 +384,7 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); taosConvInit(); - + rpcInit(); SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 06bd3f388741edd21e69dc5f1128937f1739feea..7031a1ebcab1adcb146f4b4b6c2e63ea8fcbf8f2 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -286,6 +286,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (pInst == NULL || NULL == *pInst) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); + taosMemoryFree(pMsg->pData); tFreeClientHbBatchRsp(&pRsp); return -1; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 480fd99c81b7e30309745c97f5df3e19e8054cf9..38f562172b61e0696cf4a91377dcb9591e72c84b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1308,8 +1308,8 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, code: %s, gtid: %s", pMsg->info.handle, - TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), tbuf); + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle, + TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); @@ -1937,7 +1937,7 @@ _OVER: return code; } -int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, +int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) { SName name; @@ -1972,20 +1972,33 @@ int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, i return -1; } - taosArrayPush(pList, &name); + char dbFName[TSDB_DB_FNAME_LEN]; + sprintf(dbFName, "%d.%.*s", acctId, dbLen, dbName); + + STablesReq* pDb = taosHashGet(pHash, dbFName, strlen(dbFName)); + if (pDb) { + taosArrayPush(pDb->pTables, &name); + } else { + STablesReq db; + db.pTables = taosArrayInit(20, sizeof(SName)); + strcpy(db.dbFName, dbFName); + taosArrayPush(db.pTables, &name); + taosHashPut(pHash, dbFName, strlen(dbFName), &db, sizeof(db)); + } return TSDB_CODE_SUCCESS; } int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq) { - *pReq = taosArrayInit(10, sizeof(SName)); - if (NULL == *pReq) { + SHashObj* pHash = taosHashInit(3, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } bool inEscape = false; int32_t code = 0; + void *pIter = NULL; int32_t vIdx = 0; int32_t vPos[2]; @@ -2000,7 +2013,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2050,7 +2063,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2082,14 +2095,31 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, goto _return; } + int32_t dbNum = taosHashGetSize(pHash); + *pReq = taosArrayInit(dbNum, sizeof(STablesReq)); + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayPush(*pReq, pDb); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); + return TSDB_CODE_SUCCESS; _return: terrno = TSDB_CODE_TSC_INVALID_OPERATION; - taosArrayDestroy(*pReq); - *pReq = NULL; + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayDestroy(pDb->pTables); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); return terrno; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 4372a3439017bb373a1a4e371b40e552cb472025..e2619a78b1a3449358020ff3802fc9d63b14c93a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -189,20 +189,6 @@ typedef struct { tsem_t rspSem; } SMqPollCbParam; -#if 0 -typedef struct { - tmq_t* tmq; - int8_t async; - int8_t automatic; - int8_t freeOffsets; - tmq_commit_cb* userCb; - tsem_t rspSem; - int32_t rspErr; - SArray* offsets; - void* userParam; -} SMqCommitCbParam; -#endif - typedef struct { tmq_t* tmq; int8_t automatic; @@ -385,29 +371,6 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } -#if 0 -int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { - SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; - pParam->rspErr = code; - if (pParam->async) { - if (pParam->automatic && pParam->tmq->commitCb) { - pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, pParam->tmq->commitCbUserParam); - } else if (!pParam->automatic && pParam->userCb) { - pParam->userCb(pParam->tmq, pParam->rspErr, pParam->userParam); - } - - if (pParam->freeOffsets) { - taosArrayDestroy(pParam->offsets); - } - - taosMemoryFree(pParam); - } else { - tsem_post(&pParam->rspSem); - } - return 0; -} -#endif - int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; @@ -660,123 +623,6 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ return 0; } -#if 0 -int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, - tmq_commit_cb* userCb, void* userParam) { - SMqCMCommitOffsetReq req; - SArray* pOffsets = NULL; - void* buf = NULL; - SMqCommitCbParam* pParam = NULL; - SMsgSendInfo* sendInfo = NULL; - int8_t freeOffsets; - int32_t code = -1; - - if (msg == NULL) { - freeOffsets = 1; - pOffsets = taosArrayInit(0, sizeof(SMqOffset)); - for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - SMqOffset offset; - tstrncpy(offset.topicName, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); - tstrncpy(offset.cgroup, tmq->groupId, TSDB_CGROUP_LEN); - offset.vgId = pVg->vgId; - offset.offset = pVg->currentOffset; - taosArrayPush(pOffsets, &offset); - } - } - } else { - freeOffsets = 0; - pOffsets = (SArray*)&msg->container; - } - - req.num = (int32_t)pOffsets->size; - req.offsets = pOffsets->pData; - - SEncoder encoder; - - tEncoderInit(&encoder, NULL, 0); - code = tEncodeSMqCMCommitOffsetReq(&encoder, &req); - if (code < 0) { - goto END; - } - int32_t tlen = encoder.pos; - buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - tEncoderClear(&encoder); - goto END; - } - tEncoderClear(&encoder); - - tEncoderInit(&encoder, buf, tlen); - tEncodeSMqCMCommitOffsetReq(&encoder, &req); - tEncoderClear(&encoder); - - pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); - if (pParam == NULL) { - goto END; - } - pParam->tmq = tmq; - pParam->automatic = automatic; - pParam->async = async; - pParam->offsets = pOffsets; - pParam->freeOffsets = freeOffsets; - pParam->userCb = userCb; - pParam->userParam = userParam; - if (!async) tsem_init(&pParam->rspSem, 0, 0); - - sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (sendInfo == NULL) goto END; - sendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; - - sendInfo->requestId = generateRequestId(); - sendInfo->requestObjRefId = 0; - sendInfo->param = pParam; - sendInfo->fp = tmqCommitCb; - sendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET; - - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - if (!async) { - tsem_wait(&pParam->rspSem); - code = pParam->rspErr; - tsem_destroy(&pParam->rspSem); - taosMemoryFree(pParam); - } else { - code = 0; - } - - // avoid double free if msg is sent - buf = NULL; - -END: - if (buf) taosMemoryFree(buf); - /*if (pParam) taosMemoryFree(pParam);*/ - /*if (sendInfo) taosMemoryFree(sendInfo);*/ - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam); - } else { - userCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, userParam); - } - } - - if (!async && freeOffsets) { - taosArrayDestroy(pOffsets); - } - return code; -} -#endif - void tmqAssignAskEpTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); @@ -1161,7 +1007,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pParam); if (code != 0) { tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); - if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (pMsg->pData) taosMemoryFreeClear(pMsg->pData); if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1839,13 +1685,22 @@ int32_t tmq_consumer_close(tmq_t* tmq) { return rsp; } + int32_t retryCnt = 0; tmq_list_t* lst = tmq_list_new(); - rsp = tmq_subscribe(tmq, lst); + while (1) { + rsp = tmq_subscribe(tmq, lst); + if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) { + break; + } else { + retryCnt++; + taosMsleep(500); + } + } + tmq_list_destroy(lst); - if (rsp != 0) { - return rsp; - } + /*return rsp;*/ + return 0; } // TODO: free resources return 0; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 29d7b557c807b40720cac1ab3c81d26dca4470ee..11faaaad97cbf00d042fa736775df713be6f93f8 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -16,15 +16,15 @@ #include "systable.h" #include "taos.h" #include "tdef.h" -#include "types.h" #include "tgrant.h" +#include "types.h" #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) static const SSysDbTableSchema dnodesSchema[] = { - {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, {.name = "support_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT}, @@ -66,7 +66,7 @@ static const SSysDbTableSchema bnodesSchema[] = { }; static const SSysDbTableSchema clusterSchema[] = { - {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; @@ -97,7 +97,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_retention_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "wal_seg_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, }; static const SSysDbTableSchema userFuncSchema[] = { @@ -243,8 +243,8 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, -// {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, -// {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, + // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, + // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema)}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, @@ -308,9 +308,9 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, + {.name = "kill_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "query_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -354,11 +354,19 @@ static const SSysTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}}; void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { - *pInfosTableMeta = infosMeta; - *size = tListLen(infosMeta); + if (pInfosTableMeta) { + *pInfosTableMeta = infosMeta; + } + if (size) { + *size = tListLen(infosMeta); + } } void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size) { - *pPerfsTableMeta = perfsMeta; - *size = tListLen(perfsMeta); + if (pPerfsTableMeta) { + *pPerfsTableMeta = perfsMeta; + } + if (size) { + *size = tListLen(perfsMeta); + } } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 45bbe969238e3e31927522cf1ce43780e88e177b..d21b639e3d547df9ffbadcb8ac65b24bbcc6c54f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1272,8 +1272,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { colDataAssign(pDst, pSrc, src->info.rows, &src->info); } - dst->info.rows = src->info.rows; - dst->info.capacity = src->info.rows; + dst->info = src->info; return 0; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8dd8a29c865163e86e1004667c77a722b2201fd7..a0f02d96f91913bbe127b992052ecd03a2008b61 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -89,7 +89,7 @@ bool tsSmlDataFormat = // query int32_t tsQueryPolicy = 1; -int32_t tsQuerySmaOptimize = 1; +int32_t tsQuerySmaOptimize = 0; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -389,7 +389,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, 0) != 0) return -1; - tsNumOfVnodeStreamThreads = tsNumOfCores; + tsNumOfVnodeStreamThreads = tsNumOfCores / 4; tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4); if (cfgAddInt32(pCfg, "numOfVnodeStreamThreads", tsNumOfVnodeStreamThreads, 4, 1024, 0) != 0) return -1; @@ -401,7 +401,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeSyncThreads = tsNumOfCores; + // tsNumOfVnodeSyncThreads = tsNumOfCores; + tsNumOfVnodeSyncThreads = 32; tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 06c64dcea6aed0cf0b563e858df110a6b4be8266..c4da9b5c3d5a282c7b80136fdf8421edb53edf9e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -604,11 +604,11 @@ typedef struct { int64_t createTime; int64_t updateTime; int32_t version; + int32_t totalLevel; int64_t smaId; // 0 for unused // info int64_t uid; int8_t status; - int8_t isDistributed; // config int8_t igExpired; int8_t trigger; @@ -647,7 +647,6 @@ typedef struct { typedef struct { int64_t uid; int64_t streamId; - int8_t isDistributed; int8_t status; int8_t stage; } SStreamRecoverObj; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0120e1146d99dab97946445b9b28f425e56a4090..66f81a3dba4945ce06490f432ebe509f483cbd29 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1691,13 +1691,17 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1); + size_t numOfTables = 0; + getInfosDbMeta(NULL, &numOfTables); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1); + numOfTables = 0; + getPerfDbMeta(NULL, &numOfTables); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); numOfRows += 1; pShow->sysDbRsp = true; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index abac0573da0eea96a047b5163d3aedc09858d153..08ce161409037316478b083750187fd10a7f8b9e 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -23,11 +23,11 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->totalLevel) < 0) return -1; if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->isDistributed) < 0) return -1; if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; @@ -69,11 +69,11 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->totalLevel) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->isDistributed) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 2beeb10335b02c6d08a75c0f2c498c825e56de54..654f46ec85682a21e8ef0009c3fcb654180c93b1 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -84,6 +84,9 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -111,6 +114,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } else { rsp.rspCode = 0; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.info.rspLen; rsp.msg = reqMsg.info.rsp; @@ -136,6 +140,8 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { *(int32_t*)((char*)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t*)((char*)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t*)((char*)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t*)((char*)pRsp + offset) = htonl(p->rspCode); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 9d7fa537bb3ed9fffde4dc5b49e37e7e0e4afc84..a24b7ef4597ceb1d5aba35efe907e9a7e12cb0a8 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -98,13 +98,11 @@ END: } int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { - pTask->dispatchType = TASK_DISPATCH__NONE; - // sink if (pStream->smaId != 0) { - pTask->sinkType = TASK_SINK__SMA; + pTask->outputType = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; } else { - pTask->sinkType = TASK_SINK__TABLE; + pTask->outputType = TASK_OUTPUT__TABLE; pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); @@ -113,8 +111,6 @@ int32_t mndAddSinkToTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask } int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStreamTask* pTask) { - pTask->sinkType = TASK_SINK__NONE; - bool isShuffle = false; if (pStream->fixedSinkVgId == 0) { @@ -122,7 +118,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream ASSERT(pDb); if (pDb->cfg.numOfVgroups > 1) { isShuffle = true; - pTask->dispatchType = TASK_DISPATCH__SHUFFLE; + pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { ASSERT(0); @@ -152,7 +148,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream } } } else { - pTask->dispatchType = TASK_DISPATCH__FIXED; + pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only @@ -178,7 +174,6 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, co terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE || pTask->sinkType != TASK_SINK__NONE); return 0; } @@ -249,26 +244,20 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->nodeId = pVgroup->vgId; pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); - // source - pTask->isDataScan = 0; - - // exec - pTask->execType = TASK_EXEC__NONE; + // type + pTask->taskLevel = TASK_LEVEL__SINK; // sink if (pStream->smaId != 0) { - pTask->sinkType = TASK_SINK__SMA; + pTask->outputType = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; } else { - pTask->sinkType = TASK_SINK__TABLE; + pTask->outputType = TASK_OUTPUT__TABLE; pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); ASSERT(pTask->tbSink.pSchemaWrapper); } - - // dispatch - pTask->dispatchType = TASK_DISPATCH__NONE; } return 0; } @@ -295,25 +284,19 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { #endif pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); - // source - pTask->isDataScan = 0; - - // exec - pTask->execType = TASK_EXEC__NONE; + pTask->taskLevel = TASK_LEVEL__SINK; // sink if (pStream->smaId != 0) { - pTask->sinkType = TASK_SINK__SMA; + pTask->outputType = TASK_OUTPUT__SMA; pTask->smaSink.smaId = pStream->smaId; } else { - pTask->sinkType = TASK_SINK__TABLE; + pTask->outputType = TASK_OUTPUT__TABLE; pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); } - // dispatch - pTask->dispatchType = TASK_DISPATCH__NONE; return 0; } @@ -324,10 +307,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); - ASSERT(totLevel <= 2); - pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); - pStream->isDistributed = totLevel == 2; + int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); + ASSERT(planTotLevel <= 2); + pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; @@ -337,7 +319,8 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { bool multiTarget = pDbObj->cfg.numOfVgroups > 1; - if (totLevel == 2 || externalTargetDB || multiTarget) { + if (planTotLevel == 2 || externalTargetDB || multiTarget) { + /*if (true) {*/ SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); // add extra sink @@ -354,8 +337,9 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } } + pStream->totalLevel = planTotLevel + hasExtraSink; - if (totLevel > 1) { + if (planTotLevel > 1) { SStreamTask* pInnerTask; // inner level { @@ -376,8 +360,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); - // source - pInnerTask->isDataScan = 0; + pInnerTask->taskLevel = TASK_LEVEL__AGG; // trigger pInnerTask->triggerParam = pStream->triggerParam; @@ -388,16 +371,6 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - // exec - pInnerTask->execType = TASK_EXEC__PIPE; - -#if 0 - SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); - ASSERT(pDbObj != NULL); - sdbRelease(pSdb, pSourceDb); - pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; -#endif - if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { @@ -452,19 +425,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { mndAddTaskToTaskSet(taskSourceLevel, pTask); // source - pTask->isDataScan = 1; + pTask->taskLevel = TASK_LEVEL__SOURCE; // add fixed vg dispatch - pTask->sinkType = TASK_SINK__NONE; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; - pTask->dispatchType = TASK_DISPATCH__FIXED; + pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; pTask->fixedEpDispatcher.taskId = pInnerTask->taskId; pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; - // exec - pTask->execType = TASK_EXEC__PIPE; if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); @@ -487,7 +457,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } } - if (totLevel == 1) { + if (planTotLevel == 1) { SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); @@ -515,7 +485,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { mndAddTaskToTaskSet(taskOneLevel, pTask); // source - pTask->isDataScan = 1; + pTask->taskLevel = TASK_LEVEL__SOURCE; // trigger pTask->triggerParam = pStream->triggerParam; @@ -527,8 +497,6 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { mndAddSinkToTask(pMnode, pStream, pTask); } - // exec - pTask->execType = TASK_EXEC__PIPE; if (mndAssignTaskToVg(pMnode, pTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index e82e5e087042f936336d416cfa1cf1443ace42c6..006d9e749cfd273f4f112e84e435d495f29125b1 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -489,7 +489,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); ASSERT(smaObj.uid != 0); char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; - snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name); + snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb",pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN); smaObj.stbUid = pStb->uid; @@ -603,6 +603,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + mDebug("mndSma: create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", + pCreate->name, smaObj.uid, smaObj.stbUid, smaObj.dstTbUid, smaObj.dstTbName, smaObj.dstVgId); + code = 0; _OVER: @@ -795,11 +798,12 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p pStb = mndAcquireStb(pMnode, pSma->stb); if (pStb == NULL) goto _OVER; - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); + mndTransSetSerial(pTrans); char streamName[TSDB_TABLE_FNAME_LEN] = {0}; mndGetStreamNameFromSmaName(streamName, pSma->name); @@ -812,12 +816,14 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); sdbRelease(pMnode->pSdb, pStream); + ASSERT(0); goto _OVER; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); + ASSERT(0); goto _OVER; } } @@ -850,6 +856,7 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p if (pIter == NULL) break; if (pSma->stbUid == pStb->uid) { + mndTransSetSerial(pTrans); pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVgroup == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b8501db9fb2a372e9a8ad7a919a904c9296fdea9..ba9bb2982f88c82ffff59648c6f988a9a163204c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -36,7 +36,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); -static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); +/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -55,7 +55,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); - mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq); + /*mndSetMsgHandle(pMnode, TDMT_MND_RECOVER_STREAM, mndProcessRecoverStreamReq);*/ mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); @@ -323,8 +323,7 @@ FAIL: } int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { - ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1); - if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) { + if (pTask->taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); } SEncoder encoder; @@ -541,6 +540,7 @@ static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { return 0; } +#if 0 int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { if (pStream->isDistributed) { int32_t lv = taosArrayGetSize(pStream->tasks); @@ -548,7 +548,7 @@ int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea SArray *pTasks = taosArrayGetP(pStream->tasks, i); int32_t sz = taosArrayGetSize(pTasks); SStreamTask *pTask = taosArrayGetP(pTasks, 0); - if (!pTask->isDataScan && pTask->execType != TASK_EXEC__NONE) { + if (pTask->taskLevel == TASK_LEVEL__AGG) { ASSERT(sz == 1); if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { return -1; @@ -564,8 +564,8 @@ int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (!pTask->isDataScan) break; - ASSERT(pTask->execType != TASK_EXEC__NONE); + if (pTask->taskLevel != TASK_LEVEL__SOURCE) break; + ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { return -1; } @@ -574,6 +574,7 @@ int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea } return 0; } +#endif int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); @@ -756,6 +757,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +#if 0 static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; @@ -818,6 +820,7 @@ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } +#endif int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 2561031bac637079d9426959833deca4b97da33b..cda4663285ec560c1ea635a56e242d62efa45d41 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -110,9 +110,6 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) { pTask->pMsgCb = &pNode->msgCb; - ASSERT(pTask->execType != TASK_EXEC__NONE); - - ASSERT(pTask->isDataScan == 0); pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); ASSERT(pTask->exec.executor); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1aee08027c50402d2329bfcec75e4811fb558d3d..944d7759b28c41a901d7ed0f666eca2ffef30b6a 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -115,24 +115,29 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - void *taskInfo; // qTaskInfo_t - int64_t refId; - tmr_h tmrId; - int32_t maxDelay; int8_t level; int8_t triggerStat; + int32_t maxDelay; + tmr_h tmrId; }; struct SRSmaInfo { STSchema *pTSchema; int64_t suid; + int64_t refId; // refId of SRSmaStat int8_t delFlag; T_REF_DECLARE() SRSmaInfoItem items[TSDB_RETENTION_L2]; + void *taskInfo[TSDB_RETENTION_L2]; // qTaskInfo_t + void *iTaskInfo[TSDB_RETENTION_L2]; // immutable }; -#define RSMA_INFO_HEAD_LEN 24 -#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) -#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) + +#define RSMA_INFO_HEAD_LEN 32 +#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) +#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) +#define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) +#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) +#define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -168,8 +173,8 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); -int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId); +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId); int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); @@ -223,12 +228,11 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc); +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc); void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c093b2cd5d1eb6142dc33d0d2f4fa4693debdce7..a1dba41c941f9baf8a28304b80eb12530e929e9d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -117,10 +117,9 @@ typedef struct { struct STQ { SVnode* pVnode; char* path; - SHashObj* pushMgr; // consumerId -> STqHandle* - SHashObj* handles; // subKey -> STqHandle - SHashObj* pStreamTasks; // taksId -> SStreamTask - SHashObj* pAlterInfo; // topic -> SAlterCheckInfo + SHashObj* pushMgr; // consumerId -> STqHandle* + SHashObj* handles; // subKey -> STqHandle + SHashObj* pAlterInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; @@ -129,9 +128,7 @@ struct STQ { TTB* pAlterInfoStore; - TDB* pStreamStore; - TTB* pTaskDb; - TTB* pTaskState; + SStreamMeta* pStreamMeta; }; typedef struct { @@ -188,6 +185,9 @@ static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ve pOffsetVal->version = ver; } +// tqStream +int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index db71c21615f86737f156db3b3f599bcb0bee81e9..797a60baf6bae8d6e2cfeb08e057574a4cb902ef 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -599,9 +599,11 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); if (metaGetTableEntryByUid(&mr, smaId) < 0) { + tDecoderClear(&mr.coder); metaWarn("vgId:%d, no entry for tbId:%" PRIi64 ", smaId:%" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; } + tDecoderClear(&mr.coder); pTSma = pSW->tSma + smaIdx; memcpy(pTSma, mr.me.smaEntry.tsma, sizeof(STSma)); if (deepCopy) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index fdf2da5558d36af12dda1a3143ab27550bc3d633..7427f79509d863bf9b16c7d9cbc10b4f5f8fecc6 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -381,6 +381,8 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); return -1; + } else if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + terrno = TSDB_CODE_SUCCESS; } metaReaderClear(&mr); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 3c88b6f5cb6ee9f7f2c4a071236134a55f877e68..373cfdfb47a47ceaba6be6ca0e5e9531f61e2441 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -308,12 +308,12 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); // step 1: set rsma stat @@ -337,18 +337,26 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } // step 3: swap rsmaInfoHash and iRsmaInfoHash - ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + ASSERT(RSMA_INFO_HASH(pRSmaStat)); + ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); RSMA_INFO_HASH(pRSmaStat) = taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + // step 4: others pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; @@ -383,26 +391,52 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { * @return int32_t */ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + if (!pEnv) { return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); // step 1: merge rsmaInfoHash and iRsmaInfoHash - taosWLockLatch(SMA_ENV_LOCK(pSmaEnv)); - + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); +#if 0 if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { - // TODO: optimization - just switch the hash pointer if rsmaInfoHash is empty - } - + // just switch the hash pointer if rsmaInfoHash is empty + if (taosHashGetSize(RSMA_IMU_INFO_HASH(pRSmaStat)) > 0) { + SHashObj *infoHash = RSMA_INFO_HASH(pRSmaStat); + RSMA_INFO_HASH(pRSmaStat) = RSMA_IMU_INFO_HASH(pRSmaStat); + RSMA_IMU_INFO_HASH(pRSmaStat) = infoHash; + } + } else { +#endif +#if 1 void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); while (pIter) { tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + int32_t refVal = T_REF_VAL_GET(pRSmaInfo); + if (refVal == 0) { + tdFreeRSmaInfo(pSma, pRSmaInfo, true); + smaDebug( + "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " + "table:%" PRIi64, + SMA_VID(pSma), *pSuid); + } else { + smaDebug( + "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " + "table:%" PRIi64, + SMA_VID(pSma), refVal, *pSuid); + } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + continue; + } taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); @@ -416,11 +450,14 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); } +#endif + // } taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; - taosWUnLockLatch(SMA_ENV_LOCK(pSmaEnv)); + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); // step 2: cleanup outdated qtaskinfo files tdCleanupQTaskInfoFiles(pSma, pRSmaStat); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4b831225bc73725a417f07ce029a5d541e406208..ccb6ad3a72c2358772b652d04709d5eebc2fd2fb 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -17,7 +17,7 @@ typedef struct SSmaStat SSmaStat; -#define SMA_MGMT_REF_NUM 10240 +#define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; @@ -171,7 +171,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { if (!pRSmaInfo) return 0; - + int ref = T_REF_INC(pRSmaInfo); smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); return 0; @@ -183,9 +183,6 @@ int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int ref = T_REF_DEC(pRSmaInfo); smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); - if (ref == 0) { - tdRemoveRSmaInfoBySuid(pSma, pRSmaInfo->suid); - } return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index beebf2b0e27e1b6700a47ece0c9ddee633d4a5c0..fd2222c5e4e7aa2c38ffd9788446f4f025cab71f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -32,12 +32,14 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *ui static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, - STSchema *pTSchema, tb_uid_t suid, int8_t level); -static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid); -static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, - int8_t blkType); -static void tdRSmaFetchTrigger(void *param, void *tmrId); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, + int8_t level); +static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); +static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); + +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, + SRSmaStat *pStat, int8_t blkType); +static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); @@ -92,7 +94,7 @@ static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC - if (!taskHandle) return; + if (!taskHandle || !(*taskHandle)) return; qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); @@ -115,17 +117,26 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; - if (pItem->taskInfo) { - if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, - pItem->tmrId, i + 1); - taosTmrStopA(&pItem->tmrId); - } - tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); + + if (isDeepFree && pItem->tmrId) { + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, + pItem->tmrId, i + 1); + taosTmrStopA(&pItem->tmrId); + } + + if (isDeepFree && pInfo->taskInfo[i]) { + tdFreeQTaskInfo(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); } + + if (pInfo->iTaskInfo[i]) { + tdFreeQTaskInfo(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); + } else { + smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", + SMA_VID(pSma), pInfo->suid, i + 1); + } } if (isDeepFree) { taosMemoryFreeClear(pInfo->pTSchema); @@ -155,7 +166,12 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_FAILED; } - pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid); + if (!taosArrayGetSize(tbUids)) { + smaDebug("vgId:%d, no need to update tbUidList for suid:%" PRIi64 " since Empty tbUids", SMA_VID(pSma), *suid); + return TSDB_CODE_SUCCESS; + } + + pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, *suid); if (!pRSmaInfo) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); @@ -163,26 +179,21 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_FAILED; } - if (pRSmaInfo->items[0].taskInfo) { - if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); - } - } - - if (pRSmaInfo->items[1].taskInfo) { - if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pRSmaInfo->taskInfo[i]) { + if ((qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true) < 0)) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, + terrstr()); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); + } } } + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -267,13 +278,12 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat .initTqReader = 1, }; - SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - pItem->refId = RSMA_REF_ID(pStat); - pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); - if (!pItem->taskInfo) { + pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; return TSDB_CODE_FAILED; } + SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = @@ -342,6 +352,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con } pRSmaInfo->pTSchema = pTSchema; pRSmaInfo->suid = suid; + pRSmaInfo->refId = RSMA_REF_ID(pStat); T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { @@ -411,7 +422,7 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pReq->suid); + SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pReq->suid); if (!pRSmaInfo) { smaWarn("vgId:%d, drop rsma for stable %s %" PRIi64 " failed no rsma in hash", TD_VID(pVnode), pReq->name, @@ -423,8 +434,10 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { RSMA_INFO_SET_DEL(pRSmaInfo); tdUnRefRSmaInfo(pSma, pRSmaInfo); - // save to file + tdReleaseRSmaInfo(pSma, pRSmaInfo); + // save to file + // TODO smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -567,8 +580,32 @@ static void tdDestroySDataBlockArray(SArray *pArray) { taosArrayDestroy(pArray); } -static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat, - int8_t blkType) { +/** + * @brief retention of rsma1/rsma2 + * + * @param pSma + * @param now + * @return int32_t + */ +int32_t smaDoRetention(SSma *pSma, int64_t now) { + int32_t code = TSDB_CODE_SUCCESS; + if (VND_IS_RSMA(pSma->pVnode)) { + return code; + } + + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pSma->pRSmaTsdb[i]) { + code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); + if (code) goto _end; + } + } + +_end: + return code; +} + +static int32_t tdRSmaFetchAndSubmitResult(qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, + SRSmaStat *pStat, int8_t blkType) { SArray *pResult = NULL; SSma *pSma = pStat->pSma; @@ -576,7 +613,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSDataBlock *output = NULL; uint64_t ts; - int32_t code = qExecTask(pItem->taskInfo, &output, &ts); + int32_t code = qExecTask(taskInfo, &output, &ts); if (code < 0) { smaError("vgId:%d, qExecTask for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr(code)); @@ -637,29 +674,32 @@ _err: return TSDB_CODE_FAILED; } -static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, - STSchema *pTSchema, tb_uid_t suid, int8_t level) { - if (!pItem || !pItem->taskInfo) { +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, tb_uid_t suid, + int8_t level) { + int32_t idx = level - 1; + if (!pInfo || !RSMA_INFO_QTASK(pInfo, idx)) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; } - if (!pTSchema) { + if (!pInfo->pTSchema) { smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_FAILED; } smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, - pItem->taskInfo, suid); + RSMA_INFO_QTASK(pInfo, idx), suid); - if (qSetMultiStreamInput(pItem->taskInfo, pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT + if (qSetMultiStreamInput(RSMA_INFO_QTASK(pInfo, idx), pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT); + tdRSmaFetchAndSubmitResult(RSMA_INFO_QTASK(pInfo, idx), pItem, pInfo->pTSchema, suid, pStat, + STREAM_INPUT__DATA_SUBMIT); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); if (smaMgmt.tmrHandle) { @@ -678,7 +718,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType * @param suid * @return SRSmaInfo* */ -static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { +static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; @@ -692,94 +732,86 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } + taosRLockLatch(SMA_ENV_LOCK(pEnv)); pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } + tdRefRSmaInfo(pSma, pRSmaInfo); + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return pRSmaInfo; } - if (RSMA_COMMIT_STAT(pStat) == 0) { + if (RSMA_COMMIT_STAT(pStat) == 0) { // return NULL if not in committing stat + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat SRSmaInfo *pCowRSmaInfo = NULL; // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - if (!taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t))) { // 2-phase lock + if (!(pCowRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)))) { // 2-phase lock void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (iRSmaInfo) { SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; - if (pIRSmaInfo) { - if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + if (pIRSmaInfo && !RSMA_INFO_IS_DEL(pIRSmaInfo)) { + if (tdCloneRSmaInfo(pSma, &pCowRSmaInfo, pIRSmaInfo) < 0) { + // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); return NULL; } smaDebug("vgId:%d, clone rsma info succeed for suid:%" PRIu64, SMA_VID(pSma), suid); if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); return NULL; } } } + } else { + pCowRSmaInfo = *(SRSmaInfo **)pCowRSmaInfo; + ASSERT(!pCowRSmaInfo); + } + + if(pCowRSmaInfo) { + tdRefRSmaInfo(pSma, pCowRSmaInfo); } // unlock taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); return pCowRSmaInfo; } -/** - * @brief During the drop procedure, only need to delete the object in rsmaInfoHash. - * - * @param pSma - * @param suid - * @return SRSmaInfo* - */ -void tdRemoveRSmaInfoBySuid(SSma *pSma, int64_t suid) { - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SRSmaInfo *pRSmaInfo = NULL; - - if (!pEnv) { - return; - } - - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - if (!pStat || !RSMA_INFO_HASH(pStat)) { - return; - } - - pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (pRSmaInfo) { - if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tdFreeRSmaInfo(pSma, pRSmaInfo, true); - } - taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - smaDebug("vgId:%d, remove from infoHash for table:%" PRIu64 " succeed", SMA_VID(pSma), suid); +static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { + if (pInfo) { + tdUnRefRSmaInfo(pSma, pInfo); } } static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid); + SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } - if (!pRSmaInfo->items[0].taskInfo) { + if (!RSMA_INFO_QTASK(pRSmaInfo, 0)) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaDebug("vgId:%d, execute rsma, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - tdRefRSmaInfo(pSma, pRSmaInfo); - - tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1); - tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2); - - tdUnRefRSmaInfo(pSma, pRSmaInfo); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L1); + tdExecuteRSmaImpl(pSma, pMsg, inputType, pRSmaInfo, suid, TSDB_RETENTION_L2); } + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -848,6 +880,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { terrstr()); goto _err; } + tDecoderClear(&mr.coder); ASSERT(mr.me.type == TSDB_SUPER_TABLE); ASSERT(mr.me.uid == suid); if (TABLE_IS_ROLLUP(mr.me.flags)) { @@ -989,26 +1022,28 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * SRSmaInfo *pRSmaInfo = NULL; void *qTaskInfo = NULL; - pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid); + pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pItem->suid); if (!pRSmaInfo) { smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid); return TSDB_CODE_SUCCESS; } if (pItem->type == TSDB_RETENTION_L1) { - qTaskInfo = pRSmaInfo->items[0].taskInfo; + qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 0); } else if (pItem->type == TSDB_RETENTION_L2) { - qTaskInfo = pRSmaInfo->items[1].taskInfo; + qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 1); } else { ASSERT(0); } if (!qTaskInfo) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid); return TSDB_CODE_SUCCESS; } if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { + tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, pItem->type, terrstr()); return TSDB_CODE_FAILED; @@ -1016,6 +1051,7 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type); + tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } @@ -1194,8 +1230,14 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + infoHash = taosHashIterate(pInfoHash, infoHash); + continue; + } + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; + qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); if (!taskInfo) { smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; @@ -1289,11 +1331,17 @@ _err: static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; SSma *pSma = NULL; - SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + return; + } + + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); if (!pStat) { smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, - pItem->refId); + pRSmaInfo->refId); return; } @@ -1304,10 +1352,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: { - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", - SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); + SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaInfo->refId); if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); @@ -1318,11 +1366,6 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { break; } - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - goto _end; - } - int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (fetchTriggerStat) { @@ -1331,15 +1374,14 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { pItem->level, pRSmaInfo->suid); // sync procedure => async process - tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetMultiStreamInput(pItem->taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); - tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); + qTaskInfo_t taskInfo = pRSmaInfo->taskInfo[pItem->level - 1]; + qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); + tdRSmaFetchAndSubmitResult(taskInfo, pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, + STREAM_INPUT__DATA_BLOCK); + tdCleanupStreamInputDataBlock(taskInfo); - tdUnRefRSmaInfo(pSma, pRSmaInfo); - // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId); } break; case TASK_TRIGGER_STAT_PAUSED: { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused", @@ -1360,22 +1402,5 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } _end: - tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); } - -int32_t smaDoRetention(SSma *pSma, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; - if (VND_IS_RSMA(pSma->pVnode)) { - return code; - } - - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (pSma->pRSmaTsdb[i]) { - code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); - if (code) goto _end; - } - } - -_end: - return code; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 6d71f3a250f4b02ae9d1acc2c5b11e5cf8721c56..7f69acc8895df7c921126b368a0cbb092d58daec 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -116,8 +116,10 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { } // create stable to save tsma result in dstVgId + SName stbFullName = {0}; + tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateStbReq pReq = {0}; - pReq.name = pCfg->dstTbName; + pReq.name = (char*)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; @@ -125,6 +127,12 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { return -1; } + + smaDebug("vgId:%d, success to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 + " dstTb:%s dstVg:%d", + SMA_VID(pSma), pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, pCfg->dstVgId); + } else { + ASSERT(0); } return 0; diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 9dce166afa8fd43e349b446f1f1f04a05c761b7a..d9f38ffd090fffcb6cf110265f1d2929714da669 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -287,22 +287,22 @@ int32_t tdRemoveTFile(STFile *pTFile) { } // smaXXXUtil ================ -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); if (!pResult) { - smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + smaWarn("rsma acquire ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); } else { - smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + smaDebug("rsma acquire ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); } return pResult; } -int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { if (taosReleaseRef(rsetId, refId) < 0) { - smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + smaWarn("rsma release ref for rsetId:%" PRIi64 " refId:%d failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + smaDebug("rsma release ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); return TSDB_CODE_SUCCESS; } @@ -313,7 +313,7 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t char *pOutput = NULL; int32_t len = 0; - if (qSerializeTaskStatus(srcTaskInfo, &pOutput, &len) < 0) { + if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, terrstr()); goto _err; @@ -337,41 +337,34 @@ static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t goto _err; } - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); + smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); taosMemoryFreeClear(pOutput); return TSDB_CODE_SUCCESS; _err: taosMemoryFreeClear(pOutput); tdFreeQTaskInfo(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); return TSDB_CODE_FAILED; } /** * @brief pTSchema is shared - * - * @param pSma - * @param pDest - * @param pSrc - * @return int32_t + * + * @param pSma + * @param pDest + * @param pSrc + * @return int32_t */ -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo **pDest, SRSmaInfo *pSrc) { SVnode *pVnode = pSma->pVnode; SRSmaParam *param = NULL; if (!pSrc) { + *pDest = NULL; return TSDB_CODE_SUCCESS; } - if (!pDest) { - pDest = taosMemoryCalloc(1, sizeof(SRSmaInfo)); - if (!pDest) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - - memcpy(pDest, pSrc, sizeof(SRSmaInfo)); - SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid); @@ -384,21 +377,22 @@ int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { ASSERT(mr.me.uid == pSrc->suid); if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; - for (int i = 0; i < TSDB_RETENTION_L2; ++i) { - SRSmaInfoItem *pItem = &pSrc->items[i]; - if (pItem->taskInfo) { - tdCloneQTaskInfo(pSma, pDest->items[i].taskInfo, pItem->taskInfo, param, pSrc->suid, i); + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (tdCloneQTaskInfo(pSma, pSrc->iTaskInfo[i], pSrc->taskInfo[i], param, pSrc->suid, i) < 0) { + goto _err; } } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); } metaReaderClear(&mr); + + *pDest = pSrc; // pointer copy + return TSDB_CODE_SUCCESS; _err: + *pDest = NULL; metaReaderClear(&mr); - tdFreeRSmaInfo(pSma, pDest, false); + smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, terrstr()); return TSDB_CODE_FAILED; -} - -// ... \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b6293bd6cf3fb494acc7ed86f1c110f575960ae7..c1c680fc56df0ff76f5c1c9c13897bfd233cf34c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -62,8 +62,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->handles = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); - pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); pTq->pAlterInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); @@ -76,6 +74,11 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { ASSERT(0); } + pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask); + if (pTq->pStreamMeta == NULL) { + ASSERT(0); + } + return pTq; } @@ -83,18 +86,11 @@ void tqClose(STQ* pTq) { if (pTq) { tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->handles); - void* pIter = NULL; - while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); - if (pIter == NULL) break; - SStreamTask* pTask = *(SStreamTask**)pIter; - tFreeSStreamTask(pTask); - } - taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); taosHashCleanup(pTq->pAlterInfo); taosMemoryFree(pTq->path); tqMetaClose(pTq); + streamMetaClose(pTq->pStreamMeta); taosMemoryFree(pTq); } } @@ -216,9 +212,11 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { if (offset.val.type == TMQ_OFFSET__LOG) { STqHandle* pHandle = taosHashGet(pTq->handles, offset.subKey, strlen(offset.subKey)); - if (walRefVer(pHandle->pRef, offset.val.version) < 0) { - ASSERT(0); - return -1; + if (pHandle) { + if (walRefVer(pHandle->pRef, offset.val.version) < 0) { + ASSERT(0); + return -1; + } } } @@ -515,7 +513,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { // todo lock STqHandle* pHandle = taosHashGet(pTq->handles, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { - ASSERT(req.oldConsumerId == -1); + if (req.oldConsumerId != -1) { + tqError("vgId:%d, build new consumer handle %s for consumer %d, but old consumerId is %ld", req.vgId, req.subKey, + req.newConsumerId, req.oldConsumerId); + } ASSERT(req.newConsumerId != -1); STqHandle tqHandle = {0}; pHandle = &tqHandle; @@ -599,12 +600,12 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { int32_t code = 0; - ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1); - if (pTask->isDataScan == 0 && pTask->sinkType == TASK_SINK__NONE) { + + if (pTask->taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); } - pTask->execStatus = TASK_EXEC_STATUS__IDLE; + pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -619,32 +620,30 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->pMsgCb = &pTq->pVnode->msgCb; - // exec - if (pTask->execType != TASK_EXEC__NONE) { - // expand runners - if (pTask->isDataScan) { - SReadHandle handle = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTqReader = 1, - }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - } else { - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); - } + // expand executor + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + SReadHandle handle = { + .meta = pTq->pVnode->pMeta, + .vnode = pTq->pVnode, + .initTqReader = 1, + }; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); + ASSERT(pTask->exec.executor); + } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + SReadHandle mgHandle = { + .vnode = NULL, + .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), + }; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); ASSERT(pTask->exec.executor); } // sink /*pTask->ahandle = pTq->pVnode;*/ - if (pTask->sinkType == TASK_SINK__SMA) { + if (pTask->outputType == TASK_OUTPUT__SMA) { pTask->smaSink.vnode = pTq->pVnode; pTask->smaSink.smaSink = smaHandleRes; - } else if (pTask->sinkType == TASK_SINK__TABLE) { + } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; pTask->tbSink.tbSinkFunc = tqTableSink; @@ -669,6 +668,9 @@ FAIL: } int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { + // + return streamMetaAddSerializedTask(pTq->pStreamMeta, msg, msgLen); +#if 0 SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return -1; @@ -692,6 +694,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { FAIL: if (pTask) taosMemoryFree(pTask); return -1; +#endif } int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { @@ -707,10 +710,10 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { } while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; - if (!pTask->isDataScan) continue; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; qDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); @@ -720,7 +723,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - if (streamLaunchByWrite(pTask, TD_VID(pTq->pVnode)) < 0) { + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } @@ -741,9 +744,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { // SStreamTaskRunReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRunReq(*ppTask); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRunReq(pTask); return 0; } else { return -1; @@ -751,21 +754,23 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { + ASSERT(0); char* msgStr = pMsg->pCont; char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); SStreamDispatchReq req; SDecoder decoder; - tDecoderInit(&decoder, msgBody, msgLen); + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.taskId; + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessDispatchReq(*ppTask, &req, &rsp, exec); + streamProcessDispatchReq(pTask, &req, &rsp, exec); return 0; } else { return -1; @@ -775,9 +780,9 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRecoverReq(*ppTask, pReq, pMsg); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRecoverReq(pTask, pReq, pMsg); return 0; } else { return -1; @@ -787,9 +792,9 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessDispatchRsp(*ppTask, pRsp); + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessDispatchRsp(pTask, pRsp); return 0; } else { return -1; @@ -799,9 +804,10 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->rspTaskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - streamProcessRecoverRsp(*ppTask, pRsp); + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { + streamProcessRecoverRsp(pTask, pRsp); return 0; } else { return -1; @@ -811,18 +817,7 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - taosHashRemove(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); - } - // todo - // clear queue - // push drop req into queue - // launch exec to free memory - // remove from hash - return 0; + return streamMetaRemoveTask(pTq->pStreamMeta, pReq->taskId); } int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { @@ -833,18 +828,18 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { SDecoder decoder; tDecoderInit(&decoder, msgBody, msgLen); tDecodeStreamRetrieveReq(&decoder, &req); - int32_t taskId = req.dstTaskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.dstTaskId; + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessRetrieveReq(*ppTask, &req, &rsp); + streamProcessRetrieveReq(pTask, &req, &rsp); + return 0; } else { return -1; } - return 0; } int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -867,16 +862,18 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { goto FAIL; } - int32_t taskId = req.taskId; - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - if (ppTask) { + int32_t taskId = req.taskId; + + SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); + if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0, }; - streamProcessDispatchReq(*ppTask, &req, &rsp, false); + streamProcessDispatchReq(pTask, &req, &rsp, false); return; } + FAIL: if (pMsg->info.handle == NULL) return; SRpcMsg rsp = { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 0debeaef90028a3c8fb902c0e870c3902d4cafbe..ae3fef9b4b7ebf02654e93e09b5bf6c52f6e2354 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -136,31 +136,6 @@ int32_t tqSendExecReq(STQ* pTq, STqHandle* pHandle) { return 0; } -int32_t tqEnqueueAll(STQ* pTq, SSubmitReq* pReq) { - void* pIter = NULL; - SStreamDataSubmit* pSubmit = streamDataSubmitNew(pReq); - if (pSubmit == NULL) { - return -1; - } - - while (1) { - pIter = taosHashIterate(pTq->handles, pIter); - if (pIter == NULL) break; - STqHandle* pHandle = (STqHandle*)pIter; - if (tqEnqueue(pHandle, pSubmit) < 0) { - continue; - } - int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus); - if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) { - tqSendExecReq(pTq, pHandle); - } - } - - streamDataSubmitRefDec(pSubmit); - - return 0; -} - int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver, SRpcHandleInfo handleInfo) { if (msgType != TDMT_VND_SUBMIT) return 0; void* pIter = NULL; @@ -240,7 +215,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) walApplyVer(pTq->pVnode->pWal, ver); if (msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0; + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; void* data = taosMemoryMalloc(msgLen); if (data == NULL) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index f20c7e7e55f587ae72ce6864ade4ef25c5da0464..6ce8dbe5d90e5d7448a9be03cb42eaf27c3f9bff 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -413,10 +413,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } } while (1) { - pIter = taosHashIterate(pTq->pStreamTasks, pIter); + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->isDataScan) { + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); ASSERT(code == 0); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index a9f07cbf24f062f367919f05949cf10a8fe8a50c..c40fb98d62b8dd6b1a76c1f6ec2fd870cbff3624 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -115,12 +115,19 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); int64_t version = TSDBROW_VERSION(&row); + tsdbTrace("vgId:%d, vnode snapshot tsdb read for %s, %" PRId64 "(%" PRId64 " , %" PRId64 ")", + TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path, version, pReader->sver, pReader->ever); + if (version < pReader->sver || version > pReader->ever) continue; code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); if (code) goto _err; } + if (pReader->nBlockData.nRow <= 0) { + continue; + } + // org data // compress data (todo) int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); @@ -808,7 +815,8 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { if (code) goto _err; _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), + pWriter->pTsdb->path); return code; _err: diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 650c98eb884ea1c50a4f91288176b3f8522b705f..460d0b6fbb52825e67a8fc4b1f09f7e9591bd90e 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -223,6 +223,8 @@ int vnodeCommit(SVnode *pVnode) { vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + pVnode->state.commitTerm = pVnode->state.applyTerm; + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d18ba88268c6f1a1700f5b13c8255c9b24c0c71b..707c4bc4717b08c4a392a4974fbda2877cd68368 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -273,6 +273,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -301,6 +304,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { break; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.contLen; rsp.rspCode = reqMsg.code; @@ -327,6 +331,8 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { *(int32_t *)((char *)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t *)((char *)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t *)((char *)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t *)((char *)pRsp + offset) = htonl(p->rspCode); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fbbe5bc69555c75c67813f589ed12a1932dba827..15cf183b2ad5cea0706388d3c3ee92e0911319b0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -351,7 +351,6 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { // TODO: remove the function void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO - blockDebugShowDataBlocks(data, __func__); tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } @@ -852,6 +851,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { submitBlkRsp.code = terrno; + pRsp->code = terrno; tDecoderClear(&decoder); taosArrayDestroy(createTbReq.ctb.tagName); goto _exit; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 52698d0be7c25ee9367d0bcecf3168c9f64c4909..50ca81e58bad7e9d629f66e0069f9c51e8b7fcae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -448,7 +448,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } - vTrace("vgId:%d, sync msg:%p is processed, type:%s code:0x%x", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType), code); + vTrace("vgId:%d, sync msg:%p is processed, type:%s code:0x%x", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType), + code); syncNodeRelease(pSyncNode); if (code != 0 && terrno == 0) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -629,8 +630,8 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { #ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; - vInfo("vgId:%d, stop write vnode snapshot, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64, pVnode->config.vgId, isApply, - pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex); + vInfo("vgId:%d, stop write vnode snapshot, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64, + pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex); int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot); vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code); @@ -707,8 +708,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } setPingTimerMS(pVnode->sync, 5000); - setElectTimerMS(pVnode->sync, 1300); - setHeartbeatTimerMS(pVnode->sync, 900); + setElectTimerMS(pVnode->sync, 4000); + setHeartbeatTimerMS(pVnode->sync, 700); return 0; } @@ -721,10 +722,12 @@ void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } bool vnodeIsLeader(SVnode *pVnode) { if (!syncIsReady(pVnode->sync)) { + vDebug("vgId:%d, vnode not ready", pVnode->config.vgId); return false; } if (!pVnode->restored) { + vDebug("vgId:%d, vnode not restored", pVnode->config.vgId); terrno = TSDB_CODE_APP_NOT_READY; return false; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 816309beb15ee171cbc72773c5bbc98ac51eb5ce..4ef23860cd2c53031ca2166505b77c73f970de4c 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -32,6 +32,7 @@ extern "C" { #define CTG_DEFAULT_RENT_SLOT_SIZE 10 #define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_DEFAULT_BATCH_NUM 64 +#define CTG_DEFAULT_FETCH_NUM 8 #define CTG_RENT_SLOT_SECOND 1.5 @@ -80,6 +81,8 @@ typedef enum { CTG_TASK_GET_UDF, CTG_TASK_GET_USER, CTG_TASK_GET_SVR_VER, + CTG_TASK_GET_TB_META_BATCH, + CTG_TASK_GET_TB_HASH_BATCH, } CTG_TASK_TYPE; typedef enum { @@ -110,6 +113,23 @@ typedef struct SCtgTbMetaCtx { int32_t flag; } SCtgTbMetaCtx; +typedef struct SCtgFetch { + int32_t dbIdx; + int32_t tbIdx; + int32_t fetchIdx; + int32_t resIdx; + int32_t flag; + SCtgTbCacheInfo tbInfo; + int32_t vgId; +} SCtgFetch; + +typedef struct SCtgTbMetasCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pResList; + SArray* pFetchs; +} SCtgTbMetasCtx; + typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; @@ -137,6 +157,14 @@ typedef struct SCtgTbHashCtx { SName* pName; } SCtgTbHashCtx; +typedef struct SCtgTbHashsCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pResList; + SArray* pFetchs; +} SCtgTbHashsCtx; + + typedef struct SCtgIndexCtx { char indexFName[TSDB_INDEX_FNAME_LEN]; } SCtgIndexCtx; @@ -211,6 +239,7 @@ typedef struct SCtgBatch { SRequestConnInfo conn; char dbFName[TSDB_DB_FNAME_LEN]; SArray* pTaskIds; + SArray* pMsgIdxs; } SCtgBatch; typedef struct SCtgJob { @@ -218,6 +247,7 @@ typedef struct SCtgJob { int32_t batchId; SHashObj* pBatchs; SArray* pTasks; + int32_t subTaskNum; int32_t taskDone; SMetaData jobRes; int32_t jobResCode; @@ -258,6 +288,7 @@ typedef struct SCtgTaskCallbackParam { SArray* taskId; int32_t reqType; int32_t batchId; + SArray* msgIdx; } SCtgTaskCallbackParam; @@ -276,6 +307,7 @@ typedef struct SCtgTask { int32_t taskId; SCtgJob* pJob; void* taskCtx; + SArray* msgCtxs; SCtgMsgCtx msgCtx; int32_t code; void* res; @@ -286,9 +318,14 @@ typedef struct SCtgTask { SHashObj* pBatchs; } SCtgTask; +typedef struct SCtgTaskReq { + SCtgTask* pTask; + int32_t msgIdx; +} SCtgTaskReq; + typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); -typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); +typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTaskReq*, int32_t, const SDataBuf *, int32_t); typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); typedef int32_t (*ctgCloneTaskResFp)(SCtgTask*, void**); typedef int32_t (*ctgCompTaskFp)(SCtgTask*, void*, bool*); @@ -487,6 +524,8 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_GET_TASK_MSGCTX(_task, _id) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx) + #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) @@ -586,6 +625,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -631,7 +671,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTaskReq* tReq); int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); @@ -639,9 +679,9 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTaskReq* tReq); int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask); @@ -664,6 +704,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); @@ -672,8 +713,11 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); +int32_t ctgGetTablesReqNum(SArray *pList); +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); void ctgFreeSTableIndex(void *info); void ctgClearSubTaskRes(SCtgSubRes *pRes); @@ -682,6 +726,7 @@ void ctgClearHandle(SCatalog* pCtg); void ctgFreeTbCacheImpl(SCtgTbCache *pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup); +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 0184ac3a607a3dbf6c0ddec6cd9c8ea2df3e3781..63d99cc58b0fe67df9fd2c7c61a1e90659b2f206 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -50,6 +50,31 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbMetasTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_META_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetasCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbMetasCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { char *dbFName = (char*)param; SCtgTask task = {0}; @@ -153,6 +178,32 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbHashsTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_HASH_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashsCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbHashsCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTask task = {0}; @@ -328,10 +379,12 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { } - int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == pDb) { + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb || NULL == pTb) { + taosHashCleanup(pDb); + taosHashCleanup(pTb); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -350,18 +403,26 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); } - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableMeta, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + + dbNum = taosArrayGetSize(pReq->pTableHash); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableHash, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { @@ -380,16 +441,6 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashCleanup(pDb); // REFRESH TABLE META - SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); @@ -429,9 +480,9 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param) { int32_t code = 0; - int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); + int32_t tbMetaNum = (int32_t)ctgGetTablesReqNum(pReq->pTableMeta); int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); - int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); + int32_t tbHashNum = (int32_t)ctgGetTablesReqNum(pReq->pTableHash); int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; int32_t dnodeNum = pReq->dNodeRequired ? 1 : 0; @@ -452,7 +503,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const } SCtgJob *pJob = *job; - + + pJob->subTaskNum = taskNum; pJob->queryId = pConn->requestId; pJob->userFp = fp; pJob->pCtg = pCtg; @@ -506,15 +558,27 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } +#if 0 for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META, name, NULL)); } +#else + if (tbMetaNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META_BATCH, pReq->pTableMeta, NULL)); + } +#endif +#if 0 for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH, name, NULL)); } +#else + if (tbHashNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH_BATCH, pReq->pTableHash, NULL)); + } +#endif for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); @@ -586,6 +650,15 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableMeta = pTask->res; + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpDbVgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbVgroup) { @@ -616,6 +689,14 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableHash = pTask->res; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { @@ -844,46 +925,51 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SName* pName = ctx->pName; + int32_t flag = ctx->flag; + int32_t* vgId = &ctx->vgId; - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); return TSDB_CODE_SUCCESS; } case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -892,58 +978,64 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } return TSDB_CODE_SUCCESS; } - ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (pTask->msgCtx.lastOut) { - TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut); - STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out; + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; TSWAP(pLastOut->tbMeta, pOut->tbMeta); } break; } case TDMT_VND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { break; } if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; - if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + exist = 1; + } } if (0 == exist) { - TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); - } else { - taosMemoryFreeClear(pOut->tbMeta); - - SET_META_TYPE_CTABLE(pOut->metaType); + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); } } break; @@ -954,17 +1046,20 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * break; } - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; ctgUpdateTbMetaToCache(pCtg, pOut, false); if (CTG_IS_META_BOTH(pOut->metaType)) { memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); - } else if (CTG_IS_META_CTABLE(pOut->metaType)) { - SName stbName = *ctx->pName; + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; strcpy(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; - stbCtx.flag = ctx->flag; + stbCtx.flag = flag; stbCtx.pName = &stbName; CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); @@ -977,6 +1072,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } +*/ TSWAP(pTask->res, pOut->tbMeta); @@ -986,13 +1082,199 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } - ctgHandleTaskEnd(pTask, code); + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); + + return TSDB_CODE_SUCCESS; + } + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + if (CTG_FLAG_IS_STB(flag)) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL != dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); + } + + return TSDB_CODE_SUCCESS; + } + + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; + TSWAP(pLastOut->tbMeta, pOut->tbMeta); + } + + break; + } + case TDMT_VND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (CTG_FLAG_IS_STB(flag)) { + break; + } + + if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); + + taosMemoryFreeClear(pOut->tbMeta); + + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); + } else if (CTG_IS_META_BOTH(pOut->metaType)) { + int32_t exist = 0; + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); + exist = 1; + } + } + + if (0 == exist) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); + } + } + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + ctgUpdateTbMetaToCache(pCtg, pOut, false); + + if (CTG_IS_META_BOTH(pOut->metaType)) { + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (NULL == pOut->tbMeta) { + ctgDebug("stb no longer exist, stbName:%s", stbName.tname); + CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; + } + + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } +*/ + + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + pRes->code = 0; + pRes->pRes = pOut->tbMeta; + pOut->tbMeta = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + + if (code) { + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + pRes->code = code; + pRes->pRes = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + } + + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + CTG_RET(code); } -int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + +int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1024,8 +1306,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1061,8 +1344,65 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); + + CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (code) { + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + int32_t num = taosArrayGetSize(pReq->pTables); + for (int32_t i = 0; i < num; ++i) { + SMetaRes *pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); + pRes->code = code; + pRes->pRes = NULL; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + } + + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1083,8 +1423,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1096,8 +1437,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1109,13 +1451,14 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbInfoRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { CTG_RET(TSDB_CODE_APP_ERROR); } -int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1127,8 +1470,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1140,8 +1484,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1153,8 +1498,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1166,8 +1512,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; bool pass = false; @@ -1210,8 +1557,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetSvrVerRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1224,45 +1572,45 @@ _return: CTG_RET(code); } -int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { +int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq *tReq, int32_t flag, SName* pName, int32_t* vgId) { + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; int32_t code = 0; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_SYS_DB(flag)) { + ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(pName)); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, tReq)); } - if (CTG_FLAG_IS_STB(ctx->flag)) { - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_STB(flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pName)); // if get from mnode failed, will not try vnode - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } SCtgDBCache *dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); } else { SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } _return: @@ -1284,8 +1632,51 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask)); + SCtgTbMetaCtx* pCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pCtx->flag, pCtx->pName, &pCtx->vgId)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); + baseResIdx += taosArrayGetSize(pReq->pTables); + } + + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); + } + return TSDB_CODE_SUCCESS; } @@ -1309,8 +1700,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1346,8 +1740,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1359,6 +1756,75 @@ _return: CTG_RET(code); } +int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbHashsCtx* pCtx = (SCtgTbHashsCtx*)pTask->taskCtx; + SCtgDBCache *dbCache = NULL; + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; + int32_t code = 0; + + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); + + if (NULL != dbCache) { + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; + + baseResIdx += taosArrayGetSize(pReq->pTables); + } else { + ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + + baseResIdx += taosArrayGetSize(pReq->pTables); + taosArraySetSize(pCtx->pResList, baseResIdx); + } + } + + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + + SBuildUseDBInput input = {0}; + strcpy(input.db, pReq->dbFName); + + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + + return code; +} + + int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1597,19 +2063,21 @@ int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) { SCtgAsyncFps gCtgAsyncFps[] = { - {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, - {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, - {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, - {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, - {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, - {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, - {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, - {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, - {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, - {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, - {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, - {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, - {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, + {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, + {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, + {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, + {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, + {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, + {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, + {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, + {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, + {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, + {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, + {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, + {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, + {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2e8e259151b3b19a06dfd271ad4c47c7b30673cb..930361419eadaeb7041b83c9cf5ae5c89e97e513 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -242,7 +242,6 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S goto _return; } - int32_t sz = 0; pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); @@ -282,7 +281,6 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, goto _return; } - int32_t sz = 0; char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); @@ -2152,6 +2150,254 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet return TSDB_CODE_SUCCESS; } +#if 0 +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, SArray** pResList) { + int32_t tbNum = taosArrayGetSize(ctx->pNames); + SName* fName = taosArrayGet(ctx->pNames, 0); + int32_t fIdx = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(ctx->pNames, i); + SCtgTbMetaCtx nctx = {0}; + nctx.flag = CTG_FLAG_UNKNOWN_STB; + nctx.pName = pName; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(nctx.flag); + } + + STableMeta *pTableMeta = NULL; + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, &nctx, &pTableMeta)); + SMetaRes res = {0}; + + if (pTableMeta) { + if (CTG_FLAG_MATCH_STB(nctx.flag, pTableMeta->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(nctx.flag)) || (CTG_FLAG_IS_SYS_DB(nctx.flag)))) { + res.pRes = pTableMeta; + } else { + taosMemoryFreeClear(pTableMeta); + } + } + + if (NULL == res.pRes) { + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(nctx.flag)) { + CTG_FLAG_SET_STB(nctx.flag, nctx.tbInfo.tbType); + } + + SCtgFetch fetch = {0}; + fetch.tbIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = nctx.flag; + + taosArrayPush(ctx->pFetchs, &fetch); + } + + taosArrayPush(ctx->pResList, &res); + } + + if (NULL == ctx->pFetchs) { + TSWAP(*pResList, ctx->pResList); + } + + return TSDB_CODE_SUCCESS; +} +#endif + +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList) { + int32_t tbNum = taosArrayGetSize(pList); + SName* pName = taosArrayGet(pList, 0); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t flag = CTG_FLAG_UNKNOWN_STB; + uint64_t lastSuid = 0; + STableMeta* lastTableMeta = NULL; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(flag); + strcpy(dbFName, pName->dbname); + } else { + tNameGetFullDbName(pName, dbFName); + } + + SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + for (int32_t i = 0; i < tbNum; ++i) { + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + } + + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(pList, i); + + pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + continue; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + continue; + } + + STableMeta* tbMeta = pCache->pMeta; + + SCtgTbMetaCtx nctx = {0}; + nctx.flag = flag; + nctx.tbInfo.inCache = true; + nctx.tbInfo.dbId = dbCache->dbId; + nctx.tbInfo.suid = tbMeta->suid; + nctx.tbInfo.tbType = tbMeta->tableType; + + SMetaRes res = {0}; + STableMeta* pTableMeta = NULL; + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pResList, &res); + + continue; + } + + // PROCESS FOR CHILD TABLE + + if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { + cloneTableMeta(lastTableMeta, &pTableMeta); + memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); + + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pResList, &res); + + continue; + } + + int32_t metaSize = sizeof(SCTableMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + pName->tname, nctx.tbInfo.tbType, dbFName); + + char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + STableMeta* stbMeta = pCache->pMeta; + if (stbMeta->suid != nctx.tbInfo.suid) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); + + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + metaSize = CTG_META_SIZE(stbMeta); + pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); + if (NULL == pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pResList, &res); + + lastSuid = pTableMeta->suid; + lastTableMeta = pTableMeta; + } + + ctgReleaseDBCache(pCtg, dbCache); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { int32_t code = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 4652c66ebc4df6dbe35b69afc12b8b067c113759..45f97865ce55f11508f6f67fc68dae864f476dd7 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -22,9 +22,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; - SArray* pTaskId = cbParam->taskId; SCatalog* pCtg = pJob->pCtg; - int32_t taskNum = taosArrayGetSize(pTaskId); + int32_t taskNum = taosArrayGetSize(cbParam->taskId); SDataBuf taskMsg = *pMsg; int32_t offset = 0; int32_t msgNum = (TSDB_CODE_SUCCESS == rspCode && pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; @@ -42,11 +41,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } for (int32_t i = 0; i < taskNum; ++i) { - int32_t* taskId = taosArrayGet(pTaskId, i); + int32_t* taskId = taosArrayGet(cbParam->taskId, i); + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); if (msgNum > 0) { rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.reqType); + rsp.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); + offset += sizeof(rsp.msgIdx); rsp.msgLen = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.msgLen); rsp.rspCode = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); @@ -57,7 +59,10 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.msgType = rsp.reqType; taskMsg.pData = rsp.msg; taskMsg.len = rsp.msgLen; + + ASSERT(rsp.msgIdx == *msgIdx); } else { + rsp.msgIdx = *msgIdx; rsp.reqType = -1; taskMsg.msgType = -1; taskMsg.pData = NULL; @@ -65,11 +70,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } pTask->pBatchs = pBatchs; + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = rsp.msgIdx; - ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, - TMSG_INFO(taskMsg.msgType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1)); - (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); + (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -338,7 +346,10 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { pTask->pBatchs = pBatchs; #endif - CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, cbParam->reqType, pMsg, rspCode)); #if CTG_BATCH_FETCH CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -356,7 +367,7 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int32_t msgType, +int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArray* pMsgIdx, int32_t msgType, SMsgSendInfo** pMsgSendInfo) { int32_t code = 0; SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -376,6 +387,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 param->refId = pJob->refId; param->taskId = pTaskId; param->batchId = batchId; + param->msgIdx = pMsgIdx; msgSendInfo->param = param; msgSendInfo->paramFreeFp = ctgFreeMsgSendParam; @@ -394,10 +406,10 @@ _return: } int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, SArray* pTaskId, int32_t batchId, - char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { + SArray* pMsgIdx, char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; SMsgSendInfo* pMsgSendInfo = NULL; - CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, msgType, &pMsgSendInfo)); + CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); @@ -428,47 +440,62 @@ _return: CTG_RET(code); } -int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTask* pTask, int32_t msgType, void* msg, +int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTaskReq* tReq, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SHashObj* pBatchs = pTask->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); - int32_t taskNum = taosArrayGetSize(pTask->pJob->pTasks); - SCtgBatch newBatch = {0}; - SBatchMsg req = {0}; - + SCtgBatch newBatch = {0}; + SBatchMsg req = {0}; + if (NULL == pBatch) { - newBatch.pMsgs = taosArrayInit(taskNum, sizeof(SBatchMsg)); - newBatch.pTaskIds = taosArrayInit(taskNum, sizeof(int32_t)); - if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { + newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); + newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); + newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); + if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } newBatch.conn = *pConn; + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; if (NULL == taosArrayPush(newBatch.pMsgs, &req)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + msg = NULL; if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(newBatch.pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, newBatch.dbFName); } newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; @@ -484,28 +511,43 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT return TSDB_CODE_SUCCESS; } + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; if (NULL == taosArrayPush(pBatch->pMsgs, &req)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + msg = NULL; if (NULL == taosArrayPush(pBatch->pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(pBatch->pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, pBatch->dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, @@ -537,6 +579,8 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { for (int32_t i = 0; i < num; ++i) { SBatchMsg* pReq = taosArrayGet(pBatch->pMsgs, i); + *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgIdx); + offset += sizeof(pReq->msgIdx); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgType); offset += sizeof(pReq->msgType); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgLen); @@ -564,8 +608,8 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg)); - code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->dbFName, *vgId, - pBatch->msgType, msg, pBatch->msgSize); + code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, + pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize); pBatch->pTaskIds = NULL; CTG_ERR_JRET(code); @@ -603,10 +647,14 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -614,7 +662,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -649,10 +697,13 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -660,7 +711,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -681,10 +732,11 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -700,10 +752,11 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, input->db)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -711,7 +764,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -751,10 +804,14 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)dbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -762,7 +819,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -802,10 +859,14 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)indexName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -813,7 +874,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -855,11 +916,14 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -867,7 +931,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -907,10 +971,14 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)funcName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -918,7 +986,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -958,10 +1026,14 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)user)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -969,7 +1041,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -990,7 +1062,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char* msg = NULL; SEpSet* pVnodeEpSet = NULL; @@ -1013,9 +1086,11 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1023,7 +1098,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1044,15 +1119,16 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, pTask); + return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); } int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgroupInfo, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); int32_t reqType = TDMT_VND_TABLE_META; @@ -1080,15 +1156,16 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1099,7 +1176,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); #endif } @@ -1142,14 +1219,17 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, &tReq, reqType, msg, msgLen)); #else SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1160,7 +1240,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); #endif } @@ -1201,9 +1281,13 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); + #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1211,7 +1295,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1246,10 +1330,13 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1257,7 +1344,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index bdf60b110b082ffef530b76dbc033ac2fa272506..1ca60c89cd96937ed4844fecfdbe78057c368d9f 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -26,6 +26,7 @@ void ctgFreeMsgSendParam(void* param) { SCtgTaskCallbackParam* pParam = (SCtgTaskCallbackParam*)param; taosArrayDestroy(pParam->taskId); + taosArrayDestroy(pParam->msgIdx); taosMemoryFree(param); } @@ -88,6 +89,10 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get user]"; case CTG_TASK_GET_SVR_VER: return "[get svr ver]"; + case CTG_TASK_GET_TB_META_BATCH: + return "[bget table meta]"; + case CTG_TASK_GET_TB_HASH_BATCH: + return "[bget table hash]"; default: return "unknown"; } @@ -433,6 +438,14 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { } } +void ctgFreeTbMetasMsgCtx(SCtgMsgCtx* pCtx) { + ctgFreeMsgCtx(pCtx); + if (pCtx->lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pCtx->lastOut); + pCtx->lastOut = NULL; + } +} + void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { if (NULL == pOutput) { return; @@ -460,6 +473,25 @@ void ctgResetTbMetaTask(SCtgTask* pTask) { taosMemoryFreeClear(pTask->res); } +void ctgFreeBatchMeta(void* meta) { + if (NULL == meta) { + return; + } + + SMetaRes* pRes = (SMetaRes*)meta; + taosMemoryFreeClear(pRes->pRes); +} + +void ctgFreeBatchHash(void* hash) { + if (NULL == hash) { + return; + } + + SMetaRes* pRes = (SMetaRes*)hash; + taosMemoryFreeClear(pRes->pRes); +} + + void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -500,6 +532,24 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchMeta(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } + case CTG_TASK_GET_TB_HASH_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchHash(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } default: qError("invalid task type %d", type); break; @@ -554,6 +604,16 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchMeta); + *pRes = NULL; + break; + } + case CTG_TASK_GET_TB_HASH_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchHash); + *pRes = NULL; + break; + } default: qError("invalid task type %d", type); break; @@ -583,12 +643,38 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SCtgTbMetasCtx* taskCtx = (SCtgTbMetasCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchMeta); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeTbMetasMsgCtx); + + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_HASH: { SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + SCtgTbHashsCtx* taskCtx = (SCtgTbHashsCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchHash); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_INDEX: { SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -679,6 +765,23 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ return TSDB_CODE_SUCCESS; } +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { + SCtgMsgCtx ctx = {0}; + + ctx.reqType = reqType; + ctx.out = out; + if (target) { + ctx.target = strdup(target); + if (NULL == ctx.target) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pCtxs, &ctx); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { switch (hashMethod) { @@ -780,6 +883,104 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SMetaRes res = {0}; + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; + + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + + int32_t tbNum = taosArrayGetSize(pNames); + + if (1 == vgNum) { + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + for (int32_t i = 0; i < tbNum; ++i) { + vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == vgInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *vgInfo = *(SVgroupInfo*)pIter; + + ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = vgInfo; + } else { + res.pRes = vgInfo; + taosArrayPush(pCtx->pResList, &res); + } + } + + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFullName, "%s.", dbFName); + int32_t offset = strlen(tbFullName); + SName* pName = NULL; + int32_t tbNameLen = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + pName = taosArrayGet(pNames, i); + + tbNameLen = offset + strlen(pName->tname); + strcpy(tbFullName + offset, pName->tname); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); + + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pNewVg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *pNewVg = *vgInfo; + + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = pNewVg; + } else { + res.pRes = pNewVg; + taosArrayPush(pCtx->pResList, &res); + } + } + + CTG_RET(code); +} + + int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; @@ -921,4 +1122,41 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, cha return TSDB_CODE_SUCCESS; } +int32_t ctgGetTablesReqNum(SArray *pList) { + if (NULL == pList) { + return 0; + } + + int32_t total = 0; + int32_t n = taosArrayGetSize(pList); + for (int32_t i = 0; i < n; ++i) { + STablesReq *pReq = taosArrayGet(pList, i); + total += taosArrayGetSize(pReq->pTables); + } + + return total; +} + +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag) { + if (NULL == (*pFetchs)) { + *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.dbIdx = dbIdx; + fetch.tbIdx = tbIdx; + fetch.fetchIdx = (*fetchIdx)++; + fetch.resIdx = resIdx; + fetch.flag = flag; + + taosArrayPush(*pFetchs, &fetch); + + return TSDB_CODE_SUCCESS; +} + +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); + return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); +} + diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1b8be3b53c560065b6f025cf4fbdb002088877af..e67210ccb06c0703981b3b830a6260ea13a57c80 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -739,6 +739,9 @@ typedef struct STimeSliceOperatorInfo { SInterval interval; int64_t current; SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + bool isPrevRowSet; + bool isNextRowSet; int32_t fillType; // fill type SColumn tsCol; // primary timestamp column SExprSupp scalarSup; // scalar calculation @@ -990,7 +993,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - const char* sql, EOPTR_EXEC_MODEL model); + char* sql, EOPTR_EXEC_MODEL model); int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 405aceb0bbe15c07ec0f56342a2eaef0c60fa3b4..34247d3b47768a92ee4c0fa658dac3bb08b6f1fd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -191,6 +191,7 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { pBlock->info.blockId = pNode->dataBlockId; pBlock->info.type = STREAM_INVALID; pBlock->info.calWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pBlock->info.watermark = INT64_MIN; for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); @@ -632,7 +633,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0) { + if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b2d4a5fc68df02fcffe2fd1547d123332cd787e0..897ddb59ec5e7aa2670d27090b19a62ef3ea83e6 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -93,6 +93,29 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) { + return; + } + SOperatorInfo* pOptrInfo = pTaskInfo->pRoot->pDownstream[0]; + + if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOptrInfo->info; + if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) { + SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i); + taosArrayDestroy(p->pDataBlock); + taosMemoryFreeClear(p); + } + } else { + ASSERT(0); + } + } else { + ASSERT(0); + } +} + int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; @@ -104,7 +127,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { @@ -193,6 +216,8 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S continue; } + tDecoderClear(&mr.coder); + // TODO handle ntb case if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { continue; @@ -241,7 +266,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } // todo refactor STableList - bool assignUid = false; + bool assignUid = false; size_t bufLen = (pScanInfo->pGroupTags != NULL) ? getTableTagsBufLen(pScanInfo->pGroupTags) : 0; char* keyBuf = NULL; if (bufLen > 0) { @@ -316,7 +341,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table } int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { assert(pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b7e099d0b1127a61c5cd6d7bfb04777200773b4b..844ee4cc6ea9a7f8226fa8ec5bf0482e19698c4a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1427,7 +1427,8 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u pAggInfo->groupId = groupId; } -static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_t* rowCellOffset) { +static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowCellOffset) { + bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset); if (!isRowEntryInitialized(pResInfo)) { @@ -1437,6 +1438,15 @@ static void doUpdateNumOfRows(SResultRow* pRow, int32_t numOfExprs, const int32_ if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } + + if (fmIsNotNullOutputFunc(pCtx[j].functionId)) { + returnNotNull = true; + } + } + // if all expr skips all blocks, e.g. all null inputs for max function, output one row in final result. + // except for first/last, which require not null output, output no rows + if (pRow->numOfRows == 0 && !returnNotNull) { + pRow->numOfRows = 1; } } @@ -1448,7 +1458,7 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); - doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowCellOffset); if (pRow->numOfRows == 0) { releaseBufPage(pBuf, page); return 0; @@ -3334,18 +3344,16 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { - if (pExpr) { - for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* pExprInfo = &pExpr[i]; - for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { - if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_COLUMN) { - taosMemoryFreeClear(pExprInfo->base.pParam[j].pCol); - } + for (int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; + for (int32_t j = 0; j < pExprInfo->base.numOfParams; ++j) { + if (pExprInfo->base.pParam[j].type == FUNC_PARAM_TYPE_COLUMN) { + taosMemoryFreeClear(pExprInfo->base.pParam[j].pCol); } - - taosMemoryFree(pExprInfo->base.pParam); - taosMemoryFree(pExprInfo->pExpr); } + + taosMemoryFree(pExprInfo->base.pParam); + taosMemoryFree(pExprInfo->pExpr); } } @@ -3487,9 +3495,8 @@ void cleanupExprSupp(SExprSupp* pSupp) { destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs); if (pSupp->pExprInfo != NULL) { destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); + taosMemoryFreeClear(pSupp->pExprInfo); } - - taosMemoryFreeClear(pSupp->pExprInfo); taosMemoryFree(pSupp->rowEntryInfoOffset); } @@ -4087,6 +4094,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pUser); if (ops[i] == NULL) { + taosMemoryFree(ops); return NULL; } else { ops[i]->resultDataBlockId = pChildNode->pOutputDataBlockDesc->dataBlockId; @@ -4193,7 +4201,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t children = 0; pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { - int32_t children = 1; + int32_t children = pHandle->numOfVgroups; pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); @@ -4505,7 +4513,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT } int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - const char* sql, EOPTR_EXEC_MODEL model) { + char* sql, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -4516,6 +4524,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; + sql = NULL; (*pTaskInfo)->pSubplan = pPlan; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); @@ -4528,7 +4537,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead return code; _complete: - taosMemoryFreeClear(*pTaskInfo); + taosMemoryFree(sql); + doDestroyTask(*pTaskInfo); terrno = code; return code; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0bd0d68caeca5e075326e0d443226e0c18f91e76..88489255624031b6808b4b4d92c28e9bed81ad7a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -790,6 +790,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con } } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; + tDecoderClear(&mr.coder); code = metaGetTableEntryByUid(&mr, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); @@ -1525,6 +1526,11 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { if (pStreamScan->pColMatchInfo) { taosArrayDestroy(pStreamScan->pColMatchInfo); } + if (pStreamScan->pPseudoExpr) { + destroyExprInfo(pStreamScan->pPseudoExpr, pStreamScan->numOfPseudoExpr); + taosMemoryFreeClear(pStreamScan->pPseudoExpr); + } + updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); @@ -2538,6 +2544,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { STableKeyInfo* item = taosArrayGet(pInfo->pTableList->pTableList, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); + tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 449f1921463ac4b6bce801a2e2412699b73d111d..77c9075a718f1006cf5fcf0571f2a7f3433a72a1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1752,30 +1752,11 @@ void increaseTs(SqlFunctionCtx* pCtx) { } } -SSDataBlock* createDeleteBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.hasVarCol = false; - pBlock->info.groupId = 0; - pBlock->info.rows = 0; - pBlock->info.type = STREAM_DELETE_RESULT; - pBlock->info.rowSize = sizeof(TSKEY) + sizeof(uint64_t); - - pBlock->pDataBlock = taosArrayInit(2, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - infoData.info.bytes = sizeof(TSKEY); - // window start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_UBIGINT; - infoData.info.bytes = sizeof(uint64_t); - taosArrayPush(pBlock->pDataBlock, &infoData); - - return pBlock; -} - void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type, SAggSupporter* pSup) { - ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + // Todo(liuyao) support partition by column + return; + } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->sessionSup.parentType = type; pScanInfo->sessionSup.pIntervalAggSup = pSup; @@ -2085,10 +2066,30 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock memcpy(pkey->pData, val, pkey->bytes); } } + + pSliceInfo->isPrevRowSet = true; +} + +static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + // null data should not be kept since it can not be used to perform interpolation + if (!colDataIsNull_s(pColInfoData, i)) { + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + memcpy(pkey->pData, val, pkey->bytes); + } + } + + pSliceInfo->isNextRowSet = true; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock) { + SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2164,6 +2165,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } case TSDB_FILL_PREV: { + if (!pSliceInfo->isPrevRowSet) { + break; + } + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; @@ -2171,8 +2176,12 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } case TSDB_FILL_NEXT: { - char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + if (!pSliceInfo->isNextRowSet) { + break; + } + + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); + colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; break; } @@ -2206,6 +2215,35 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB taosArrayPush(pInfo->pPrevRow, &key); } + pInfo->isPrevRowSet = false; + + return TSDB_CODE_SUCCESS; +} + +static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + if (pInfo->pNextRow != NULL) { + return TSDB_CODE_SUCCESS; + } + + pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pNextRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SGroupKeys key = {0}; + key.bytes = pColInfo->info.bytes; + key.type = pColInfo->info.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pColInfo->info.bytes); + taosArrayPush(pInfo->pNextRow, &key); + } + + pInfo->isNextRowSet = false; + return TSDB_CODE_SUCCESS; } @@ -2235,14 +2273,19 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); - int32_t numOfRows = 0; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { break; } - int32_t code = initPrevRowsKeeper(pSliceInfo, pBlock); + int32_t code; + code = initPrevRowsKeeper(pSliceInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + code = initNextRowsKeeper(pSliceInfo, pBlock); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2264,7 +2307,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, numOfRows, v, false); + colDataAppend(pDst, pResBlock->info.rows, v, false); } pResBlock->info.rows += 1; @@ -2281,11 +2324,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { + // in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + doKeepPrevRows(pSliceInfo, pBlock, i); + if (i < pBlock->info.rows - 1) { + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2304,8 +2352,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2313,12 +2364,48 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } + // add current row if timestamp match + if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } + + pResBlock->info.rows += 1; + doKeepPrevRows(pSliceInfo, pBlock, i); + + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + if (pSliceInfo->current > pSliceInfo->win.ekey) { doSetOperatorCompleted(pOperator); break; } } } + + // check if need to interpolate after ts range + // except for fill(next) + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } } // restore the value @@ -2376,6 +2463,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -2835,13 +2924,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { // process the rest of the data return pInfo->pUpdateRes; } - // doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); - // if (pInfo->pPullDataRes->info.rows != 0) { - // // process the rest of the data - // ASSERT(IS_FINAL_OP(pInfo)); - // printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); - // return pInfo->pPullDataRes; - // } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data @@ -2861,6 +2943,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.watermark); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA || pBlock->info.type == STREAM_INVALID) { @@ -2949,6 +3032,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); + } else { + pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; } finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); @@ -2983,7 +3068,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pDelRes; } - // ASSERT(false); return NULL; } @@ -2995,6 +3079,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) { pBlock->info.type = type; pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); + pBlock->info.watermark = INT64_MIN; pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; @@ -3184,7 +3269,6 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -3949,6 +4033,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } maxTs = TMAX(maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.watermark); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); @@ -4072,6 +4157,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; // restore the value pOperator->status = OP_RES_TO_RETURN; // semi operator diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 13ebcd4518ab97a17b9b1f0d8a3147c24a40ea7d..1f810f87da6448b381935b9c9d0549ed6bff861d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2297,7 +2297,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "derivative", .type = FUNCTION_TYPE_DERIVATIVE, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_CUMULATIVE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDerivative, .getEnvFunc = getDerivativeFuncEnv, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 34d58434c9c19454420fb3a23b3487b2dd149151..2be370ca0a4c5c09056ce2951bc61760807c2e2b 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -64,6 +64,9 @@ typedef struct SMinmaxResInfo { bool assign; // assign the first value or not int64_t v; STuplePos tuplePos; + + STuplePos nullTuplePos; + bool nullTupleSaved; } SMinmaxResInfo; typedef struct STopBotResItem { @@ -75,6 +78,10 @@ typedef struct STopBotResItem { typedef struct STopBotRes { int32_t maxSize; int16_t type; + + STuplePos nullTuplePos; + bool nullTupleSaved; + STopBotResItem* pItems; } STopBotRes; @@ -221,6 +228,10 @@ typedef struct SSampleInfo { int32_t numSampled; uint8_t colType; int16_t colBytes; + + STuplePos nullTuplePos; + bool nullTupleSaved; + char* data; STuplePos* tuplePos; } SSampleInfo; @@ -1134,6 +1145,9 @@ bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) SMinmaxResInfo* buf = GET_ROWCELL_INTERBUF(pResultInfo); buf->assign = false; buf->tuplePos.pageId = -1; + + buf->nullTupleSaved = false; + buf->nullTuplePos.pageId = -1; return true; } @@ -1575,6 +1589,10 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } _min_max_over: + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved ) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pBuf->nullTuplePos); + pBuf->nullTupleSaved = true; + } return numOfElems; } @@ -1615,7 +1633,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pEntryInfo->numOfRes > 0) { setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); } else { - setNullSelectivityValue(pCtx, pBlock, currentRow); + setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); } return pEntryInfo->numOfRes; @@ -3369,6 +3387,8 @@ bool topBotFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pRes->maxSize = pCtx->param[1].param.i; + pRes->nullTupleSaved = false; + pRes->nullTuplePos.pageId = -1; return true; } @@ -3406,6 +3426,10 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, true); } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + pRes->nullTupleSaved = true; + } return TSDB_CODE_SUCCESS; } @@ -3430,6 +3454,11 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, pRes->type, pInput->uid, pResInfo, false); } + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pRes->nullTuplePos); + pRes->nullTupleSaved = true; + } + return TSDB_CODE_SUCCESS; } @@ -3628,6 +3657,11 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { // todo assign the tag value and the corresponding row data int32_t currentRow = pBlock->info.rows; + if (pEntryInfo->numOfRes <= 0) { + colDataAppendNULL(pCol, currentRow); + setSelectivityValue(pCtx, pBlock, &pRes->nullTuplePos, currentRow); + return pEntryInfo->numOfRes; + } for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; if (type == TSDB_DATA_TYPE_FLOAT) { @@ -4900,7 +4934,8 @@ bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) pInfo->numSampled = 0; pInfo->colType = pCtx->resDataInfo.type; pInfo->colBytes = pCtx->resDataInfo.bytes; - + pInfo->nullTuplePos.pageId = -1; + pInfo->nullTupleSaved = false; pInfo->data = (char*)pInfo + sizeof(SSampleInfo); pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); @@ -4946,6 +4981,11 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { doReservoirSample(pCtx, pInfo, data, i); } + if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { + doSaveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, &pInfo->nullTuplePos); + pInfo->nullTupleSaved = true; + } + SET_VAL(pResInfo, pInfo->numSampled, pInfo->numSampled); return TSDB_CODE_SUCCESS; } @@ -4960,6 +5000,11 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); int32_t currentRow = pBlock->info.rows; + if (pInfo->numSampled == 0) { + colDataAppendNULL(pCol, currentRow); + setSelectivityValue(pCtx, pBlock, &pInfo->nullTuplePos, currentRow); + return pInfo->numSampled; + } for (int32_t i = 0; i < pInfo->numSampled; ++i) { colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); setSelectivityValue(pCtx, pBlock, &pInfo->tuplePos[i], currentRow + i); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 72ba81c3a7c32b5d2dd9d98a49378e9d62551121..4f61b750d383c6495e52184fb6cd6f40ff56b7f9 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -226,6 +226,18 @@ bool fmIsLastRowFunc(int32_t funcId) { return FUNCTION_TYPE_LAST_ROW == funcMgtBuiltins[funcId].type; } +bool fmIsNotNullOutputFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_LAST == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_LAST_PARTIAL == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_LAST_MERGE == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST_PARTIAL == funcMgtBuiltins[funcId].type || + FUNCTION_TYPE_FIRST_MERGE == funcMgtBuiltins[funcId].type; +} + bool fmIsSelectValueFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 896e2bc2398ebe50ed6deafe11d8e456403d8246..2249bc7823a49589e99f1714d06401b419c3d72d 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -31,13 +31,19 @@ extern "C" { #define parserDebug(param, ...) qDebug("PARSER: " param, ##__VA_ARGS__) #define parserTrace(param, ...) qTrace("PARSER: " param, ##__VA_ARGS__) -#define PK_TS_COL_INTERNAL_NAME "_rowts" +#define ROWTS_PSEUDO_COLUMN_NAME "_rowts" +#define C0_PSEUDO_COLUMN_NAME "_c0" typedef struct SMsgBuf { int32_t len; char* buf; } SMsgBuf; +typedef struct SParseTablesMetaReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SHashObj* pTables; +} SParseTablesMetaReq; + typedef struct SParseMetaCache { SHashObj* pTableMeta; // key is tbFName, element is STableMeta* SHashObj* pDbVgroup; // key is dbFName, element is SArray* @@ -94,7 +100,7 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); -void destoryParseMetaCache(SParseMetaCache* pMetaCache); +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index b2006f8fc947c5a0d6e1e575a0f827b52439da45..d19c203ffeefce0fce6057b2a20788a1df8caa6f 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -443,19 +443,23 @@ SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, nodesCloneNode(pExpr), pRight)); } -static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt) { +static SNode* createPrimaryKeyCol(SAstCreateContext* pCxt, const SToken* pFuncName) { CHECK_PARSER_STATUS(pCxt); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); CHECK_OUT_OF_MEM(pCol); pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME); + if (NULL == pFuncName) { + strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); + } else { + strncpy(pCol->colName, pFuncName->z, pFuncName->n); + } return (SNode*)pCol; } SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { CHECK_PARSER_STATUS(pCxt); if (0 == strncasecmp("_rowts", pFuncName->z, pFuncName->n) || 0 == strncasecmp("_c0", pFuncName->z, pFuncName->n)) { - return createPrimaryKeyCol(pCxt); + return createPrimaryKeyCol(pCxt, pFuncName); } SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(func); @@ -586,7 +590,7 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { CHECK_PARSER_STATUS(pCxt); SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); CHECK_OUT_OF_MEM(state); - state->pCol = createPrimaryKeyCol(pCxt); + state->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == state->pCol) { nodesDestroyNode((SNode*)state); CHECK_OUT_OF_MEM(state->pCol); @@ -600,7 +604,7 @@ SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode CHECK_PARSER_STATUS(pCxt); SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); CHECK_OUT_OF_MEM(interval); - interval->pCol = createPrimaryKeyCol(pCxt); + interval->pCol = createPrimaryKeyCol(pCxt, NULL); if (NULL == interval->pCol) { nodesDestroyNode((SNode*)interval); CHECK_OUT_OF_MEM(interval->pCol); @@ -639,7 +643,7 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd) { CHECK_PARSER_STATUS(pCxt); - return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt), pStart, pEnd); + return createBetweenAnd(pCxt, createPrimaryKeyCol(pCxt, NULL), pStart, pEnd); } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias) { @@ -752,7 +756,7 @@ SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt) && NULL != pFill) { SFillNode* pFillClause = (SFillNode*)pFill; nodesDestroyNode(pFillClause->pWStartTs); - pFillClause->pWStartTs = createPrimaryKeyCol(pCxt); + pFillClause->pWStartTs = createPrimaryKeyCol(pCxt, NULL); ((SSelectStmt*)pStmt)->pFill = (SNode*)pFillClause; } return pStmt; @@ -1731,7 +1735,7 @@ SNode* createCountFuncForDelete(SAstCreateContext* pCxt) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); strcpy(pFunc->functionName, "count"); - if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt))) { + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2f6725acaf8e3fb6b221f187afa22801fda24117..5c49a6e0aba6cc6e6daa190c5eb957ef42b5dacc 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -612,7 +612,8 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p } static bool isInternalPrimaryKey(const SColumnNode* pCol) { - return PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME); + return PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && + (0 == strcmp(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME) || 0 == strcmp(pCol->colName, C0_PSEUDO_COLUMN_NAME)); } static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, const STableNode* pTable, @@ -1252,20 +1253,21 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (isSelectStmt(pCxt->pCurrStmt)) { - // select percentile() without from clause is also valid - if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { - return TSDB_CODE_SUCCESS; - } - SNode* pTable = ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; - if (QUERY_NODE_REAL_TABLE == nodeType(pTable) && - (TSDB_CHILD_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType || - TSDB_NORMAL_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType)) { - return TSDB_CODE_SUCCESS; - } + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, + "%s is only supported in single table query", pFunc->functionName); } - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, - "%s is only supported in single table query", pFunc->functionName); + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SNode* pTable = pSelect->pFromTable; + // select percentile() without from clause is also valid + if ((NULL != pTable && (QUERY_NODE_REAL_TABLE != nodeType(pTable) || + (TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && + TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType))) || + NULL != pSelect->pPartitionByList) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, + "%s is only supported in single table query", pFunc->functionName); + } + return TSDB_CODE_SUCCESS; } static bool isStar(SNode* pNode) { @@ -2508,9 +2510,31 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t translateStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { +static bool isPartitionByTbname(SNodeList* pPartitionByList) { + if (1 != LIST_LENGTH(pPartitionByList)) { + return false; + } + SNode* pPartKey = nodesListGetNode(pPartitionByList, 0); + return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; +} + +static int32_t checkStateWindowForStream(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (!pCxt->createStream) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !isPartitionByTbname(pSelect->pPartitionByList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateStateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SStateWindowNode* pState = (SStateWindowNode*)pSelect->pWindow; nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); - // todo check for "function not support for state_window" + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = checkStateWindowForStream(pCxt, pSelect); + } return pCxt->errCode; } @@ -2521,14 +2545,13 @@ static int32_t translateSessionWindow(STranslateContext* pCxt, SSessionWindowNod if (PRIMARYKEY_TIMESTAMP_COL_ID != pSession->pCol->colId) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SESSION_COL); } - // todo check for "function not support for session" return TSDB_CODE_SUCCESS; } static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return translateStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); + return translateStateWindow(pCxt, pSelect); case QUERY_NODE_SESSION_WINDOW: return translateSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: @@ -2565,7 +2588,7 @@ static int32_t createDefaultFillNode(STranslateContext* pCxt, SNode** pOutput) { return TSDB_CODE_OUT_OF_MEMORY; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME); + strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); pFill->pWStartTs = (SNode*)pCol; *pOutput = (SNode*)pFill; @@ -2650,7 +2673,7 @@ static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* p return TSDB_CODE_OUT_OF_MEMORY; } pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(pCol->colName, PK_TS_COL_INTERNAL_NAME); + strcpy(pCol->colName, ROWTS_PSEUDO_COLUMN_NAME); bool found = false; int32_t code = findAndSetColumn(pCxt, &pCol, pTable, &found); if (TSDB_CODE_SUCCESS != code || !found) { @@ -3876,7 +3899,7 @@ static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, ch return TSDB_CODE_OUT_OF_MEMORY; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - strcpy(((SColumnNode*)pInterval->pCol)->colName, PK_TS_COL_INTERNAL_NAME); + strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); pCxt->createStream = true; int32_t code = translateQuery(pCxt, (SNode*)pSelect); @@ -4683,6 +4706,12 @@ static int32_t translateKillTransaction(STranslateContext* pCxt, SKillStmt* pStm return buildCmdMsg(pCxt, TDMT_MND_KILL_TRANS, (FSerializeFunc)tSerializeSKillTransReq, &killReq); } +static bool crossTableWithoutAggOper(SSelectStmt* pSelect) { + return NULL == pSelect->pWindow && !pSelect->hasAggFuncs && !pSelect->hasIndefiniteRowsFunc && + !pSelect->hasInterpFunc && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !isPartitionByTbname(pSelect->pPartitionByList); +} + static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { if (NULL != pStmt->pOptions->pWatermark && (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark))) { @@ -4698,14 +4727,12 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { - SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - if (QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable)) { - return TSDB_CODE_SUCCESS; - } + if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) || + QUERY_NODE_REAL_TABLE != nodeType(((SSelectStmt*)pStmt->pQuery)->pFromTable)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY); + return TSDB_CODE_SUCCESS; } static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { @@ -4734,12 +4761,23 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { return code; } +static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || + !pSelect->isTimeLineResult || crossTableWithoutAggOper(pSelect)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; int32_t code = addWstartTsToCreateStreamQuery(pStmt); if (TSDB_CODE_SUCCESS == code) { code = translateQuery(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + code = checkStreamQuery(pCxt, (SSelectStmt*)pStmt); + } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt, false, &pReq->ast, NULL); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e51800aece4cdb4bdaee5386490277b65ec366f5..ae5a281aab92ab5e365fe19e1769d95b2b43ea47 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -474,6 +474,24 @@ static int32_t buildDbReq(SHashObj* pDbsHash, SArray** pDbs) { return TSDB_CODE_SUCCESS; } +static int32_t buildTableReqFromDb(SHashObj* pDbsHash, SArray** pDbs) { + if (NULL != pDbsHash) { + *pDbs = taosArrayInit(taosHashGetSize(pDbsHash), sizeof(STablesReq)); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SParseTablesMetaReq* p = taosHashIterate(pDbsHash, NULL); + while (NULL != p) { + STablesReq req = {0}; + strcpy(req.dbFName, p->dbFName); + buildTableReq(p->pTables, &req.pTables); + taosArrayPush(*pDbs, &req); + p = taosHashIterate(pDbsHash, p); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) { if (NULL != pUserAuthHash) { *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo)); @@ -513,12 +531,12 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) { } int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - int32_t code = buildTableReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); + int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = buildTableReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); + code = buildTableReqFromDb(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); } if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg); @@ -587,6 +605,24 @@ static int32_t putDbDataToCache(const SArray* pDbReq, const SArray* pDbData, SHa return TSDB_CODE_SUCCESS; } +static int32_t putDbTableDataToCache(const SArray* pDbReq, const SArray* pTableData, SHashObj** pTable) { + int32_t ndbs = taosArrayGetSize(pDbReq); + int32_t tableNo = 0; + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = taosArrayGet(pDbReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(taosArrayGet(pReq->pTables, j), fullName); + if (TSDB_CODE_SUCCESS != putMetaDataToHash(fullName, strlen(fullName), pTableData, tableNo, pTable)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ++tableNo; + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUserAuthData, SHashObj** pUserAuth) { int32_t nvgs = taosArrayGetSize(pUserAuthReq); for (int32_t i = 0; i < nvgs; ++i) { @@ -612,12 +648,12 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas } int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { - int32_t code = putTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); + int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = putTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); + code = putDbTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); } if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, &pMetaCache->pDbCfg); @@ -657,14 +693,38 @@ static int32_t reserveTableReqInCache(int32_t acctId, const char* pDb, const cha return reserveTableReqInCacheImpl(fullName, len, pTables); } +static int32_t reserveTableReqInDbCacheImpl(int32_t acctId, const char* pDb, const char* pTable, SHashObj* pDbs) { + SParseTablesMetaReq req = {0}; + int32_t len = snprintf(req.dbFName, sizeof(req.dbFName), "%d.%s", acctId, pDb); + int32_t code = reserveTableReqInCache(acctId, pDb, pTable, &req.pTables); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pDbs, req.dbFName, len, &req, sizeof(SParseTablesMetaReq)); + } + return code; +} + +static int32_t reserveTableReqInDbCache(int32_t acctId, const char* pDb, const char* pTable, SHashObj** pDbs) { + if (NULL == *pDbs) { + *pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + char fullName[TSDB_DB_FNAME_LEN]; + int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); + SParseTablesMetaReq* pReq = taosHashGet(*pDbs, fullName, len); + if (NULL == pReq) { + return reserveTableReqInDbCacheImpl(acctId, pDb, pTable, *pDbs); + } + return reserveTableReqInCache(acctId, pDb, pTable, &pReq->pTables); +} + int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); } int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableMeta); } int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { @@ -711,13 +771,11 @@ int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, } int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); } int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableVgroup); } int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) { @@ -919,10 +977,24 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { return TSDB_CODE_SUCCESS; } -void destoryParseMetaCache(SParseMetaCache* pMetaCache) { - taosHashCleanup(pMetaCache->pTableMeta); +void destoryParseTablesMetaReqHash(SHashObj* pHash) { + SParseTablesMetaReq* p = taosHashIterate(pHash, NULL); + while (NULL != p) { + taosHashCleanup(p->pTables); + p = taosHashIterate(pHash, p); + } + taosHashCleanup(pHash); +} + +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + if (request) { + destoryParseTablesMetaReqHash(pMetaCache->pTableMeta); + destoryParseTablesMetaReqHash(pMetaCache->pTableVgroup); + } else { + taosHashCleanup(pMetaCache->pTableMeta); + taosHashCleanup(pMetaCache->pTableVgroup); + } taosHashCleanup(pMetaCache->pDbVgroup); - taosHashCleanup(pMetaCache->pTableVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); taosHashCleanup(pMetaCache->pUserAuth); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 53ee717af299b64c07c2050b5d3659210c731fff..34cd783ace5c84608de6d62ae6b994c2fbb9e3c3 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -85,13 +85,13 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { taosMemoryFreeClear(pVal->datum.p); } - + if (pParam->is_null && 1 == *(pParam->is_null)) { pVal->node.resType.type = TSDB_DATA_TYPE_NULL; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } - + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; @@ -187,7 +187,7 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(&metaCache, pCatalogReq); } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, true); terrno = code; return code; } @@ -203,7 +203,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, false); terrno = code; return code; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4158453110f4c3b511ab01380525d8d4f1e726bb..6717a1fdf15719c4cc6ff59e891a4bbd5713d323 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -472,12 +472,16 @@ class MockCatalogServiceImpl { int32_t getAllTableMeta(SArray* pTableMetaReq, SArray** pTableMetaData) const { if (NULL != pTableMetaReq) { - int32_t ntables = taosArrayGetSize(pTableMetaReq); - *pTableMetaData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), (STableMeta**)&res.pRes); - taosArrayPush(*pTableMetaData, &res); + int32_t ndbs = taosArrayGetSize(pTableMetaReq); + *pTableMetaData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableMetaReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.code = catalogGetTableMeta((const SName*)taosArrayGet(pReq->pTables, j), (STableMeta**)&res.pRes); + taosArrayPush(*pTableMetaData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -485,13 +489,17 @@ class MockCatalogServiceImpl { int32_t getAllTableVgroup(SArray* pTableVgroupReq, SArray** pTableVgroupData) const { if (NULL != pTableVgroupReq) { - int32_t ntables = taosArrayGetSize(pTableVgroupReq); - *pTableVgroupData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), (SVgroupInfo*)res.pRes); - taosArrayPush(*pTableVgroupData, &res); + int32_t ndbs = taosArrayGetSize(pTableVgroupReq); + *pTableVgroupData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableVgroupReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pReq->pTables, j), (SVgroupInfo*)res.pRes); + taosArrayPush(*pTableVgroupData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -677,12 +685,17 @@ int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SM return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } +void MockCatalogService::destoryTablesReq(void* p) { + STablesReq* pRes = (STablesReq*)p; + taosArrayDestroy(pRes->pTables); +} + void MockCatalogService::destoryCatalogReq(SCatalogReq* pReq) { taosArrayDestroy(pReq->pDbVgroup); taosArrayDestroy(pReq->pDbCfg); taosArrayDestroy(pReq->pDbInfo); - taosArrayDestroy(pReq->pTableMeta); - taosArrayDestroy(pReq->pTableHash); + taosArrayDestroyEx(pReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pReq->pTableHash, destoryTablesReq); taosArrayDestroy(pReq->pUdf); taosArrayDestroy(pReq->pIndex); taosArrayDestroy(pReq->pUser); diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index d76a6abca8e1d23c733459aa190868c8a7f0e439..c0330692ee4bbcf32cdcbeb6cd48822708976965 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -50,6 +50,7 @@ struct MockTableMeta { class MockCatalogServiceImpl; class MockCatalogService { public: + static void destoryTablesReq(void* p); static void destoryCatalogReq(SCatalogReq* pReq); static void destoryMetaRes(void* p); static void destoryMetaArrayRes(void* p); diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index f3f87e945aea3c93158d1e57ff9fb48ad7d64bf8..7302491ba7b15daca8333c4b9870eb3615e0c015 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include + #include #include "mockCatalogService.h" @@ -20,6 +22,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace { @@ -63,7 +66,9 @@ class InsertTest : public Test { int32_t runAsync() { cxt_.async = true; - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), std::bind(_destoryParseMetaCache, _1, cref(request))); code_ = parseInsertSyntax(&cxt_, &res_, metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "parseInsertSyntax code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -81,6 +86,8 @@ class InsertTest : public Test { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); g_mockCatalogService->catalogGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; code_ = putMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "putMetaDataToCache code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -144,8 +151,8 @@ class InsertTest : public Test { static const int max_err_len = 1024; static const int max_sql_len = 1024 * 1024; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 235cc487fb90a0d71b63d4e95dcf376468bf3f62..3fe4b533e44fe70e8e999ef3cacd15715cd632dd 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -24,6 +24,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace ParserTest { @@ -118,8 +119,8 @@ class ParserTestBaseImpl { TEST_INTERFACE_ASYNC_API }; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } @@ -340,7 +341,9 @@ class ParserTestBaseImpl { doParse(&cxt, query.get()); SQuery* pQuery = *(query.get()); - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); doCollectMetaKey(&cxt, pQuery, metaCache.get()); unique_ptr catalogReq(new SCatalogReq(), @@ -353,6 +356,8 @@ class ParserTestBaseImpl { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); doGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); doAuthenticate(&cxt, pQuery, metaCache.get()); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 86a4441cfff314691cb412a3ea3d746484991cf5..f4f7c9aefdfe64f5e14aa02a4adecc1129683080 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1711,7 +1711,7 @@ static bool eliminateProjOptCanChildConditionUseChildTargets(SLogicNode* pChild, if (!cxt.canUse) return false; } if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild) && NULL != ((SJoinLogicNode*)pChild)->pOnConditions) { - SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; + SJoinLogicNode* pJoinLogicNode = (SJoinLogicNode*)pChild; CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = false}; nodesWalkExpr(pJoinLogicNode->pOnConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); if (!cxt.canUse) return false; @@ -1768,7 +1768,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); - //if pChild is a project logic node, remove its projection which is not reference by its target. + // if pChild is a project logic node, remove its projection which is not reference by its target. alignProjectionWithTarget(pChild); } pCxt->optimized = true; @@ -2404,6 +2404,9 @@ static const SOptimizeRule optimizeRuleSet[] = { static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) { + if (0 == (qDebugFlag & DEBUG_DEBUG)) { + return; + } char* pStr = NULL; nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL); if (NULL == pRuleName) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 6ddb6d90c0509b78e442e24f46af5cf41217495c..c582994b7c319778477238ab26b99ce844cb8c1c 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -264,11 +264,11 @@ static bool stbSplNeedSplitJoin(bool streamQuery, SJoinLogicNode* pJoin) { static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); + return streamQuery ? false : stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); case QUERY_NODE_LOGIC_PLAN_JOIN: return stbSplNeedSplitJoin(streamQuery, (SJoinLogicNode*)pNode); case QUERY_NODE_LOGIC_PLAN_PARTITION: - return stbSplIsMultiTbScanChild(streamQuery, pNode); + return streamQuery ? false : stbSplIsMultiTbScanChild(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: @@ -1427,6 +1427,9 @@ static const SSplitRule splitRuleSet[] = { static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) { + if (0 == (qDebugFlag & DEBUG_DEBUG)) { + return; + } char* pStr = NULL; nodesNodeToString((SNode*)pSubplan, false, &pStr, NULL); if (NULL == pRuleName) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 0554779746ac6fc6f0ebf2badeee55d24f9b9ef5..c1296982e0217ae9b3c2e67b210f1922492cf547 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -19,6 +19,9 @@ #include "scalar.h" static void dumpQueryPlan(SQueryPlan* pPlan) { + if (0 == (qDebugFlag & DEBUG_DEBUG)) { + return; + } char* pStr = NULL; nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); planDebugL("QID:0x%" PRIx64 " Query Plan: %s", pPlan->queryId, pStr); @@ -42,6 +45,9 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo if (TSDB_CODE_SUCCESS == code) { code = createPhysiPlan(pCxt, pLogicPlan, pPlan, pExecNodeList); } + if (TSDB_CODE_SUCCESS == code) { + dumpQueryPlan(*pPlan); + } nodesDestroyNode((SNode*)pLogicSubplan); nodesDestroyNode((SNode*)pLogicPlan); @@ -79,6 +85,7 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown } int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { + planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -97,7 +104,10 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { FOREACH(pChild, pNode->pChildren) { clearSubplanExecutionNode((SPhysiNode*)pChild); } } -void qClearSubplanExecutionNode(SSubplan* pSubplan) { clearSubplanExecutionNode(pSubplan->pNode); } +void qClearSubplanExecutionNode(SSubplan* pSubplan) { + planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + clearSubplanExecutionNode(pSubplan->pNode); +} int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index d8fda577910c69ce963bcc35e2f94ba2e079234f..4cad6a078b02e34777e052248a429a3626a8b81e 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -462,3 +462,5 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + + diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 3ee870ef96235c1be598c600ccce0e51b4f97b21..3ff5b5950f6e241af51d77da843d7f00e9fdd0b6 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -25,7 +25,7 @@ extern "C" { int32_t qwAbortPrerocessQuery(QW_FPARAMS_DEF); int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql); +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char* sql); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 63a3c1bfeaf86a3cba3de1f2cb5ea5a72f50a666..ecc25e9c2df6fd1d0efb35cc21d871fde64784c5 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -390,10 +390,13 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); - QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); - QW_SCH_TASK_DLOG("processQuery end, node:%p", node); + QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); - return TSDB_CODE_SUCCESS; +_return: + + QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%d", node, code); + + return code; } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d93c07ce1ec26c7dd25ad737e35044eca7053c36..7c5cfda81fd7d6380777b203c64af44929968652 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -539,7 +539,7 @@ _return: } -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char* sql) { int32_t code = 0; bool queryRsped = false; SSubplan *plan = NULL; @@ -567,6 +567,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { } code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, sql, OPTR_EXEC_MODEL_BATCH); + sql = NULL; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -593,6 +594,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { _return: + taosMemoryFree(sql); + input.code = code; input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a6a47da2e5442b29aa26cefd843f0c4151e75d87..726e6e1a23d58d337fd9d1e8a0951d4f72d11089 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3246,6 +3246,10 @@ _return: } bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { + if (info->scalarMode) { + return true; + } + if (FILTER_EMPTY_RES(info)) { return false; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 254c99f05dde624794db615e142a8a6fa246c00b..aaa70ef5ae5f8ab00ce88b56433885cd00004893 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1094,6 +1094,7 @@ static SColumnInfoData* doVectorConvert(SScalarParam* pInput, int32_t* doConvert static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { if (type == VECTOR_DO_CONVERT) { colDataDestroy(pCol); + taosMemoryFree(pCol); } } diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index a4fa4f283913c785dca72944b2b3178e4ce5d84f..64cda573f1fa8bd022a548787151b2b9fd64d44f 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -64,7 +64,7 @@ _return: int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { SSchJob *pJob = schAcquireJob(jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ecd0253e1c7c5c904d3b14bb64eaa124ef00571d..cabca0dc0ce208a5912dff9b1373ce80228e429c 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -284,7 +284,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { @@ -298,6 +297,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &parent->planLock); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); + if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId); SCH_ERR_RET(schLaunchTask(pJob, parent)); @@ -536,6 +537,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { pTask->maxExecTimes++; + pTask->maxRetryTimes++; if (pTask->timeoutUsec < SCH_MAX_TASK_TIMEOUT_USEC) { pTask->timeoutUsec *= 2; if (pTask->timeoutUsec > SCH_MAX_TASK_TIMEOUT_USEC) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3a15523040f7c0c108273563c17f8000c43ff582..a37cd4fd9e6abd59ad59bf698d2d8f6e0dacd948 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -150,7 +150,7 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { SSchJob *pJob = schAcquireJob(*jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 33e864158ad4b107237356c805260a913bd89094..ceddf4f2153735c4d8423dde806272419dc35fc5 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -8,7 +8,8 @@ target_include_directories( target_link_libraries( stream - PRIVATE os util transport qcom executor tdb + PUBLIC tdb + PRIVATE os util transport qcom executor ) if(${BUILD_TEST}) diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 7a19998d0286c58409a67deb84a849a58acd1f0c..30f0919ceeaedc52912189ec7150cb9c48d0000e 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -47,7 +47,7 @@ void streamCleanUp() { } } -void streamTriggerByTimer(void* param, void* tmrId) { +void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { @@ -65,31 +65,33 @@ void streamTriggerByTimer(void* param, void* tmrId) { } trigger->pBlock->info.type = STREAM_GET_ALL; - atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); + atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); streamTaskInput(pTask, (SStreamQueueItem*)trigger); - streamLaunchByWrite(pTask, pTask->nodeId); + streamSchedExec(pTask); } - taosTmrReset(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); + taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); } int32_t streamSetupTrigger(SStreamTask* pTask) { if (pTask->triggerParam != 0) { - pTask->timer = taosTmrStart(streamTriggerByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); - pTask->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + pTask->timer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer); + pTask->triggerStatus = TASK_TRIGGER_STATUS__INACTIVE; } return 0; } -int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId) { - int8_t execStatus = atomic_load_8(&pTask->execStatus); - if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) { +int32_t streamSchedExec(SStreamTask* pTask) { + int8_t schedStatus = + atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); - if (pRunReq == NULL) return -1; - - // TODO: do we need htonl? - pRunReq->head.vgId = vgId; + if (pRunReq == NULL) { + atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + return -1; + } + pRunReq->head.vgId = pTask->nodeId; pRunReq->streamId = pTask->streamId; pRunReq->taskId = pTask->taskId; SRpcMsg msg = { @@ -182,14 +184,13 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S streamTaskEnqueue(pTask, pReq, pRsp); if (exec) { - streamExec(pTask); + streamTryExec(pTask); - if (pTask->dispatchType != TASK_DISPATCH__NONE) { - ASSERT(pTask->sinkType == TASK_SINK__NONE); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); } } else { - streamLaunchByWrite(pTask, pTask->nodeId); + streamSchedExec(pTask); } return 0; @@ -200,7 +201,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { qDebug("task %d receive dispatch rsp", pTask->taskId); - if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp); if (leftRsp > 0) return 0; @@ -219,9 +220,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { } int32_t streamProcessRunReq(SStreamTask* pTask) { - streamExec(pTask); + streamTryExec(pTask); - if (pTask->dispatchType != TASK_DISPATCH__NONE) { + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); } return 0; @@ -249,7 +250,7 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) streamProcessRunReq(pTask); - if (pTask->isDataScan) { + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { // scan data to recover pTask->inputStatus = TASK_INPUT_STATUS__RECOVER; pTask->taskStatus = TASK_STATUS__RECOVERING; @@ -271,11 +272,12 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S streamTaskEnqueueRetrieve(pTask, pReq, pRsp); - ASSERT(pTask->execType != TASK_EXEC__NONE); - streamExec(pTask); + ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); + streamSchedExec(pTask); - ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); - streamDispatch(pTask); + /*streamTryExec(pTask);*/ + + /*streamDispatch(pTask);*/ return 0; } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index c96854b198d764de17ffa1673077ee8cda57e8cf..629a6e8b068e59e13053322e8059d63051d95db0 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -34,6 +34,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); pDataBlock->info.version = be64toh(pRetrieve->version); + pDataBlock->info.watermark = be64toh(pRetrieve->watermark); pDataBlock->info.type = pRetrieve->streamBlockType; pDataBlock->info.childId = pReq->upstreamChildId; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 26cd9111bf7c3b9efee3a232755c841add2932df..8d6d31e37f1f13f41c5f7d969f6040a68fb02a15 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -184,6 +184,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); + pRetrieve->watermark = htobe64(pBlock->info.watermark); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); @@ -242,7 +243,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat int32_t blockNum = taosArrayGetSize(pData->blocks); ASSERT(blockNum != 0); - if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { SStreamDispatchReq req = { .streamId = pTask->streamId, .dataSrcVgId = pData->srcVgId, @@ -282,7 +283,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat taosArrayDestroy(req.dataLen); return code; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t rspCnt = atomic_load_32(&pTask->shuffleDispatcher.waitingRspCnt); ASSERT(rspCnt == 0); @@ -393,11 +394,11 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, int32_t vgId = 0; int32_t downstreamTaskId = 0; // find ep - if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { vgId = pTask->fixedEpDispatcher.nodeId; *ppEpSet = &pTask->fixedEpDispatcher.epSet; downstreamTaskId = pTask->fixedEpDispatcher.taskId; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { // TODO get ctbName for each block SSDataBlock* pBlock = taosArrayGet(data->blocks, 0); char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); @@ -439,14 +440,13 @@ FAIL: } int32_t streamDispatch(SStreamTask* pTask) { - ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); -#if 1 + ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); + int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); if (old != TASK_OUTPUT_STATUS__NORMAL) { return 0; } -#endif SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue); if (pBlock == NULL) { @@ -466,22 +466,8 @@ int32_t streamDispatch(SStreamTask* pTask) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); goto FREE; } - /*atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL);*/ FREE: taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); -#if 0 - SRpcMsg dispatchMsg = {0}; - SEpSet* pEpSet = NULL; - if (streamBuildDispatchMsg(pTask, pBlock, &dispatchMsg, &pEpSet) < 0) { - ASSERT(0); - atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); - return -1; - } - taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); - taosFreeQitem(pBlock); - - tmsgSendReq(pEpSet, &dispatchMsg); -#endif return code; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 3c008f79344117210a653454496da9be187c7b94..e662c18a15e269aa31131841c10b1328e55a2e98 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,7 +24,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) SStreamTrigger* pTrigger = (SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(pTask->isDataScan); + ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); @@ -92,7 +92,7 @@ static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock } int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { - ASSERT(pTask->execType != TASK_EXEC__NONE); + ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); void* exec = pTask->exec.executor; @@ -139,32 +139,30 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum) { return -1; } - if (pTask->dispatchType != TASK_DISPATCH__NONE) { - ASSERT(pTask->sinkType == TASK_SINK__NONE); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); } } return 0; } - -static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { +// TODO: handle version +int32_t streamExecForAll(SStreamTask* pTask) { while (1) { int32_t cnt = 1; void* data = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { - qDebug("stream exec over, queue empty"); + qDebug("stream task exec over, queue empty, task: %d", pTask->taskId); break; } if (data == NULL) { data = qItem; streamQueueProcessSuccess(pTask->inputQueue); - if (pTask->execType == TASK_EXEC__NONE) break; - /*if (qItem->type == STREAM_INPUT__DATA_BLOCK) {*/ - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ - /*}*/ + if (pTask->taskLevel == TASK_LEVEL__SINK) { + break; + } } else { void* newRet; if ((newRet = streamAppendQueueItem(data, qItem)) == NULL) { @@ -181,18 +179,21 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (pTask->taskStatus == TASK_STATUS__DROPPING) { if (data) streamFreeQitem(data); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return NULL; + return 0; } - if (data == NULL) break; + if (data == NULL) { + break; + } - if (pTask->execType == TASK_EXEC__NONE) { + if (pTask->taskLevel == TASK_LEVEL__SINK) { ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK); streamTaskOutput(pTask, data); continue; } + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); @@ -203,76 +204,44 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { // TODO log failed ver streamQueueProcessFail(pTask->inputQueue); taosArrayDestroy(pRes); - return NULL; + return -1; } qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - if (streamTaskOutput(pTask, qRes) < 0) { - // TODO log failed ver - /*streamQueueProcessFail(pTask->inputQueue);*/ - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); - return NULL; - } + if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } + + if (streamTaskOutput(pTask, qRes) < 0) { + // TODO save failed ver + /*streamQueueProcessFail(pTask->inputQueue);*/ + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); + return -1; + } /*streamQueueProcessSuccess(pTask->inputQueue);*/ - pRes = taosArrayInit(0, sizeof(SSDataBlock)); } - - streamFreeQitem(data); } - return pRes; + return 0; } -// TODO: handle version -int32_t streamExec(SStreamTask* pTask) { - SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - if (pRes == NULL) return -1; - while (1) { - int8_t execStatus = - atomic_val_compare_exchange_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE, TASK_EXEC_STATUS__EXECUTING); - if (execStatus == TASK_EXEC_STATUS__IDLE) { - // first run - qDebug("stream exec, enter exec status"); - pRes = streamExecForQall(pTask, pRes); - if (pRes == NULL) goto FAIL; - -// temporarily disable status closing, since it runs out of threads -#if 0 - // set status closing - atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__CLOSING); - - // second run, make sure inputQ and qall are cleared - qDebug("stream exec, enter closing status"); - pRes = streamExecForQall(pTask, pRes); - if (pRes == NULL) goto FAIL; -#endif +int32_t streamTryExec(SStreamTask* pTask) { + int8_t schedStatus = + atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + if (schedStatus == TASK_SCHED_STATUS__WAITING) { + int32_t code = streamExecForAll(pTask); + if (code < 0) { + atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED); + return -1; + } + atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); - qDebug("stream exec, return result"); - return 0; - } else if (execStatus == TASK_EXEC_STATUS__CLOSING) { - continue; - } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) { - ASSERT(taosArrayGetSize(pRes) == 0); - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return 0; - } else { - ASSERT(0); + if (!taosQueueEmpty(pTask->inputQueue->queue)) { + streamSchedExec(pTask); } } -FAIL: - if (pRes) taosArrayDestroy(pRes); - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - tFreeSStreamTask(pTask); - return 0; - } else { - atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); - return -1; - } + return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index be9dc81c3c72dc3437648bac8c48d653b17dff96..8faa22d6439487bad9c76ab5ad554489b752976e 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -14,22 +14,8 @@ */ #include "executor.h" -#include "tdbInt.h" #include "tstream.h" -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask); - -typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pStateDb; - SHashObj* pTasks; - void* ahandle; - TXN txn; - FTaskExpand* expandFunc; -} SStreamMeta; - SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc) { SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -50,17 +36,70 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + if (pMeta->pTasks == NULL) { + goto _err; + } + + if (streamMetaBegin(pMeta) < 0) { + goto _err; + } + pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; + return pMeta; _err: - return NULL; } void streamMetaClose(SStreamMeta* pMeta) { - // - return; + tdbCommit(pMeta->db, &pMeta->txn); + tdbTbClose(pMeta->pTaskDb); + tdbTbClose(pMeta->pStateDb); + tdbClose(pMeta->db); + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pMeta->pTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = *(SStreamTask**)pIter; + tFreeSStreamTask(pTask); + } + taosHashCleanup(pMeta->pTasks); + taosMemoryFree(pMeta->path); + taosMemoryFree(pMeta); +} + +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, char* msg, int32_t msgLen) { + SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); + if (pTask == NULL) { + return -1; + } + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + if (tDecodeSStreamTask(&decoder, pTask) < 0) { + ASSERT(0); + goto FAIL; + } + tDecoderClear(&decoder); + + if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + ASSERT(0); + goto FAIL; + } + + taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); + + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + ASSERT(0); + return -1; + } + return 0; + +FAIL: + if (pTask) taosMemoryFree(pTask); + return -1; } int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask) { @@ -93,6 +132,16 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } +SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); + if (ppTask) { + ASSERT((*ppTask)->taskId == taskId); + return *ppTask; + } else { + return NULL; + } +} + int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { @@ -123,14 +172,33 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) { if (tdbCommit(pMeta->db, &pMeta->txn) < 0) { return -1; } + memset(&pMeta->txn, 0, sizeof(TXN)); + if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { + return -1; + } + if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + return -1; + } return 0; } -int32_t streamMetaRollBack(SStreamMeta* pMeta) { - // TODO tdb rollback +int32_t streamMetaAbort(SStreamMeta* pMeta) { + if (tdbAbort(pMeta->db, &pMeta->txn) < 0) { + return -1; + } + memset(&pMeta->txn, 0, sizeof(TXN)); + if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { + return -1; + } + if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + return -1; + } return 0; } -int32_t streamRestoreTask(SStreamMeta* pMeta) { + +int32_t streamLoadTasks(SStreamMeta* pMeta) { TBC* pCur = NULL; if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { ASSERT(0); @@ -153,6 +221,18 @@ int32_t streamRestoreTask(SStreamMeta* pMeta) { tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecodeSStreamTask(&decoder, pTask); tDecoderClear(&decoder); + + if (pMeta->expandFunc(pMeta->ahandle, pTask) < 0) { + return -1; + } + + if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + return -1; + } + } + + if (tdbTbcClose(pCur) < 0) { + return -1; } return 0; diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index f82ef1b42fff735e92106eee97731a2c14b1d380..6819e5329fdcbac011ae258e89e2f665f85ebbca 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -35,9 +35,10 @@ FAIL: void streamQueueClose(SStreamQueue* queue) { while (1) { void* qItem = streamQueueNextItem(queue); - if (qItem) + if (qItem) { taosFreeQitem(qItem); - else + } else { return; + } } } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index dec23cd151fe687dafe44f53bb3653a4a1d2b75c..2a77ce8a91a0f6e416b32b1362f013238f99c82c 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -87,51 +87,80 @@ int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp return 0; } -int32_t streamProcessFailRecoverReq(SStreamTask* pTask, SMStreamTaskRecoverReq* pReq, SRpcMsg* pRsp) { - if (pTask->taskStatus != TASK_STATUS__FAIL) { - return 0; +typedef struct { + int32_t vgId; + int32_t childId; + int64_t ver; +} SStreamVgVerCheckpoint; + +int32_t tEncodeSStreamVgVerCheckpoint(SEncoder* pEncoder, const SStreamVgVerCheckpoint* pCheckpoint) { + if (tEncodeI32(pEncoder, pCheckpoint->vgId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->childId) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->ver) < 0) return -1; + return 0; +} + +int32_t tDecodeSStreamVgVerCheckpoint(SDecoder* pDecoder, SStreamVgVerCheckpoint* pCheckpoint) { + if (tDecodeI32(pDecoder, &pCheckpoint->vgId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->childId) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->ver) < 0) return -1; + return 0; +} + +typedef struct { + int64_t streamId; + int64_t checkTs; + int64_t checkpointId; + int32_t taskId; + SArray* checkpointVer; // SArray +} SStreamAggVerCheckpoint; + +int32_t tEncodeSStreamAggVerCheckpoint(SEncoder* pEncoder, const SStreamAggVerCheckpoint* pCheckpoint) { + if (tEncodeI64(pEncoder, pCheckpoint->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->checkTs) < 0) return -1; + if (tEncodeI64(pEncoder, pCheckpoint->checkpointId) < 0) return -1; + if (tEncodeI32(pEncoder, pCheckpoint->taskId) < 0) return -1; + int32_t sz = taosArrayGetSize(pCheckpoint->checkpointVer); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamVgVerCheckpoint* pOneVgCkpoint = taosArrayGet(pCheckpoint->checkpointVer, i); + if (tEncodeSStreamVgVerCheckpoint(pEncoder, pOneVgCkpoint) < 0) return -1; } + return 0; +} - if (pTask->isStreamDistributed) { - if (pTask->isDataScan) { - pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; - } else if (pTask->execType != TASK_EXEC__NONE) { - pTask->taskStatus = TASK_STATUS__PREPARE_RECOVER; - bool hasCheckpoint = false; - int32_t childSz = taosArrayGetSize(pTask->childEpInfo); - for (int32_t i = 0; i < childSz; i++) { - SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); - if (pEpInfo->checkpointVer == -1) { - hasCheckpoint = true; - break; - } - } - if (hasCheckpoint) { - // load from checkpoint - } else { - // recover child - } - } - } else { - if (pTask->isDataScan) { - if (pTask->checkpointVer != -1) { - // load from checkpoint - } else { - // reset stream query task info - // TODO get snapshot ver - pTask->recoverSnapVer = -1; - qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer); - pTask->taskStatus = TASK_STATUS__RECOVERING; - } - } +int32_t tDecodeSStreamAggVerCheckpoint(SDecoder* pDecoder, SStreamAggVerCheckpoint* pCheckpoint) { + if (tDecodeI64(pDecoder, &pCheckpoint->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->checkTs) < 0) return -1; + if (tDecodeI64(pDecoder, &pCheckpoint->checkpointId) < 0) return -1; + if (tDecodeI32(pDecoder, &pCheckpoint->taskId) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SStreamVgVerCheckpoint oneVgCheckpoint; + if (tDecodeSStreamVgVerCheckpoint(pDecoder, &oneVgCheckpoint) < 0) return -1; + taosArrayPush(pCheckpoint->checkpointVer, &oneVgCheckpoint); } + return 0; +} - if (pTask->taskStatus == TASK_STATUS__RECOVERING) { - if (streamPipelineExec(pTask, 100) < 0) { - // set fail - return -1; - } +int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) { + ASSERT(pTask->taskLevel == TASK_LEVEL__SINK); + // load status + void* pVal = NULL; + int32_t vLen = 0; + if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) { + return -1; } + SDecoder decoder; + tDecoderInit(&decoder, pVal, vLen); + SStreamAggVerCheckpoint aggCheckpoint; + tDecodeSStreamAggVerCheckpoint(&decoder, &aggCheckpoint); + /*pTask->*/ + return 0; +} +int32_t streamRecoverTask(SStreamTask* pTask) { + // return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 216e3fa761e910cee7749b4aabf0f4bc7b708b8b..8b5bd849f6627802d8b3098efe0c4288f9ecf323 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -23,7 +23,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { } pTask->taskId = tGenIdPI32(); pTask->streamId = streamId; - pTask->execStatus = TASK_EXEC_STATUS__IDLE; + pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; @@ -52,19 +52,17 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->schedStatus) < 0) return -1; if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - /*if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -73,27 +71,23 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1; } - if (pTask->execType != TASK_EXEC__NONE) { + if (pTask->taskLevel != TASK_LEVEL__SINK) { if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; } - if (pTask->sinkType == TASK_SINK__TABLE) { + if (pTask->outputType == TASK_OUTPUT__TABLE) { if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1; if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SMA) { + } else if (pTask->outputType == TASK_OUTPUT__SMA) { if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__FETCH) { + } else if (pTask->outputType == TASK_OUTPUT__FETCH) { if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1; - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } @@ -107,19 +101,17 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->schedStatus) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - /*if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; @@ -131,29 +123,25 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { taosArrayPush(pTask->childEpInfo, &pInfo); } - if (pTask->execType != TASK_EXEC__NONE) { + if (pTask->taskLevel != TASK_LEVEL__SINK) { if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; } - if (pTask->sinkType == TASK_SINK__TABLE) { + if (pTask->outputType == TASK_OUTPUT__TABLE) { if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1; pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); if (pTask->tbSink.pSchemaWrapper == NULL) return -1; if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__SMA) { + } else if (pTask->outputType == TASK_OUTPUT__SMA) { if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; - } else if (pTask->sinkType == TASK_SINK__FETCH) { + } else if (pTask->outputType == TASK_OUTPUT__FETCH) { if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1; - } else { - ASSERT(pTask->sinkType == TASK_SINK__NONE); - } - - if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { + } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->shuffleDispatcher.stbFullName) < 0) return -1; } diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index dd032f148177eaf9d4773fcd698cdffedc67887b..e6d2bd49202fe4d5caba72183dcc6b99673cd066 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -28,13 +28,13 @@ extern "C" { #include "trpc.h" #include "ttimer.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 5000 -#define ELECT_TIMER_MS_MIN 1300 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 5000 +#define ELECT_TIMER_MS_MIN 5000 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 900 +#define HEARTBEAT_TIMER_MS 900 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index bc3275a9714f5db4f7866b18153953a1881bf8cb..82399f52b93b7d87e5adfdda116c1d60e6535239 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -212,6 +212,7 @@ void syncNodeRelease(SSyncNode* pNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); +void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term); void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr); void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr); diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 65ec77e38ff10ff77de1d4000515439ad7844ef9..ff59189a9d6a96566e3246a5ed8fa25ca819e440 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -47,6 +47,8 @@ char* logStoreSimple2Str(SSyncLogStore* pLogStore); SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore); +SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore); + // for debug void logStorePrint(SSyncLogStore* pLogStore); void logStorePrint2(char* s, SSyncLogStore* pLogStore); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index a7bc4df281719f7ee50d672518829d18a9abf65c..4f93d8197dc801ae86619858b15d9055059565eb 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -357,16 +357,14 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "log truncate, from %" PRId64 " to %" PRId64, delBegin, delEnd); - syncNodeEventLog(ths, eventLog); - logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); - return code; } +// if FromIndex > walCommitVer, return 0 +// else return num of pass entries static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { - int32_t code; + int32_t code = 0; + int32_t pass = 0; SyncIndex delBegin = FromIndex; SyncIndex delEnd = ths->pLogStore->syncLogLastIndex(ths->pLogStore); @@ -398,16 +396,31 @@ static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { } } + // update delete begin + SyncIndex walCommitVer = logStoreWalCommitVer(ths->pLogStore); + + if (delBegin <= walCommitVer) { + delBegin = walCommitVer + 1; + pass = walCommitVer - delBegin + 1; + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "update delete begin to %ld", delBegin); + syncNodeEventLog(ths, logBuf); + } while (0); + } + // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "log truncate, from %" PRId64 " to %" PRId64, delBegin, delEnd); - syncNodeEventLog(ths, eventLog); - logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "make log same from:%ld, delbegin:%ld, pass:%d", FromIndex, delBegin, pass); + syncNodeEventLog(ths, logBuf); + } while (0); - return code; + return pass; } int32_t syncNodePreCommit(SSyncNode* ths, SSyncRaftEntry* pEntry, int32_t code) { @@ -543,31 +556,34 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SOffsetAndContLen* metaTableArr = syncAppendEntriesBatchMetaTableArray(pMsg); if (hasAppendEntries && pMsg->prevLogIndex == ths->commitIndex) { - // make log same - do { - SyncIndex logLastIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); - bool hasExtraEntries = logLastIndex > pMsg->prevLogIndex; - - if (hasExtraEntries) { - // make log same, rollback deleted entries - code = syncNodeDoMakeLogSame(ths, pMsg->prevLogIndex + 1); - ASSERT(code == 0); - } + int32_t pass = 0; + SyncIndex logLastIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); + bool hasExtraEntries = logLastIndex > pMsg->prevLogIndex; - } while (0); + // make log same + if (hasExtraEntries) { + // make log same, rollback deleted entries + pass = syncNodeDoMakeLogSame(ths, pMsg->prevLogIndex + 1); + ASSERT(pass >= 0); + } // append entry batch - for (int32_t i = 0; i < pMsg->dataCount; ++i) { - SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); - code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - if (code != 0) { - return -1; - } + if (pass == 0) { + // assert! no batch + ASSERT(pMsg->dataCount <= 1); + + for (int32_t i = 0; i < pMsg->dataCount; ++i) { + SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + if (code != 0) { + return -1; + } - code = syncNodePreCommit(ths, pAppendEntry, 0); - ASSERT(code == 0); + code = syncNodePreCommit(ths, pAppendEntry, 0); + ASSERT(code == 0); - // syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); + } } // fsync once @@ -670,25 +686,33 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc syncLogRecvAppendEntriesBatch(ths, pMsg, "really match"); + int32_t pass = 0; + if (hasExtraEntries) { // make log same, rollback deleted entries - code = syncNodeDoMakeLogSame(ths, pMsg->prevLogIndex + 1); - ASSERT(code == 0); + pass = syncNodeDoMakeLogSame(ths, pMsg->prevLogIndex + 1); + ASSERT(pass >= 0); } if (hasAppendEntries) { // append entry batch - for (int32_t i = 0; i < pMsg->dataCount; ++i) { - SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); - code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - if (code != 0) { - return -1; - } + if (pass == 0) { + // assert! no batch + ASSERT(pMsg->dataCount <= 1); + + // append entry batch + for (int32_t i = 0; i < pMsg->dataCount; ++i) { + SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + if (code != 0) { + return -1; + } - code = syncNodePreCommit(ths, pAppendEntry, 0); - ASSERT(code == 0); + code = syncNodePreCommit(ths, pAppendEntry, 0); + ASSERT(code == 0); - // syncEntryDestory(pAppendEntry); + // syncEntryDestory(pAppendEntry); + } } // fsync once @@ -717,24 +741,15 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc // maybe update commit index, leader notice me if (pMsg->commitIndex > ths->commitIndex) { - // has commit entry in local - if (pMsg->commitIndex <= ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { - // advance commit index to sanpshot first - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - if (snapshot.lastApplyIndex >= 0 && snapshot.lastApplyIndex > ths->commitIndex) { - SyncIndex commitBegin = ths->commitIndex; - SyncIndex commitEnd = snapshot.lastApplyIndex; - ths->commitIndex = snapshot.lastApplyIndex; + SyncIndex lastIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, - commitBegin, commitEnd); - syncNodeEventLog(ths, eventLog); - } + SyncIndex beginIndex = 0; + SyncIndex endIndex = -1; - SyncIndex beginIndex = ths->commitIndex + 1; - SyncIndex endIndex = pMsg->commitIndex; + // has commit entry in local + if (pMsg->commitIndex <= lastIndex) { + beginIndex = ths->commitIndex + 1; + endIndex = pMsg->commitIndex; // update commit index ths->commitIndex = pMsg->commitIndex; @@ -743,10 +758,22 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); ASSERT(code == 0); - code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + } else if (pMsg->commitIndex > lastIndex && ths->commitIndex < lastIndex) { + beginIndex = ths->commitIndex + 1; + endIndex = lastIndex; + + // update commit index, speed up + ths->commitIndex = lastIndex; + + // call back Wal + code = ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); ASSERT(code == 0); } + + code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); } + return 0; } } while (0); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index c18c2cc0d596082b256a2a0ee5a670d347068cf6..3a94ed9713ba2b12e2ce766b3dfd9e615b309d9f 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -73,7 +73,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { ASSERT(pEntry != NULL); // cannot commit, even if quorum agree. need check term! - if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { + if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { // update commit index newCommitIndex = index; @@ -92,6 +92,12 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } } + // advance commit index as large as possible + SyncIndex walCommitVer = logStoreWalCommitVer(pSyncNode->pLogStore); + if (walCommitVer > newCommitIndex) { + newCommitIndex = walCommitVer; + } + // maybe execute fsm if (newCommitIndex > pSyncNode->commitIndex) { SyncIndex beginIndex = pSyncNode->commitIndex + 1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f6f1f4e032fe143c5200f18a9aa587b075113830..3f9aa6c080e2e7f4e83b9cd7460abc9cf7629737 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -730,7 +730,8 @@ int32_t syncNodeProposeBatch(SSyncNode* pSyncNode, SRpcMsg** pMsgPArr, bool* pIs for (int i = 0; i < arrSize; ++i) { do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "propose message, type:%s batch:%d", TMSG_INFO(pMsgPArr[i]->msgType), arrSize); + snprintf(eventLog, sizeof(eventLog), "propose message, type:%s batch:%d", TMSG_INFO(pMsgPArr[i]->msgType), + arrSize); syncNodeEventLog(pSyncNode, eventLog); } while (0); @@ -834,7 +835,8 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { rpcFreeCont(rpcMsg.pCont); syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); ret = 1; - sDebug("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType)); + sDebug("vgId:%d, sync optimize index:%" PRId64 ", type:%s", pSyncNode->vgId, retIndex, + TMSG_INFO(pMsg->msgType)); } else { ret = -1; terrno = TSDB_CODE_SYN_INTERNAL_ERROR; @@ -1114,7 +1116,7 @@ void syncNodeStart(SSyncNode* pSyncNode) { } int32_t ret = 0; - ret = syncNodeStartPingTimer(pSyncNode); + // ret = syncNodeStartPingTimer(pSyncNode); ASSERT(ret == 0); } @@ -1250,6 +1252,13 @@ int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pElectTimer); atomic_store_64(&pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "elect timer reset, ms:%d", ms); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + } else { sError("vgId:%d, start elect timer error, sync env is stop", pSyncNode->vgId); } @@ -1261,6 +1270,8 @@ int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); taosTmrStop(pSyncNode->pElectTimer); pSyncNode->pElectTimer = NULL; + + sTrace("vgId:%d, sync %s stop elect timer", pSyncNode->vgId, syncUtilState2String(pSyncNode->state)); return ret; } @@ -1281,6 +1292,14 @@ int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); } ret = syncNodeRestartElectTimer(pSyncNode, electMS); + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "reset elect timer, min:%d, max:%d, ms:%d", pSyncNode->electBaseLine, + 2 * pSyncNode->electBaseLine, electMS); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + return ret; } @@ -1293,6 +1312,13 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode) { } else { sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); } + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "start heartbeat timer, ms:%d", pSyncNode->heartbeatTimerMS); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + return ret; } @@ -1304,6 +1330,13 @@ int32_t syncNodeStartNowHeartbeatTimer(SSyncNode* pSyncNode) { } else { sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId); } + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "start heartbeat timer, ms:%d", 1); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + return ret; } @@ -1312,6 +1345,9 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); taosTmrStop(pSyncNode->pHeartbeatTimer); pSyncNode->pHeartbeatTimer = NULL; + + sTrace("vgId:%d, sync %s stop heartbeat timer", pSyncNode->vgId, syncUtilState2String(pSyncNode->state)); + return ret; } @@ -1529,7 +1565,7 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } -void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { +inline void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; @@ -1559,12 +1595,13 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { ", sby:%d, " "stgy:%d, bch:%d, " "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", + "lcfg:%" PRId64 ", chging:%d, rsto:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, - pSyncNode->restoreFinish, printStr); + pSyncNode->restoreFinish, pSyncNode->electTimerLogicClockUser, pSyncNode->heartbeatTimerLogicClockUser, + printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1600,7 +1637,7 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { taosMemoryFree(pCfgStr); } -void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { +inline void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; @@ -1667,7 +1704,7 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { taosMemoryFree(pCfgStr); } -char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { +inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { int len = 256; char* s = (char*)taosMemoryMalloc(len); @@ -1690,7 +1727,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } -bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { +inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { bool b1 = false; bool b2 = false; @@ -1894,7 +1931,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // Raft 3.6.2 Committing entries from previous terms syncNodeAppendNoop(pSyncNode); -#if 0 // simon +#if 0 // simon syncNodeReplicate(pSyncNode); #endif syncMaybeAdvanceCommitIndex(pSyncNode); @@ -1953,6 +1990,12 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { } } +void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { + if (term > pSyncNode->pRaftStore->currentTerm) { + raftStoreSetTerm(pSyncNode->pRaftStore, term); + } +} + void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -2366,6 +2409,9 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { SSyncNode* pSyncNode = (SSyncNode*)param; + + syncNodeEventLog(pSyncNode, "eq hb timer"); + if (pSyncNode->replicaNum > 1) { if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) { @@ -2580,7 +2626,7 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p // fsync once SSyncLogStoreData* pData = ths->pLogStore->data; SWal* pWal = pData->pWal; - walFsync(pWal, true); + walFsync(pWal, false); if (ths->replicaNum > 1) { // if multi replica, start replicate right now @@ -2622,7 +2668,22 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); return 0; } - syncNodeEventLog(ths, "do leader transfer"); + + if (!ths->restoreFinish) { + syncNodeEventLog(ths, "restore not finish, can not do leader transfer"); + return 0; + } + + if (ths->vgId > 1) { + syncNodeEventLog(ths, "I am vnode, can not do leader transfer"); + return 0; + } + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); + syncNodeEventLog(ths, logBuf); + } while (0); bool sameId = syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId)); bool sameNodeInfo = strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && @@ -2763,11 +2824,28 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { } int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { + if (beginIndex > endIndex) { + return 0; + } + + // advance commit index to sanpshot first + SSnapshot snapshot = {0}; + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); + if (snapshot.lastApplyIndex >= 0 && snapshot.lastApplyIndex >= beginIndex) { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, beginIndex, + snapshot.lastApplyIndex); + syncNodeEventLog(ths, eventLog); + + // update begin index + beginIndex = snapshot.lastApplyIndex + 1; + } + int32_t code = 0; ESyncState state = flag; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit wal from index:%" PRId64 " to index:%" PRId64, beginIndex, endIndex); + snprintf(eventLog, sizeof(eventLog), "commit by wal from index:%" PRId64 " to index:%" PRId64, beginIndex, endIndex); syncNodeEventLog(ths, eventLog); // execute fsm @@ -3006,7 +3084,7 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncNodeEventLog(pSyncNode, logBuf); } -void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { + void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index bf440f04a06f7c26dfcae36a110a123c39c5bf49..0649e064e45391cfe9082c24264a33b762d1a279 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -237,51 +237,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr return 0; } -#if 0 -static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { - SSyncLogStoreData* pData = pLogStore->data; - SWal* pWal = pData->pWal; - - SyncIndex writeIndex = raftLogWriteIndex(pLogStore); - if (pEntry->index != writeIndex) { - sError("vgId:%d, wal write index error, entry-index:%" PRId64 " update to %" PRId64, pData->pSyncNode->vgId, - pEntry->index, writeIndex); - pEntry->index = writeIndex; - } - - int code = 0; - SWalSyncInfo syncMeta; - syncMeta.isWeek = pEntry->isWeak; - syncMeta.seqNum = pEntry->seqNum; - syncMeta.term = pEntry->term; - code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - if (code != 0) { - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); - - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pEntry->index, err, err, errStr, sysErr, sysErrStr); - syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); - } - - // walFsync(pWal, true); - - do { - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); - syncNodeEventLog(pData->pSyncNode, eventLog); - } while (0); - - return code; -} -#endif - // entry found, return 0 // entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST // other error, return -1 @@ -350,10 +305,18 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, return code; } +// truncate semantic static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - int32_t code = walRollback(pWal, fromIndex); + + // need not truncate + SyncIndex wallastVer = walGetLastVer(pWal); + if (fromIndex > wallastVer) { + return 0; + } + + int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); @@ -368,7 +331,7 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn // event log do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%" PRId64, fromIndex); + snprintf(logBuf, sizeof(logBuf), "log truncate, from-index:%" PRId64, fromIndex); syncNodeEventLog(pData->pSyncNode, logBuf); } while (0); @@ -400,45 +363,6 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp //------------------------------- // log[0 .. n] -#if 0 -int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { - SSyncLogStoreData* pData = pLogStore->data; - SWal* pWal = pData->pWal; - - SyncIndex lastIndex = logStoreLastIndex(pLogStore); - ASSERT(pEntry->index == lastIndex + 1); - - int code = 0; - SWalSyncInfo syncMeta; - syncMeta.isWeek = pEntry->isWeak; - syncMeta.seqNum = pEntry->seqNum; - syncMeta.term = pEntry->term; - code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen); - if (code != 0) { - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); - - char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", - pEntry->index, err, err, errStr, sysErr, sysErrStr); - syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); - } - - // walFsync(pWal, true); - - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "old write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); - syncNodeEventLog(pData->pSyncNode, eventLog); - - return code; -} -#endif - int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -721,6 +645,12 @@ SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { return walGetFirstVer(pWal); } +SyncIndex logStoreWalCommitVer(SSyncLogStore* pLogStore) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + return walGetCommittedVer(pWal); +} + // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 12d6797349b55b6e75d89dc74bbac5939f714dc1..bc703e519cf9a0afbbb2788023a3cc8ea2338af1 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -140,8 +140,6 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { sError("vgId:%d, sync get pre term error, nextIndex:%" PRId64 ", update next-index:%" PRId64 ", match-index:%d, raftid:%" PRId64, pSyncNode->vgId, nextIndex, newNextIndex, SYNC_INDEX_INVALID, pDestId->addr); - - syncNodeRestartNowHeartbeatTimer(pSyncNode); return -1; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index bad32c5f911a5e0bf70aee4cbcda568590d5c36f..122a81930bec953f167030873ae2ed48bdafc555 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -51,15 +51,23 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return -1; } + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); +#if 0 + if (logOK) { + syncNodeUpdateTerm(ths, pMsg->term); + } else { + syncNodeUpdateTermWithoutStepDown(ths, pMsg->term); + } +#endif } ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); - bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || - ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && - (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); if (grant) { @@ -94,48 +102,6 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } -#if 0 -int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { - int32_t ret = 0; - - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%" PRIu64, ths->pRaftStore->currentTerm); - syncRequestVoteLog2(logBuf, pMsg); - - if (pMsg->term > ths->pRaftStore->currentTerm) { - syncNodeUpdateTerm(ths, pMsg->term); - } - ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); - - bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || - ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && - (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); - bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && - ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); - if (grant) { - // maybe has already voted for pMsg->srcId - // vote again, no harm - raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); - - // forbid elect for this round - syncNodeResetElectTimer(ths); - } - - SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); - pReply->srcId = ths->myRaftId; - pReply->destId = pMsg->srcId; - pReply->term = ths->pRaftStore->currentTerm; - pReply->voteGranted = grant; - - SRpcMsg rpcMsg; - syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); - syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); - syncRequestVoteReplyDestroy(pReply); - - return ret; -} -#endif - static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); @@ -200,13 +166,21 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { return -1; } + bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); + // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); +#if 0 + if (logOK) { + syncNodeUpdateTerm(ths, pMsg->term); + } else { + syncNodeUpdateTermWithoutStepDown(ths, pMsg->term); + } +#endif } ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); - bool logOK = syncNodeOnRequestVoteLogOK(ths, pMsg); bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); if (grant) { diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 566b80881f9786426f5aa62e6c44504f92db174e..55553d50485358a79adfadc625dd73ca0c05c251 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -93,65 +93,6 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return 0; } -#if 0 -int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { - int32_t ret = 0; - - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%" PRIu64, ths->pRaftStore->currentTerm); - syncRequestVoteReplyLog2(logBuf, pMsg); - - if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("DropStaleResponse, receive term:%" PRIu64 ", current term:%" PRIu64 "", pMsg->term, - ths->pRaftStore->currentTerm); - return ret; - } - - // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); - // no need this code, because if I receive reply.term, then I must have sent for that term. - // if (pMsg->term > ths->pRaftStore->currentTerm) { - // syncNodeUpdateTerm(ths, pMsg->term); - // } - - if (pMsg->term > ths->pRaftStore->currentTerm) { - char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%" PRIu64 " current:%" PRIu64, pMsg->term, - ths->pRaftStore->currentTerm); - syncNodePrint2(logBuf, ths); - sError("%s", logBuf); - return ret; - } - - ASSERT(pMsg->term == ths->pRaftStore->currentTerm); - - // This tallies votes even when the current state is not Candidate, - // but they won't be looked at, so it doesn't matter. - if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { - votesRespondAdd(ths->pVotesRespond, pMsg); - if (pMsg->voteGranted) { - // add vote - voteGrantedVote(ths->pVotesGranted, pMsg); - - // maybe to leader - if (voteGrantedMajority(ths->pVotesGranted)) { - if (!ths->pVotesGranted->toLeader) { - syncNodeCandidate2Leader(ths); - - // prevent to leader again! - ths->pVotesGranted->toLeader = true; - } - } - } else { - ; - // do nothing - // UNCHANGED <> - } - } - - return ret; -} -#endif - int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; @@ -184,6 +125,14 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl // This tallies votes even when the current state is not Candidate, // but they won't be looked at, so it doesn't matter. if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + if (ths->pVotesRespond->term != pMsg->term) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "vote respond error vote-respond-mgr term:%lu, msg term:lu", + ths->pVotesRespond->term, pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; + } + votesRespondAdd(ths->pVotesRespond, pMsg); if (pMsg->voteGranted) { // add vote diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 4ae803d744b3547b0b7aa534cbf63992fb0265a8..15b1054e635e8043ea73d08bd501164d2304ccdb 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -48,14 +48,16 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) { } else if (pMsg->timeoutType == SYNC_TIMEOUT_ELECTION) { if (atomic_load_64(&ths->electTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->electTimerCounter); - sInfo("vgId:%d, sync timeout, type:election count:%d", ths->vgId, ths->electTimerCounter); + sInfo("vgId:%d, sync timeout, type:election count:%d, electTimerLogicClockUser:%ld", ths->vgId, + ths->electTimerCounter, ths->electTimerLogicClockUser); syncNodeElect(ths); } } else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) { if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) { ++(ths->heartbeatTimerCounter); - sInfo("vgId:%d, sync timeout, type:replicate count:%d", ths->vgId, ths->heartbeatTimerCounter); + sInfo("vgId:%d, sync timeout, type:replicate count:%d, heartbeatTimerLogicClockUser:%ld", ths->vgId, + ths->heartbeatTimerCounter, ths->heartbeatTimerLogicClockUser); syncNodeReplicate(ths); } } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 5b73d980c4088966cc56ef0d58e2a097920bbee5..15e94baee4f4d7ff32d2d956cc04f9d2ca3e240f 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -125,7 +125,10 @@ int32_t syncUtilRand(int32_t max) { return taosRand() % max; } int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { ASSERT(min > 0 && max > 0 && max >= min); - return min + syncUtilRand(max - min); + int32_t rdm = min + syncUtilRand(max - min); + + // sDebug("random min:%d, max:%d, rdm:%d", min, max, rdm); + return rdm; } int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index 751b42b9c22077d21cbc694392f1b0bab3a0f7d7..f4ffaa50621b0f7c0c859e03a8e1f32c7442b5c4 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -81,4 +81,16 @@ for file in `ls ${logpath}/log.dnode*vgId*`;do done +echo "" +echo "generate log.commit ..." +tmpfile=${logpath}/log.commits.tmp +touch ${tmpfile} +for file in `ls ${logpath}/log.dnode*.vgId*.commit`;do + line=`cat ${file} | tail -n1` + echo $line | awk '{print $5, $0}' >> ${tmpfile} +done +cat ${tmpfile} | sort -k1 | awk 'BEGIN{vgid=$1}{if($1==vgid){print $0}else{print ""; print $0; vgid=$1;}}END{}' > ${logpath}/log.commits + exit 0 + + diff --git a/source/libs/sync/test/sh/insert.tpl.json b/source/libs/sync/test/sh/insert.tpl.json index 1d952b98e8dd855990d57fcf790a97db4281c1d5..631e490a2a0f6125e13182ca83dc2db2846f45f9 100644 --- a/source/libs/sync/test/sh/insert.tpl.json +++ b/source/libs/sync/test/sh/insert.tpl.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./tpl_insert_result_tpl", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 58563254e840c7148800fef394b60cb5f01d9b63..7a44edb12cddf5a386e3b77031920559d8b0a5e9 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -176,6 +176,8 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in tdbBtcOpen(&btc, pBt, pTxn); + tdbTrace("tdb insert, btc: %p, pTxn: %p", &btc, pTxn); + // move to the position to insert ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { @@ -214,6 +216,8 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { tdbBtcOpen(&btc, pBt, pTxn); + tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn); + // move the cursor ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { @@ -244,6 +248,8 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i tdbBtcOpen(&btc, pBt, pTxn); + tdbTrace("tdb upsert, btc: %p, pTxn: %p", &btc, pTxn); + // move the cursor ret = tdbBtcMoveTo(&btc, pKey, nKey, &c); if (ret < 0) { @@ -283,10 +289,12 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL int ret; void *pTKey = NULL; void *pTVal = NULL; - SCellDecoder cd; + SCellDecoder cd = {0}; tdbBtcOpen(&btc, pBt, NULL); + tdbTrace("tdb pget, btc: %p", &btc); + ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (ret < 0) { tdbBtcClose(&btc); @@ -295,6 +303,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL if (btc.idx < 0 || cret) { tdbBtcClose(&btc); + return -1; } @@ -330,9 +339,13 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbDebug("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbFree(cd.pVal); } + tdbTrace("tdb pget end, btc decoder: %p/0x%x, local decoder:%p", &btc.coder, btc.coder.freeKV, &cd); + tdbBtcClose(&btc); return 0; @@ -722,7 +735,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx int szCell; SBtreeInitPageArg iarg; int iNew, nNewCells; - SCellDecoder cd; + SCellDecoder cd = {0}; iarg.pBt = pBt; iarg.flags = TDB_BTREE_PAGE_GET_FLAGS(pOlds[0]); @@ -1235,6 +1248,8 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, } TDB_CELLDECODER_SET_FREE_VAL(pDecoder); + tdbDebug("tdb btc decoder: %p/0x%x pVal: %p ", pDecoder, pDecoder->freeKV, pDecoder->pVal); + memcpy(pDecoder->pVal, pCell + nHeader + kLen, nLocal - kLen - sizeof(SPgno)); nLeft -= nLocal - kLen - sizeof(SPgno); @@ -1376,6 +1391,9 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); // Clear the state of decoder + if (TDB_CELLDECODER_FREE_VAL(pDecoder)) { + tdbFree(pDecoder->pVal); + } pDecoder->kLen = -1; pDecoder->pKey = NULL; pDecoder->vLen = -1; @@ -1383,6 +1401,8 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD pDecoder->pgno = 0; TDB_CELLDECODER_SET_FREE_NIL(pDecoder); + tdbDebug("tdb btc decoder set nil: %p/0x%x ", pDecoder, pDecoder->freeKV); + // 1. Decode header part if (!leaf) { ASSERT(pPage->vLen == sizeof(SPgno)); @@ -1650,7 +1670,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { SCell *pCell; - SCellDecoder cd; + SCellDecoder cd = {0}; void *pKey, *pVal; int ret; @@ -1696,7 +1716,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { SCell *pCell; - SCellDecoder cd; + SCellDecoder cd = {0}; void *pKey, *pVal; int ret; @@ -2037,7 +2057,7 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { const void *pTKey; int tkLen; - tdbTrace("ttl moveto, pager:%p, ipage:%d", pPager, pBtc->iPage); + tdbTrace("tdb moveto, pager:%p, ipage:%d", pPager, pBtc->iPage); if (pBtc->iPage < 0) { // move from a clear cursor ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, @@ -2093,6 +2113,7 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { } // search downward to the leaf + tdbTrace("tdb search downward, pager:%p, ipage:%d", pPager, pBtc->iPage); for (;;) { int lidx, ridx; SPage *pPage; @@ -2127,6 +2148,7 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { } // binary search + tdbTrace("tdb binary search, pager:%p, ipage:%d", pPager, pBtc->iPage); for (;;) { if (lidx > ridx) break; @@ -2157,6 +2179,8 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { } } + tdbTrace("tdb moveto end, pager:%p, ipage:%d", pPager, pBtc->iPage); + return 0; } @@ -2175,6 +2199,12 @@ int tdbBtcClose(SBTC *pBtc) { pBtc->idx = pBtc->idxStack[pBtc->iPage]; } + if (TDB_CELLDECODER_FREE_VAL(&pBtc->coder)) { + tdbDebug("tdb btc/close decoder: %p pVal free: %p", &pBtc->coder, pBtc->coder.pVal); + + tdbFree(pBtc->coder.pVal); + } + return 0; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index a81d6db80fb9d40d4847878c19483ebe3d254b38..ad0ce4f5e1c03ce03b6599f000d3e9c94a01157c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -94,9 +94,10 @@ typedef void* queue[2]; /* Return the structure holding the given element. */ #define QUEUE_DATA(e, type, field) ((type*)((void*)((char*)(e)-offsetof(type, field)))) -#define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit -#define TRANS_RETRY_INTERVAL 15 // ms retry interval -#define TRANS_CONN_TIMEOUT 3 // connect timeout +#define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit +#define TRANS_RETRY_INTERVAL 15 // retry interval (ms) +#define TRANS_CONN_TIMEOUT 3 // connect timeout (s) +#define TRANS_READ_TIMEOUT 3000 // read timeout (ms) typedef SRpcMsg STransMsg; typedef SRpcCtx STransCtx; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 5fb2980cebc2b45a5f83c29c2403a4a15c841db2..6aeeffa1925038050f14b25f2d422177a072c95a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -53,6 +53,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); + bool (*startTimer)(int32_t code, tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/.transSvr.c.swo b/source/libs/transport/src/.transSvr.c.swo deleted file mode 100644 index c9486c50867994926ea395577110ee4d15de5632..0000000000000000000000000000000000000000 Binary files a/source/libs/transport/src/.transSvr.c.swo and /dev/null differ diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 7633820292ae2442f78c85537fb5c46b4caabea9..0a0dcef378bde92a18b9455b203774a3c28aa428 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -48,6 +48,7 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; + pRpc->startTimer = pInit->tfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7ab3102d1caf7a529a9be173c4858c6f829124e8..ce9488d20795a3ccb1127b1f98702ce79a8ac584 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -24,6 +24,7 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; queue wreqQueue; + uv_timer_t* timer; void* hostThrd; @@ -65,12 +66,13 @@ typedef struct SCliThrd { int64_t pid; // pid uv_loop_t* loop; SAsyncPool* asyncPool; - uv_idle_t* idle; uv_prepare_t* prepare; - uv_timer_t timer; void* pool; // conn pool + SArray* timerList; + // msg queue + queue msg; TdThreadMutex msgMtx; SDelayQueue* delayQueue; @@ -108,6 +110,8 @@ static int sockDebugInfo(struct sockaddr* sockname, char* dst) { sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); return r; } +// register timer for read +static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn // static void cliTimeoutCb(uv_timer_t* handle); // alloc buf for recv @@ -330,6 +334,16 @@ void cliHandleResp(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; + if (conn->timer) { + if (uv_is_active((uv_handle_t*)conn->timer)) { + tDebug("%s conn %p stop timer", CONN_GET_INST_LABEL(conn), conn); + uv_timer_stop(conn->timer); + } + conn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer = NULL; + } + STransMsgHead* pHead = NULL; transDumpFromBuffer(&conn->readBuf, (char**)&pHead); pHead->code = htonl(pHead->code); @@ -409,7 +423,7 @@ void cliHandleResp(SCliConn* conn) { uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); } -void cliHandleExcept(SCliConn* pConn) { +void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); @@ -428,7 +442,7 @@ void cliHandleExcept(SCliConn* pConn) { STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg transMsg = {0}; - transMsg.code = pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL; + transMsg.code = code == -1 ? (pConn->broken ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL) : code; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; @@ -459,31 +473,18 @@ void cliHandleExcept(SCliConn* pConn) { } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } +void cliHandleExcept(SCliConn* conn) { + tTrace("%s conn %p except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + cliHandleExceptImpl(conn, -1); +} -// void cliTimeoutCb(uv_timer_t* handle) { -// SCliThrd* pThrd = handle->data; -// STrans* pTransInst = pThrd->pTransInst; -// int64_t currentTime = pThrd->nextTimeout; -// tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); -// -// SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); -// while (p != NULL) { -// while (!QUEUE_IS_EMPTY(&p->conn)) { -// queue* h = QUEUE_HEAD(&p->conn); -// SCliConn* c = QUEUE_DATA(h, SCliConn, q); -// if (c->expireTime < currentTime) { -// QUEUE_REMOVE(h); -// transUnrefCliHandle(c); -// } else { -// break; -// } -// } -// p = taosHashIterate((SHashObj*)pThrd->pool, p); -// } -// -// pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); -// uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0); -// } +void cliReadTimeoutCb(uv_timer_t* handle) { + // set up timeout cb + SCliConn* conn = handle->data; + tTrace("%s conn %p timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_read_stop(conn->stream); + cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); +} void* createConnPool(int size) { // thread local, no lock @@ -654,13 +655,23 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { + SCliThrd* pThrd = conn->hostThrd; tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); transRemoveExHandle(transGetRefMgt(), conn->refId); - conn->refId = -1; - if (conn->task != NULL) transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); + + if (conn->task != NULL) { + transDQCancel(pThrd->timeoutQueue, conn->task); + conn->task = NULL; + } + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; + conn->timer = NULL; + } if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { @@ -673,8 +684,15 @@ static void cliDestroy(uv_handle_t* handle) { if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) { return; } - SCliConn* conn = handle->data; + SCliThrd* pThrd = conn->hostThrd; + if (conn->timer != NULL) { + uv_timer_stop(conn->timer); + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer->data = NULL; + conn->timer = NULL; + } + transRemoveExHandle(transGetRefMgt(), conn->refId); taosMemoryFree(conn->ip); conn->stream->data = NULL; @@ -764,6 +782,19 @@ void cliSend(SCliConn* pConn) { CONN_SET_PERSIST_BY_APP(pConn); } + if (pTransInst->startTimer != NULL && pTransInst->startTimer(0, pMsg->msgType)) { + uv_timer_t* timer = taosArrayGetSize(pThrd->timerList) > 0 ? *(uv_timer_t**)taosArrayPop(pThrd->timerList) : NULL; + if (timer == NULL) { + tDebug("no avaiable timer, create"); + timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, timer); + } + timer->data = pConn; + pConn->timer = timer; + + tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); + uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); + } uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; @@ -781,8 +812,8 @@ void cliConnCb(uv_connect_t* req, int status) { } // int addrlen = sizeof(pConn->addr); struct sockaddr peername, sockname; - int addrlen = sizeof(peername); + int addrlen = sizeof(peername); uv_tcp_getpeername((uv_tcp_t*)pConn->stream, &peername, &addrlen); transGetSockDebugInfo(&peername, pConn->dst); @@ -806,7 +837,6 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); - uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -879,8 +909,8 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { - STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; + STransConnCtx* pCtx = pMsg->ctx; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); if (!EPSET_IS_VALID(&pCtx->epSet)) { @@ -964,37 +994,9 @@ static void cliAsyncCb(uv_async_t* handle) { if (count >= 2) { tTrace("cli process batch size:%d", count); } - if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); -} -static void cliIdleCb(uv_idle_t* handle) { - SCliThrd* thrd = handle->data; - tTrace("do idle work"); + // if (!uv_is_active((uv_handle_t*)pThrd->prepare)) uv_prepare_start(pThrd->prepare, cliPrepareCb); - SAsyncPool* pool = thrd->asyncPool; - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - - queue wq; - taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - - SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - if (pMsg == NULL) { - continue; - } - (*cliAsyncHandle[pMsg->type])(pMsg, thrd); - count++; - } - } - tTrace("prepare work end"); - if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); + if (pThrd->stopMsg != NULL) cliHandleQuit(pThrd->stopMsg, pThrd); } static void cliPrepareCb(uv_prepare_t* handle) { SCliThrd* thrd = handle->data; @@ -1085,18 +1087,19 @@ static SCliThrd* createThrdObj() { uv_loop_init(pThrd->loop); pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 5, pThrd, cliAsyncCb); - uv_timer_init(pThrd->loop, &pThrd->timer); - pThrd->timer.data = pThrd; - - // pThrd->idle = taosMemoryCalloc(1, sizeof(uv_idle_t)); - // uv_idle_init(pThrd->loop, pThrd->idle); - // pThrd->idle->data = pThrd; - // uv_idle_start(pThrd->idle, cliIdleCb); pThrd->prepare = taosMemoryCalloc(1, sizeof(uv_prepare_t)); uv_prepare_init(pThrd->loop, pThrd->prepare); pThrd->prepare->data = pThrd; - uv_prepare_start(pThrd->prepare, cliPrepareCb); + // uv_prepare_start(pThrd->prepare, cliPrepareCb); + + int32_t timerSize = 512; + pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); + for (int i = 0; i < timerSize; i++) { + uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, timer); + taosArrayPush(pThrd->timerList, &timer); + } pThrd->pool = createConnPool(4); transDQCreate(pThrd->loop, &pThrd->delayQueue); @@ -1120,7 +1123,11 @@ static void destroyThrdObj(SCliThrd* pThrd) { transDQDestroy(pThrd->delayQueue, destroyCmsg); transDQDestroy(pThrd->timeoutQueue, NULL); - taosMemoryFree(pThrd->idle); + for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { + uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); + taosMemoryFree(timer); + } + taosArrayDestroy(pThrd->timerList); taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 3aebb1c6ba1174a26516abe7c52298bb61642639..e4b27292bb7b084a4b546111d026cfbf91cc823e 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -151,8 +151,8 @@ int walMetaDeserialize(SWal* pWal, const char* bytes); // meta section end // seek section -int walChangeWrite(SWal* pWal, int64_t ver); -int walInitWriteFile(SWal* pWal); +int64_t walChangeWrite(SWal* pWal, int64_t ver); +int walInitWriteFile(SWal* pWal); // seek section end int64_t walGetSeq(); diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 87ab155065021cb902739ef18c8925afc04c2f07..1196914daea766159f20b507d2bfb908d6c309e0 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -74,7 +74,7 @@ int walInitWriteFile(SWal* pWal) { return 0; } -int walChangeWrite(SWal* pWal, int64_t ver) { +int64_t walChangeWrite(SWal* pWal, int64_t ver) { int code; TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index eaf43ba7d77531cf4f0b7cc1a62e4497fd6de36e..ad571a9e822cb4363ad13db7c1b8d60b2cc3272d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -208,6 +208,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); + taosFsyncFile(pWal->pLogFile); + taosFsyncFile(pWal->pIdxFile); + walSaveMeta(pWal); // unlock diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index d1d768fbcc5ecb21e9287c1f327843020e0f379c..a9eb23abba4066bad4c54322fa0b65aa63458f71 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -193,7 +193,13 @@ void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) { } int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, -val); } -int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { return _InterlockedExchangeAdd64(ptr, -val); } +int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + return _InterlockedExchangeAdd((int32_t volatile*)ptr, -(int32_t)val); +#else + return _InterlockedExchangeAdd64(ptr, -val); +#endif +} void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) { #ifdef WINDOWS @@ -375,7 +381,11 @@ int32_t atomic_exchange_32(int32_t volatile* ptr, int32_t val) { int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { #ifdef WINDOWS +#ifdef _TD_WINDOWS_32 + return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); +#else return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif #elif defined(_TD_NINGSI_60) return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); #else @@ -529,7 +539,11 @@ int32_t atomic_fetch_add_32(int32_t volatile* ptr, int32_t val) { int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { #ifdef WINDOWS +#ifdef _TD_WINDOWS_32 + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); +#else return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); #else @@ -631,7 +645,11 @@ int32_t atomic_fetch_sub_32(int32_t volatile* ptr, int32_t val) { int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { #ifdef WINDOWS +#ifdef _TD_WINDOWS_32 + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#else return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); +#endif #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); #else diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 98dfaa4972b001ed9a30369e7390222e637a60fc..f34032056cba6a2e2160e7b7a54abe79a959d301 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -333,7 +333,7 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nwritten; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; @@ -362,7 +362,7 @@ int32_t taosReadMsg(TdSocketPtr pSocket, void *buf, int32_t nbytes) { return -1; } int32_t nleft, nread; - char * ptr = (char *)buf; + char *ptr = (char *)buf; nleft = nbytes; @@ -912,7 +912,7 @@ uint32_t taosGetIpv4FromFqdn(const char *fqdn) { int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); if (result) { - struct sockaddr * sa = result->ai_addr; + struct sockaddr *sa = result->ai_addr; struct sockaddr_in *si = (struct sockaddr_in *)sa; struct in_addr ia = si->sin_addr; uint32_t ip = ia.s_addr; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 26aafd743fb7ad8a89e0f73674d25eb98a884147..db3aaa49a6351e4b6a26a4a7b601f4e267c4f4c7 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -48,7 +48,7 @@ char *strsep(char **stringp, const char *delim) { /* NOTREACHED */ } /* Duplicate a string, up to at most size characters */ -char *strndup(const char *s, size_t size) { +char *strndup(const char *s, int size) { size_t l; char * s2; l = strlen(s); @@ -62,7 +62,7 @@ char *strndup(const char *s, size_t size) { } /* Copy no more than N characters of SRC to DEST, returning the address of the terminating '\0' in DEST, if any, or else DEST + N. */ -char *stpncpy(char *dest, const char *src, size_t n) { +char *stpncpy(char *dest, const char *src, int n) { size_t size = strnlen(src, n); memcpy(dest, src, size); dest += size; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e4b6983dcb7fae180f00808b19260849db2d9df5..4780d85a303fb9609f8ab10cecad42affd8c67fe 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -52,6 +52,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish c TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_BROKEN_LINK, "Conn is broken") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 8275072748123a4f4fa28b0e8401b967b0d2b099..aee84a0d55336c63840d1a5df887da7752592841 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -832,8 +832,9 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pNode) { SHashEntry *pe = pHashObj->hashList[slot]; - uint16_t prevRef = atomic_load_16(&pNode->refCount); + /*uint16_t prevRef = atomic_load_16(&pNode->refCount);*/ uint16_t afterRef = atomic_add_fetch_16(&pNode->refCount, 1); +#if 0 ASSERT(prevRef < afterRef); // the reference count value is overflow, which will cause the delete node operation immediately. @@ -845,6 +846,8 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { } else { data = GET_HASH_NODE_DATA(pNode); } +#endif + data = GET_HASH_NODE_DATA(pNode); if (afterRef >= MAX_WARNING_REF_COUNT) { uWarn("hash entry ref count is abnormally high: %d", afterRef); diff --git a/tests/docs-examples-test/go.sh b/tests/docs-examples-test/go.sh new file mode 100644 index 0000000000000000000000000000000000000000..185661e8a7d223485ca8a595996a7dbdb617d257 --- /dev/null +++ b/tests/docs-examples-test/go.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +taosd >>/dev/null 2>&1 & +taosadapter >>/dev/null 2>&1 & + +cd ../../docs/examples/go + +go mod tidy + +go run ./connect/afconn/main.go +go run ./connect/cgoexample/main.go +go run ./connect/restexample/main.go + +taos -s "drop database if exists test" +go run ./insert/json/main.go +taos -s "drop database if exists test" +go run ./insert/line/main.go +taos -s "drop database if exists power" +go run ./insert/sql/main.go +taos -s "drop database if exists power" +go run ./insert/stmt/main.go +taos -s "drop database if exists test" +go run ./insert/telnet/main.go + +go run ./query/sync/main.go + +taos -s "drop topic if exists example_tmq_topic" +taos -s "drop database if exists example_tmq" +go run ./sub/main.go diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 52a5627e342c1d737982e54000e3ac019df04316..eda66a884a8e7a75de6c1c9130298b72c46fa3c5 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -49,11 +49,13 @@ if [ $ent -eq 0 ]; then export PATH=$PATH:/home/TDengine/debug/build/bin export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDengine else export PATH=$PATH:/home/TDinternal/debug/build/bin export LD_LIBRARY_PATH=/home/TDinternal/debug/build/lib ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDinternal/community fi mkdir -p /var/lib/taos/subscribe diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index affd9128a46bed4cad123d72fcd09e95d5d2bf0f..f0ee9be46fa5c3f399cde738cad29aa3f03ea7b8 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -100,6 +100,7 @@ docker run \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ + -v $WORKDIR/taos-connector-python/taosrest:/usr/local/lib/python3.8/site-packages/taosrest:ro \ --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $extra_param ret=$? exit $ret diff --git a/tests/pytest/cluster/TD-3693/insert1Data.json b/tests/pytest/cluster/TD-3693/insert1Data.json index 6900ce0366971a71a0e119f0b7cfc363f78cd656..ad83a3516042dab92164dc887dd4c7adadecc1b8 100644 --- a/tests/pytest/cluster/TD-3693/insert1Data.json +++ b/tests/pytest/cluster/TD-3693/insert1Data.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/cluster/TD-3693/insert2Data.json b/tests/pytest/cluster/TD-3693/insert2Data.json index e55fa996fb5099ba7d0702172671bb489ec28213..86495f0ce982bb3aab2321b56fa9ca611c405a93 100644 --- a/tests/pytest/cluster/TD-3693/insert2Data.json +++ b/tests/pytest/cluster/TD-3693/insert2Data.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 7ab09383bfd0791aad30b39fa7e65ed66df9d936..2fa99230bc99a1e3641c2e537dbd0d75c908d808 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -1339,8 +1339,9 @@ class Task(): 0x03A1, # STable [does] not exist 0x03AA, # Tag already exists 0x0603, # Table already exists - 0x2603, # Table does not exist + 0x2603, # Table does not exist, replaced by 2662 below 0x260d, # Tags number not matched + 0x2662, # Table does not exist #TODO: what about 2603 above? diff --git a/tests/pytest/dockerCluster/insert.json b/tests/pytest/dockerCluster/insert.json index 32e1043c4e722c379d2256ed6bb7d7a11bd7a8da..ce8d7978fa7abfc1ea39ade8852e5eea7d1b254f 100644 --- a/tests/pytest/dockerCluster/insert.json +++ b/tests/pytest/dockerCluster/insert.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "databases": [{ diff --git a/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json index dc9de1626a4da72ad0dda91a3b42191ff27b165b..4b622c3f28ad41693739e55413f6d5c84a3f8cc6 100644 --- a/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json +++ b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index 7f2dfad40338e0fd710e908e8ccce940c128d4dc..3e7de39bed86c82c1c6143c82a7c8bb1cd1c5ccb 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -185,7 +185,7 @@ class TDTestCase: "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/joinPerformance.py b/tests/pytest/perfbenchmark/joinPerformance.py index b85c09926a5c93dae1edca645e02ae223569a933..d30bec6664167b9b52ad9499212770e28ff93ec3 100644 --- a/tests/pytest/perfbenchmark/joinPerformance.py +++ b/tests/pytest/perfbenchmark/joinPerformance.py @@ -168,7 +168,7 @@ class JoinPerf: "user": self.user, "password": self.password, "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py index 774103aa853183b923edc0a4157f650a08d1eb76..a23797a62b87e9e045a08ef923969834ddee88f2 100644 --- a/tests/pytest/perfbenchmark/taosdemoInsert.py +++ b/tests/pytest/perfbenchmark/taosdemoInsert.py @@ -172,7 +172,7 @@ class Taosdemo: "user": self.user, "password": self.password, "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/query/nestedQuery/insertData.json b/tests/pytest/query/nestedQuery/insertData.json index 1aad170bb0d2f1a986d5ed7aac20b53f6456a794..18a843015c4f4fdad9cb748b15f0f04bf83517cd 100644 --- a/tests/pytest/query/nestedQuery/insertData.json +++ b/tests/pytest/query/nestedQuery/insertData.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 6a5c0796ed1eb766519f4ff0f31d9b7c94f4a49a..e7e9fa5329f1489f51ddd4d20b6f8dede3940305 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -133,7 +133,7 @@ class TDTestCase: "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-interlace.json b/tests/pytest/tools/insert-interlace.json index 0e17edf8fdc90379c93a08b861417c4fd5411d49..8d96c20fe7a86d0d07c248ea284334a9152899be 100644 --- a/tests/pytest/tools/insert-interlace.json +++ b/tests/pytest/tools/insert-interlace.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 5000, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json index bbac60872ef3e9341b69adeb0f6a4e67fb297ad8..e50e67943e9630789b54a148efb977b2c8269781 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json index 8f795338d25c05f21310bab7d020d436b4009e1a..fe4945483c0abea0d0546bc6e4482885250281b5 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset.json b/tests/pytest/tools/insert-tblimit-tboffset.json index 2c2d86c4816e6cf6c9f3469e92b7b2a2f750ab66..92b28241a625d6c18435d5698998f72944a52da4 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset.json +++ b/tests/pytest/tools/insert-tblimit-tboffset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json index ce83ea3e606f80c38f247a44bccf61fc1394329b..0c1e00976b8c2a2878096ca0faebf8749b7a1e60 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset0.json +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json index b15aaf4eed2870468f43d49f0f6578c2d91dc528..ff002e9528f7c03c55f64b716665321c92235ee8 100644 --- a/tests/pytest/tools/insert-tblimit1-tboffset.json +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert.json b/tests/pytest/tools/insert.json index 523561dc6d22cec1152d0e698976b0f8a5cf66c5..4489730722d797ef59a9f1cb3f77f9a1109d1176 100644 --- a/tests/pytest/tools/insert.json +++ b/tests/pytest/tools/insert.json @@ -7,7 +7,7 @@ "password": "taosdata", "thread_count": 2, "num_of_records_per_req": 10, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "databases": [{ "dbinfo": { "name": "db01", diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json index a11261681a78b4edc85280c666d98db86f370d94..3c876c61c75b58708293f2068c4a804e37925566 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json index 080231551e306a458a4664adb7f9a68df63a1d52..b9162242d49b67a34d1edfea1a0d1914a4e355ce 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json index fe0ecbe2deed56e8ab2c90fc655ff92833215de7..3fbaeceeba129bd04446b33340e9c68670fe0fda 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json index 1af2952a6940bc78dcc589184f599f5a7d640f1d..6b0631da39c562c0fe78119cf27e39467ecf28c0 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json index 39c5e499096bd6082090f74f2c307629a18f56e2..bf9b0151544409c404285e069ff0c10523931512 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json index f4dbf1ee411377af6c3779d9e5cba6c3e233ed39..346fe31be929385d9f4618b290047321242665c0 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json index 84b511a44621d89b2f23f7fabe38fe0cac489ac6..65a2836a497c073d5814554b28124d5d687ca98f 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json index 75dbcb443230f9528530962242aff1a3a4ac4789..b7b6c186e6e7db8a1ba38626004fd31c4c8ff869 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json index 0c2e9cf34ae9a7529d9430655c67594cb0202114..edb9ed7cb81604056c378d279183e7cc7a47e85e 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json index e90474e872b050ccb33c4e40da76d86f14975b7a..b1d7dc49352c25aba0ece068af194a5f3b28ddba 100755 --- a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json index 21603b190272519373b5771616ad3679892653a5..c1c27cf6d770b3f20588405253e342e198c93bc4 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json index c944c26915063c9e5169f8bb45442f87f47db423..360ec073703edea5a777bd48a150f65ee6fd97f4 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-disorder.json b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json index 4908d3999cad2037a6ce90b9ab85ddcf69df2ddd..930496a877fff73b14627de215212d0f4591b481 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-disorder.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json index 03f531f52b74605bd101b246a9ad0b4cb4dbb7ff..12dadf80063ac4a69eee82936186902745f32380 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json index ce2a34627b68780105bbc0a6c233c8d8365b8569..759a3f074dac9362a18f6510664a6b02ebf8f24e 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal.json index 6e438b33df5af7321cd40b125cee553f98032b02..321495782d86d64e1867cffa74715eaee7d72240 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-illegal.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json index 54e646a5a049d36d83b1e6e56856ff1dda6aaa46..5dd37ee8b05d9518435837478b5d7c2646740482 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json index 9a47a873dddaebb4710827b3cb60840252d62f4c..7fbee6fee078ce225276946e3cc19357723ec3d0 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 100, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newdb.json b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json index 2eb17b1aab5cf26a1cbde8456000a19dd1bef926..16e1f944812bdf8dc292aff162edb3908c380559 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-newdb.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newtable.json b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json index abe277bf5b2bf3f60aebd96f315cc67fb0c9caeb..86c9359ffbb206517a0bb7a5937e0d6e7e716b90 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-newtable.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json index 2dae7eb1d727632dca9cfaa6905d33c9fde39487..7eee9ce55bf93e6a57ba94d629167ee578ffdbfd 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-offset.json b/tests/pytest/tools/taosdemoAllTest/insert-offset.json index 642d01db3eb97a5611d5fe587d2e77929cb23e84..d3946cee3ced7ab9c4588fb0d39acfaec6049e74 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-offset.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-offset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json index 3ef4360aefbca9cb3cae8c04dfe2162075430bd9..c812b4971edfda29b8be030dc893299ce2484600 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-sample.json b/tests/pytest/tools/taosdemoAllTest/insert-sample.json index 5b25281e78361f7c27bd94d024a22afcaf870a77..e24e20067c2cefc18844683434a745daa3377b40 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-sample.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-sample.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-timestep.json b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json index 6432fde4baf3d7c7810236bdf2f02e99906b6e02..ceadfc677ae03837716a503c2a8f92d98c493a89 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-timestep.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json b/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json index 4e59d8667964a909cffe9dd7f4367d814e7a917a..69ebe45e50b3a987e5eade2ab3de5b84f0451835 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json +++ b/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json b/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json index 80d6817b5d09851be7e31c864e968a5b729e063e..8b7086530e7aedaf73a9a62a6ff8f28669e7e0a1 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json b/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json index a35c28f0acd00ed01b627d2d0619bc8183d97f06..1e052ff2a47ccfa2b5af66e47980b40ec87891ed 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json index 05d47c3611dd698d86d078805fac0785bd544479..c67b1dba1428a173a54f3225b386107e118b23c0 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json index e63b3613ba6fa004f80b1eeefb39bb0011d51b27..25e43aefa736baef9be6a9867fd7493b9b3ba458 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json index 137e6083864580be49a0d02c5798f16f8046834a..af04d9c1a3557bb2e211da4808a5db2880b13b44 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json b/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json index 63a4a2ab58a67363d0b69bbf7552c76fd5948699..84a5fe94526923290472c36edb318035aa60d767 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json +++ b/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json b/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json index f3212bc30dcbdb2d8183e1c6050fe3b23ee92748..d092a41483b21d97be50fec553d4e242f0599bfe 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json +++ b/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json index 9711ead80ee17cb5f5b54c3439914262176c5633..45523618f0c25f044b9e7d1485b148896e20a861 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json index 24c61cfa8cfbc810c573e3468730d33e2132eee7..a95c40f9eb6a06f9ca050d7eeed91153ab96535a 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertRestful.json b/tests/pytest/tools/taosdemoAllTest/insertRestful.json index ab7ee9a73b3414937f0843215d1d122448e1eedb..26770c3d09497c7e7810b519a2affa7d7c97c3c6 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertRestful.json +++ b/tests/pytest/tools/taosdemoAllTest/insertRestful.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json index d835822e8f81dd371558de1002ed68487ad0d5e7..74737b4dec837be2b2bb25b7189fa80b32684dc9 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json b/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json index 4c7cdfe39d0ed2dd15abcae7ac6bf75b371e13bf..e0e9f72a5631ec73f49146d13cc40d380b972c40 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json +++ b/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json b/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json index b563dcc94b3c69256f4b2a754e9244cef7874944..fdc1994782b9aba752835afedb50323c0be4508d 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json +++ b/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json b/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json index 0f1a874cc364736a68962c1d293fc8cdc78cd8c8..91d6c1a83710ed4984fff1195dd1fb76eb0e51f3 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json +++ b/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json index bdab459987a587554c001c239c570afd3e7f8636..45a718705a535b09ad64e1fff33542abb38c6d4e 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_block2.json b/tests/pytest/tools/taosdemoAllTest/manual_block2.json index 763421c7f3bdb47509c354818e02b9a2b20ce5bd..f01e55fb5341e5389672277c976e8a53f9a4b73e 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_block2.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json index 0579aedf69a74ad111b8f92808f7046bd0de24c8..f097f15ee13bfa1c23e29870c8bbce45e878417e 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json index d541cb656778fc59fc1f3746fadcca0ced456e0a..2df1fc42aad4ee6537f873c4ae87748bdd488112 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json index c134391a5f759e32a8e9752deab7205e8cb1aa49..be1df2030fa136084e49468bb8f7048ce2753d89 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json index e9f759f8f7167c749bc3617545ee8c926248bf71..a8552404d51b4f8261af092e6ec0a022f7d5b6d6 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json index 9b46ff105b3217ee54ee6c0684136c7033995a05..316fbba4a019337c84a9b7eb649ba294e22e6e1b 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json index fdcaa131e6742f767a04ac52b7f9853b5757dcfb..d03b29d90fbd002288098a4cdb509421eb5003e0 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/query-interrupt.json index 01028f68ad9a6f3aa870d0c1b1e38562e896abe4..1b276cb2b0afd9f0ed7d04f0a2609a4d17df3706 100644 --- a/tests/pytest/tools/taosdemoAllTest/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/query-interrupt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json b/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json index 0fc789c7e30f1d0f74d4e10df635df738b4411be..8565e4a7111b654cefc0424af309e32e8f74b024 100644 --- a/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json +++ b/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json b/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json index 940adfb61c6fc294f7b286514c2808269e8c9e66..0f9be9bdc3d3587178096a8f2eeaca01aaacf594 100644 --- a/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json +++ b/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json index a4baf73689e97f1494606b8ca243d13af024245f..443da39fa127c2f182f3b2adee54a0cb53fe285e 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json index a7a514e9dc46cf62ce24fa81b22bfe9d2c58e654..bd5709ca5e36e252fe7ef496cb652f5db6320332 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json index 3c38f926808c0e08fbb3087aad139ec15997101a..209f414c1b2c045d9c7312edd0f547cb1fb1a24b 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json index 2ee489c7a3cff7deaa41bb2b17ed54ce00bbc217..903c8a9c93f947f87030efea313d503f63eb99ad 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json index 44da22aa3f54abe403c38c9ec11dcdbe346abfb9..dcbec40034801e0665ccdc23a4318a78d4c37d9a 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json index b2805a38e51d86e80838efb753c0f10c94b2c5b4..1ea4de5cfe7d921065c5115ecbca368e1f0484a6 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json index ac540befb637b0105a4f718228db11dc3f51ca01..86f2fa6c4d89a20f86d95ea3cc3965813cbb2974 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json index 9a7ad93636f6578d0adb7553c2d912f38614301d..d634ab83690242bde9beca7aa39b8705e9aa52cc 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json index 919b91839530c0bd5db3338d73698eed19aefda7..4b69118ef52832ec7b5586c64b63ebb0732de389 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json index dcf52931ad40788edd1f7f16f3e7cdd190792b16..32043996b6d5de87862ee1ea9e31715edf873aeb 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json index d2304ed537d5c18e81c2d93803947396ecb2ed5a..a1a0b89e48265c1a1a45c35fe3bb508e14aab7e5 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json index d297240613de0e51dcab3e0582fd041858010eda..f5cea2ccc30c97e5c338e5737a003feafdbe7535 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 100, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json index d117c5b3450e31a8736eea97d36d9d172c74e314..c3bdea61c6aa74b2dfae20299f0c1c8424de4c7d 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json index 1b36b3cbe9cc520a625645bf1e1e5b89a6be2a11..e92644d33eef6de46e38c359b4bb4467dfdb2f8f 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json index ea95736a00fba7630f8479699397f455b51db45c..0618c04b30dd41d87c72f97042a80c7e3b05ebdc 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json index 8318de6672bbcb8c705648f593baf647d3b3f571..356ac38d147edc0368ae07fa7c37825f50241e7e 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json index b6cb47f2c5f086fd50794fab7b84188ee1162bcf..2f8f6931667092aa8d5e3ba0c90eb0ccbc3860e1 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json index 348e93ff8b5b0a1666d22cc017f376a1da120702..c1da95ba8c830e797362043418f5d64d7f524a24 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json index edbaae60a14aa8c289d9f3854f654f3da27f37da..9522f0e7b5388d4967303983c38055c5183f0f3a 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json index 1c72b4f402d67070b9b25d6ff8c83923148e1c92..bcbda0a301cdcf59e65d7b53bd51eb919d1f2705 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json index 4626babd9519bd702373dc321a801075df655903..2b30aa3e9eea7c7cb91bece37e2ce16e02a71c0a 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json index f140883de168d77ad83253532fecfee81c9dd7c9..f3c577b30cf29dd733bc5ac920d68ce3f8d6ee55 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json index d1d2db2df388a63b7587932cfc0b980f67cce62f..a0ff8872500163c1479a230fdd357334fac96f87 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json index d79d4cace533578d4cf2d55430bef55dd64485c8..5ff9ec63a2aec226cbb1fd7850c1549e33af1c5b 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json index eb0ab0f04ac8f602d83eb5271ae7f5eab86f7d10..79ce66097ba595b5f900e50a41e9e1f1d53a8fa4 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json index 489632c645e732eb9c0fe2fa358947b1e6ba585e..4b21f0a184d37c18aba14537a9cadd439d6a56a7 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json index 19eb92bf4c8541eca4d6d3306d5e5772998ca719..9fb85aef23e45bcfeaa4e526796abf7cd5ce1e83 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json index dbda4f74a1d209c5a112028e21fdec27ff390a14..80944de3f576574cf1c80c085bb0c4dc1a1b9d58 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json index 966c285d2f7fc197dd8af6a7b8ea9c0caf58aa45..834ffb56d37dca082b8bc152851bc1e3e847b2be 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json index c1fc02553fe501e7c09769d947b3f21acc96555e..f39aa94830cfe47ffd364e5f2a1419f130ea2ec8 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json index ed3eb280f6869bed76de72bdf50b646bca4a245a..6345227788af3948dd049d09aefde0f4207eea73 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json index 1d7ad8a90eb95a86f109214f516d9484b11a53da..75a365bbff9c515ee31d94c96df1d18bc286d988 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/subInsertdata.json b/tests/pytest/tools/taosdemoAllTest/subInsertdata.json index 1ca302a320897f7fc04dbbef9aa8a2fea2808724..f5e7ac3018c59ce5d9e8702c788b3e5cd8605994 100644 --- a/tests/pytest/tools/taosdemoAllTest/subInsertdata.json +++ b/tests/pytest/tools/taosdemoAllTest/subInsertdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json b/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json index ef6354627880bf3fde91567e5de3ee518fccb995..896a72598d73b4abc82c0ee8251c8155d54352bd 100644 --- a/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json +++ b/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json index b6e5847b54897814fb9c6e7b1c7f9cb4ed8d29f3..8211a92a2d999b798bc625cb4cffc7718ecf1bb4 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json index ed97fea33e106aff8d2821a10191bd360a629a6b..304ff99c26dcddcd7a98cdbcf7da3d4c458dc4ec 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json index db34bfc6b8a617b1a57ed687562bb09ade6c24c8..444e6564bea134286422c3d612f2719186ad324e 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json index d029ddea219aca3ce79a19035e6ae1bead016795..67003a1fb5c1dae0810497bc480a88e3ab9b4919 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json index f8a181d352fad7702cf97aaca9aea3aa1801cab1..7454af6521dc46c18c3c61a1c278011affd70654 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json index b06ec55ef6157e46a435c0a10ef0144f7e648334..602a39ca24bce0580506d7ad833957d29ebf020a 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json index 6a6a6da2979869690298978676641d3279cd69b0..79d3bc5ed824b541f21750a6f4d9a206172096e1 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 82c57a656dfea12f80fe4eb2b530742c5bfb0916..9a4b564319048921e349437d9ccc3927147017a9 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -94,7 +94,7 @@ class taosdemoPerformace: "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "databases": [db] } diff --git a/tests/pytest/tsdb/insertDataDb1.json b/tests/pytest/tsdb/insertDataDb1.json index 92735dad69790f51cc35878f4c81dc7d81a64b72..f771551b26f5f0a16401312aa613450b267e8249 100644 --- a/tests/pytest/tsdb/insertDataDb1.json +++ b/tests/pytest/tsdb/insertDataDb1.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb1Replica2.json b/tests/pytest/tsdb/insertDataDb1Replica2.json index a5fc525157c9d22084f137b9057b4ebe7d2e7c5f..ec84d71d88cef197678dba8fdc6b1b80e1614718 100644 --- a/tests/pytest/tsdb/insertDataDb1Replica2.json +++ b/tests/pytest/tsdb/insertDataDb1Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2.json b/tests/pytest/tsdb/insertDataDb2.json index 02301e024271509642d4aa4c8fa5f19e2b39c939..494465d23c67036bed7b3994fece6e6a5c5f75de 100644 --- a/tests/pytest/tsdb/insertDataDb2.json +++ b/tests/pytest/tsdb/insertDataDb2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2Newstab.json b/tests/pytest/tsdb/insertDataDb2Newstab.json index 2f5f2367b4445f58f67155381536f520a3422a7a..647a587cad3ad66590fb3ef770aa8ff35a74f31e 100644 --- a/tests/pytest/tsdb/insertDataDb2Newstab.json +++ b/tests/pytest/tsdb/insertDataDb2Newstab.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json index 67f3b2cd4f2cdb08fe8337a8372e35c0b6a2e02b..13cf2e561c88893d545e425bf2d107382387c3cc 100644 --- a/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json +++ b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2Replica2.json b/tests/pytest/tsdb/insertDataDb2Replica2.json index 3d033f13cc77ac9ecdf0803cf8d014c3b5a9a882..c651657a6d9e2f91d40d77c8b7d8b2d2f9d32939 100644 --- a/tests/pytest/tsdb/insertDataDb2Replica2.json +++ b/tests/pytest/tsdb/insertDataDb2Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 4c50af6bac28e8d0d6d315189a1204725f7a0744..753c41e094701271ca3b49a53eabde1461bd1e08 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -84,10 +84,10 @@ class TDSql: self.queryResult = None tdLog.info("sql:%s, expect error occured" % (sql)) - def query(self, sql, row_tag=None,queyTimes=10): + def query(self, sql, row_tag=None,queryTimes=10): self.sql = sql i=1 - while i <= queyTimes: + while i <= queryTimes: try: self.cursor.execute(sql) self.queryResult = self.cursor.fetchall() @@ -97,26 +97,15 @@ class TDSql: return self.queryResult return self.queryRows except Exception as e: - i+=1 tdLog.notice("Try to query again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 time.sleep(1) pass - else: - try: - tdLog.notice("Try the last query ") - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - if row_tag: - return self.queryResult - return self.queryRows - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - traceback.print_exc() - raise Exception(repr(e)) def is_err_sql(self, sql): @@ -305,28 +294,23 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queyTimes=10): + def execute(self, sql,queryTimes=10): self.sql = sql i=1 - while i <= queyTimes: + while i <= queryTimes: try: self.affectedRows = self.cursor.execute(sql) return self.affectedRows except Exception as e: - i+=1 tdLog.notice("Try to execute sql again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 time.sleep(1) pass - else: - try: - tdLog.notice("Try the last execute sql ") - self.affectedRows = self.cursor.execute(sql) - return self.affectedRows - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: diff --git a/tests/pytest/util/taosdemoCfg.py b/tests/pytest/util/taosdemoCfg.py index 7523a808980fc924672cb8eedc0a338be0f8d745..f708d303de06c8ab7639453d94f0bb63d445419b 100644 --- a/tests/pytest/util/taosdemoCfg.py +++ b/tests/pytest/util/taosdemoCfg.py @@ -50,7 +50,7 @@ class TDTaosdemoCfg: "user": "root", "password": "taosdata", "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb1.json b/tests/pytest/wal/insertDataDb1.json index a14fe581412f9497b4c16b94213685f31e06aa0c..2dc0cf2b7f2b6c99fad4d049260ea1b5959d39e9 100644 --- a/tests/pytest/wal/insertDataDb1.json +++ b/tests/pytest/wal/insertDataDb1.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb1Replica2.json b/tests/pytest/wal/insertDataDb1Replica2.json index a5fc525157c9d22084f137b9057b4ebe7d2e7c5f..ec84d71d88cef197678dba8fdc6b1b80e1614718 100644 --- a/tests/pytest/wal/insertDataDb1Replica2.json +++ b/tests/pytest/wal/insertDataDb1Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2.json b/tests/pytest/wal/insertDataDb2.json index 891a21f73e195996d7bb5d8539b22b88164efa0c..35232a633315e3281d3496290f4ce5165ac4a235 100644 --- a/tests/pytest/wal/insertDataDb2.json +++ b/tests/pytest/wal/insertDataDb2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2Newstab.json b/tests/pytest/wal/insertDataDb2Newstab.json index 2f5f2367b4445f58f67155381536f520a3422a7a..647a587cad3ad66590fb3ef770aa8ff35a74f31e 100644 --- a/tests/pytest/wal/insertDataDb2Newstab.json +++ b/tests/pytest/wal/insertDataDb2Newstab.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2NewstabReplica2.json b/tests/pytest/wal/insertDataDb2NewstabReplica2.json index 67f3b2cd4f2cdb08fe8337a8372e35c0b6a2e02b..13cf2e561c88893d545e425bf2d107382387c3cc 100644 --- a/tests/pytest/wal/insertDataDb2NewstabReplica2.json +++ b/tests/pytest/wal/insertDataDb2NewstabReplica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2Replica2.json b/tests/pytest/wal/insertDataDb2Replica2.json index 3d033f13cc77ac9ecdf0803cf8d014c3b5a9a882..c651657a6d9e2f91d40d77c8b7d8b2d2f9d32939 100644 --- a/tests/pytest/wal/insertDataDb2Replica2.json +++ b/tests/pytest/wal/insertDataDb2Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 356eaed57e1518a3489010488d47a6d96b83fec4..0a859b20458b7118fd2b517e9cbda26e42c1bf16 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -179,6 +179,7 @@ ./test.sh -f tsim/query/scalarFunction.sim ./test.sh -f tsim/query/scalarNull.sim ./test.sh -f tsim/query/session.sim +./test.sh -f tsim/query/udf.sim # ---- qnode ./test.sh -f tsim/qnode/basic1.sim diff --git a/tests/script/sh/bit_and.c b/tests/script/sh/bit_and.c new file mode 100644 index 0000000000000000000000000000000000000000..2f2e48fdb0882db2a40c9aae019402e031343d13 --- /dev/null +++ b/tests/script/sh/bit_and.c @@ -0,0 +1,61 @@ +#include +#include +#include +#include "taosudf.h" + + +DLL_EXPORT int32_t bit_and_init() { + return 0; +} + +DLL_EXPORT int32_t bit_and_destroy() { + return 0; +} + +DLL_EXPORT int32_t bit_and(SUdfDataBlock* block, SUdfColumn *resultCol) { + + if (block->numOfCols < 2) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + + SUdfColumnMeta *meta = &resultCol->colMeta; + meta->bytes = 4; + meta->type = TSDB_DATA_TYPE_INT; + meta->scale = 0; + meta->precision = 0; + + + SUdfColumnData *resultData = &resultCol->colData; + + resultData->numOfRows = block->numOfRows; + + for (int32_t i = 0; i < resultData->numOfRows; ++i) { + if (udfColDataIsNull(block->udfCols[0], i)) { + udfColDataSetNull(resultCol, i); + continue; + } + int32_t result = *(int32_t*)udfColDataGetData(block->udfCols[0], i); + int j = 1; + for (; j < block->numOfCols; ++j) { + if (udfColDataIsNull(block->udfCols[j], i)) { + udfColDataSetNull(resultCol, i); + break; + } + + char* colData = udfColDataGetData(block->udfCols[j], i); + result &= *(int32_t*)colData; + } + if (j == block->numOfCols) { + udfColDataSet(resultCol, i, (char*)&result, false); + } + + } + return TSDB_CODE_SUCCESS; +} diff --git a/tests/script/sh/compile_udf.sh b/tests/script/sh/compile_udf.sh new file mode 100755 index 0000000000000000000000000000000000000000..5ff3f2bc8a21b15281a7b687d7dd9a0eb97ca7f8 --- /dev/null +++ b/tests/script/sh/compile_udf.sh @@ -0,0 +1,10 @@ +set +e + +rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so +mkdir -p /tmp/udf +echo "compile udf bit_and and sqr_sum" +gcc -fPIC -shared sh/bit_and.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libbitand.so +gcc -fPIC -shared sh/sqr_sum.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libsqrsum.so +echo "debug show /tmp/udf/*.so" +ls /tmp/udf/*.so + diff --git a/tests/script/sh/sqr_sum.c b/tests/script/sh/sqr_sum.c new file mode 100644 index 0000000000000000000000000000000000000000..af57f377abb6f593b49192adb4170af842afae0f --- /dev/null +++ b/tests/script/sh/sqr_sum.c @@ -0,0 +1,80 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +DLL_EXPORT int32_t sqr_sum_init() { + return 0; +} + +DLL_EXPORT int32_t sqr_sum_destroy() { + return 0; +} + +DLL_EXPORT int32_t sqr_sum_start(SUdfInterBuf *buf) { + *(int64_t*)(buf->buf) = 0; + buf->bufLen = sizeof(double); + buf->numOfResult = 0; + return 0; +} + +DLL_EXPORT int32_t sqr_sum(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + double sumSquares = *(double*)interBuf->buf; + int8_t numNotNull = 0; + for (int32_t i = 0; i < block->numOfCols; ++i) { + SUdfColumn* col = block->udfCols[i]; + if (!(col->colMeta.type == TSDB_DATA_TYPE_INT || + col->colMeta.type == TSDB_DATA_TYPE_DOUBLE)) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + for (int32_t i = 0; i < block->numOfCols; ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn* col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + switch (col->colMeta.type) { + case TSDB_DATA_TYPE_INT: { + char* cell = udfColDataGetData(col, j); + int32_t num = *(int32_t*)cell; + sumSquares += (double)num * num; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + char* cell = udfColDataGetData(col, j); + double num = *(double*)cell; + sumSquares += num * num; + break; + } + default: + break; + } + ++numNotNull; + } + } + + *(double*)(newInterBuf->buf) = sumSquares; + newInterBuf->bufLen = sizeof(double); + + if (interBuf->numOfResult == 0 && numNotNull == 0) { + newInterBuf->numOfResult = 0; + } else { + newInterBuf->numOfResult = 1; + } + return 0; +} + +DLL_EXPORT int32_t sqr_sum_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { + if (buf->numOfResult == 0) { + resultData->numOfResult = 0; + return 0; + } + double sumSquares = *(double*)(buf->buf); + *(double*)(resultData->buf) = sqrt(sumSquares); + resultData->bufLen = sizeof(double); + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/tsim/parser/function.sim b/tests/script/tsim/parser/function.sim index 110901a6e17ec5c429863827f777b0226bf91bd2..9423f53e3a63ec225660bc21e6663e1cb89998c9 100644 --- a/tests/script/tsim/parser/function.sim +++ b/tests/script/tsim/parser/function.sim @@ -327,8 +327,8 @@ endi print =================>td-2610 sql select stddev(k) from tm2 where ts='2020-12-29 18:46:19.109' -if $rows != 0 then - print expect 0, actual:$rows +if $rows != 1 then + print expect 1, actual:$rows return -1 endi sql select twa(k) from tm2 where ts='2020-12-29 18:46:19.109' @@ -406,7 +406,7 @@ if $data00 != 1.378704626 then endi sql select stddev(c) from m1 -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/script/tsim/parser/nestquery.sim b/tests/script/tsim/parser/nestquery.sim index 101205db74057a951c8978ac3c85fcc727170ae9..8cb7a3790bfabe18032024e928bc87db4aeae391 100644 --- a/tests/script/tsim/parser/nestquery.sim +++ b/tests/script/tsim/parser/nestquery.sim @@ -160,12 +160,12 @@ endi sql select stddev(c1) from (select c1 from nest_tb0); sql_error select percentile(c1, 20) from (select * from nest_tb0); -sql select interp(c1) from (select * from nest_tb0); +#sql select interp(c1) from (select * from nest_tb0); sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); sql_error select twa(c1) from (select c1 from nest_tb0); sql_error select irate(c1) from (select c1 from nest_tb0); sql_error select diff(c1), twa(c1) from (select * from nest_tb0); -sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); +#sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); sql select _wstart, apercentile(c1, 50) from (select * from nest_tb0) interval(1d) if $rows != 7 then diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim new file mode 100644 index 0000000000000000000000000000000000000000..7259b1e7797a990b7a9e5aea58c0139962e4ea17 --- /dev/null +++ b/tests/script/tsim/query/udf.sim @@ -0,0 +1,161 @@ +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 udf +system sh/compile_udf.sh +sql create database udf vgroups 3; +sql use udf; +sql show databases; + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi +if $system_content == Windows_NT then + sql create function bit_and as 'C:\\Windows\\Temp\\bitand.dll' outputtype int bufSize 8; + sql create aggregate function sqr_sum as 'C:\\Windows\\Temp\\sqrsum.dll' outputtype double bufSize 8; +else + sql create function bit_and as '/tmp/udf/libbitand.so' outputtype int bufSize 8; + sql create aggregate function sqr_sum as '/tmp/udf/libsqrsum.so' outputtype double bufSize 8; +endi +sql show functions; +if $rows != 2 then + return -1 +endi +sql select bit_and(f, f) from t; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi +if $data10 != 2 then + return -1 +endi + +sql select sqr_sum(f) from t; +if $rows != 1 then + print expect 1, actual $rows + return -1 +endi +if $data00 != 2.236067977 then + return -1 +endi + +sql create table t2 (ts timestamp, f1 int, f2 int); +sql insert into t2 values(now, 0, 0)(now+1s, 1, 1); +sql select bit_and(f1, f2) from t2; +if $rows != 2 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +sql select sqr_sum(f1, f2) from t2; +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi + +sql insert into t2 values(now+2s, 1, null)(now+3s, null, 2); +sql select bit_and(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 +if $rows != 4 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +sql select sqr_sum(f1, f2) from t2; +print $rows, $data00 +if $rows != 1 then + return -1 +endi +if $data00 != 2.645751311 then + return -1 +endi + +sql insert into t2 values(now+4s, 4, 8)(now+5s, 5, 9); +sql select sqr_sum(f1-f2), sqr_sum(f1+f2) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1; +endi +if $data00 != 5.656854249 then + return -1 +endi +if $data01 != 18.547236991 then + return -1 +endi + +sql select sqr_sum(bit_and(f2, f1)), sqr_sum(bit_and(f1, f2)) from t2; +print $rows , $data00 , $data01 +if $rows != 1 then + return -1 +endi +if $data00 != 1.414213562 then + return -1 +endi +if $data01 != 1.414213562 then + return -1 +endi + +sql select sqr_sum(f2) from udf.t2 group by 1-bit_and(f1, f2) order by 1-bit_and(f1,f2); +print $rows , $data00 , $data10 , $data20 +if $rows != 3 then + return -1 +endi +if $data00 != 2.000000000 then + return -1 +endi +if $data10 != 9.055385138 then + return -1 +endi +if $data20 != 8.000000000 then + return -1 +endi + +sql drop function bit_and; +sql show functions; +if $rows != 1 then + return -1 +endi +if $data00 != @sqr_sum@ then + return -1 + endi +sql drop function sqr_sum; +sql show functions; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/drop_stream.sim b/tests/script/tsim/stream/drop_stream.sim index bdd88bf780ba7fe916b39683d8a65e0d9d503119..747f59fe85084ba08a7b0679b8dc0363cb7c8db4 100644 --- a/tests/script/tsim/stream/drop_stream.sim +++ b/tests/script/tsim/stream/drop_stream.sim @@ -45,70 +45,70 @@ sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); sql create table scalar_ct1 using scalar_stb tags(10); sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); -sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; -sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; -sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; -sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; -sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; -sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; -sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; -sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; -sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; -sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; -sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; -sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; -sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; -sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb partition by tbname; sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; -sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; -sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; -sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb partition by tbname; sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; -sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb partition by tbname; sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; -sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb partition by tbname; sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; -sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb partition by tbname; sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; -sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb partition by tbname; sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; -sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb partition by tbname; sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); @@ -136,70 +136,70 @@ sql create table test.scalar_function_tb1 (ts timestamp, c1 tinyint, c2 smallint sql create table if not exists scalar_stb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)) tags (t1 int); sql create table scalar_ct1 using scalar_stb tags(10); sql create table if not exists scalar_tb (ts timestamp, c1 int, c2 double, c3 binary(20), c4 binary(20), c5 nchar(20)); -sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb; +sql create stream stb_abs_stream trigger at_once into output_abs_stb as select ts, abs(c1), abs(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_abs_stream trigger at_once into output_abs_ctb as select ts, abs(c1), abs(c2), c3 from scalar_ct1; sql create stream tb_abs_stream trigger at_once into output_abs_tb as select ts, abs(c1), abs(c2), c3 from scalar_tb; -sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb; +sql create stream stb_acos_stream trigger at_once into output_acos_stb as select ts, acos(c1), acos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_acos_stream trigger at_once into output_acos_ctb as select ts, acos(c1), acos(c2), c3 from scalar_ct1; sql create stream tb_acos_stream trigger at_once into output_acos_tb as select ts, acos(c1), acos(c2), c3 from scalar_tb; -sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb; +sql create stream stb_asin_stream trigger at_once into output_asin_stb as select ts, asin(c1), asin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_asin_stream trigger at_once into output_asin_ctb as select ts, asin(c1), asin(c2), c3 from scalar_ct1; sql create stream tb_asin_stream trigger at_once into output_asin_tb as select ts, asin(c1), asin(c2), c3 from scalar_tb; -sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb; +sql create stream stb_atan_stream trigger at_once into output_atan_stb as select ts, atan(c1), atan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_atan_stream trigger at_once into output_atan_ctb as select ts, atan(c1), atan(c2), c3 from scalar_ct1; sql create stream tb_atan_stream trigger at_once into output_atan_tb as select ts, atan(c1), atan(c2), c3 from scalar_tb; -sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb; +sql create stream stb_ceil_stream trigger at_once into output_ceil_stb as select ts, ceil(c1), ceil(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_ceil_stream trigger at_once into output_ceil_ctb as select ts, ceil(c1), ceil(c2), c3 from scalar_ct1; sql create stream tb_ceil_stream trigger at_once into output_ceil_tb as select ts, ceil(c1), ceil(c2), c3 from scalar_tb; -sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb; +sql create stream stb_cos_stream trigger at_once into output_cos_stb as select ts, cos(c1), cos(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_cos_stream trigger at_once into output_cos_ctb as select ts, cos(c1), cos(c2), c3 from scalar_ct1; sql create stream tb_cos_stream trigger at_once into output_cos_tb as select ts, cos(c1), cos(c2), c3 from scalar_tb; -sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb; +sql create stream stb_floor_stream trigger at_once into output_floor_stb as select ts, floor(c1), floor(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_floor_stream trigger at_once into output_floor_ctb as select ts, floor(c1), floor(c2), c3 from scalar_ct1; sql create stream tb_floor_stream trigger at_once into output_floor_tb as select ts, floor(c1), floor(c2), c3 from scalar_tb; -sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb; +sql create stream stb_log_stream trigger at_once into output_log_stb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_log_stream trigger at_once into output_log_ctb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_ct1; sql create stream tb_log_stream trigger at_once into output_log_tb as select ts, log(c1, 2), log(c2, 2), c3 from scalar_tb; -sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb; +sql create stream stb_pow_stream trigger at_once into output_pow_stb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_stb partition by tbname; sql create stream ctb_pow_stream trigger at_once into output_pow_ctb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_ct1; sql create stream tb_pow_stream trigger at_once into output_pow_tb as select ts, pow(c1, 2), pow(c2, 2), c3 from scalar_tb; -sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb; +sql create stream stb_round_stream trigger at_once into output_round_stb as select ts, round(c1), round(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_round_stream trigger at_once into output_round_ctb as select ts, round(c1), round(c2), c3 from scalar_ct1; sql create stream tb_round_stream trigger at_once into output_round_tb as select ts, round(c1), round(c2), c3 from scalar_tb; -sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb; +sql create stream stb_sin_stream trigger at_once into output_sin_stb as select ts, sin(c1), sin(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sin_stream trigger at_once into output_sin_ctb as select ts, sin(c1), sin(c2), c3 from scalar_ct1; sql create stream tb_sin_stream trigger at_once into output_sin_tb as select ts, sin(c1), sin(c2), c3 from scalar_tb; -sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb; +sql create stream stb_sqrt_stream trigger at_once into output_sqrt_stb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_sqrt_stream trigger at_once into output_sqrt_ctb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_ct1; sql create stream tb_sqrt_stream trigger at_once into output_sqrt_tb as select ts, sqrt(c1), sqrt(c2), c3 from scalar_tb; -sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb; +sql create stream stb_tan_stream trigger at_once into output_tan_stb as select ts, tan(c1), tan(c2), c3 from scalar_stb partition by tbname; sql create stream ctb_tan_stream trigger at_once into output_tan_ctb as select ts, tan(c1), tan(c2), c3 from scalar_ct1; sql create stream tb_tan_stream trigger at_once into output_tan_tb as select ts, tan(c1), tan(c2), c3 from scalar_tb; -sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb; +sql create stream stb_char_length_stream into output_char_length_stb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_stb partition by tbname; sql create stream ctb_char_length_stream into output_char_length_ctb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_ct1; sql create stream tb_char_length_stream into output_char_length_tb as select ts, char_length(c3), char_length(c4), char_length(c5) from scalar_tb; -sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb; +sql create stream stb_concat_stream into output_concat_stb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_stream into output_concat_ctb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_ct1; sql create stream tb_concat_stream into output_concat_tb as select ts, concat(c3, c4), concat(c3, c5), concat(c4, c5), concat(c3, c4, c5) from scalar_tb; -sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb; +sql create stream stb_concat_ws_stream into output_concat_ws_stb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_stb partition by tbname; sql create stream ctb_concat_ws_stream into output_concat_ws_ctb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_ct1; sql create stream tb_concat_ws_stream into output_concat_ws_tb as select ts, concat_ws("aND", c3, c4), concat_ws("and", c3, c5), concat_ws("And", c4, c5), concat_ws("AND", c3, c4, c5) from scalar_tb; -sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb; +sql create stream stb_length_stream into output_length_stb as select ts, length(c3), length(c4), length(c5) from scalar_stb partition by tbname; sql create stream ctb_length_stream into output_length_ctb as select ts, length(c3), length(c4), length(c5) from scalar_ct1; sql create stream tb_length_stream into output_length_tb as select ts, length(c3), length(c4), length(c5) from scalar_tb; -sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb; +sql create stream stb_lower_stream into output_lower_stb as select ts, lower(c3), lower(c4), lower(c5) from scalar_stb partition by tbname; sql create stream ctb_lower_stream into output_lower_ctb as select ts, lower(c3), lower(c4), lower(c5) from scalar_ct1; sql create stream tb_lower_stream into output_lower_tb as select ts, lower(c3), lower(c4), lower(c5) from scalar_tb; -sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb; +sql create stream stb_ltrim_stream into output_ltrim_stb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_stb partition by tbname; sql create stream ctb_ltrim_stream into output_ltrim_ctb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_ct1; sql create stream tb_ltrim_stream into output_ltrim_tb as select ts, ltrim(c3), ltrim(c4), ltrim(c5) from scalar_tb; -sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb; +sql create stream stb_rtrim_stream into output_rtrim_stb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_stb partition by tbname; sql create stream ctb_rtrim_stream into output_rtrim_ctb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_ct1; sql create stream tb_rtrim_stream into output_rtrim_tb as select ts, rtrim(c3), rtrim(c4), rtrim(c5) from scalar_tb; -sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb; +sql create stream stb_substr_stream into output_substr_stb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_stb partition by tbname; sql create stream ctb_substr_stream into output_substr_ctb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_ct1; sql create stream tb_substr_stream into output_substr_tb as select ts, substr(c3, 2), substr(c3, 2, 2), substr(c4, 5, 1), substr(c5, 3, 4) from scalar_tb; -sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb; +sql create stream stb_upper_stream into output_upper_stb as select ts, upper(c3), upper(c4), upper(c5) from scalar_stb partition by tbname; sql create stream ctb_upper_stream into output_upper_ctb as select ts, upper(c3), upper(c4), upper(c5) from scalar_ct1; sql create stream tb_upper_stream into output_upper_tb as select ts, upper(c3), upper(c4), upper(c5) from scalar_tb; sql insert into scalar_ct1 values (1656668180503, 100, 100.1, "beijing", "taos", "Taos"); diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index 4e127654ee76dee764e97407baf577fd95e2d585..7cd291e56f6f235f1abc28bb4fb5efbadab09699 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -48,7 +48,7 @@ endi $replica = 3 $vgroups = 1 -print ============= create database +print ============= create database db sql create database db replica $replica vgroups $vgroups $loop_cnt = 0 @@ -135,7 +135,7 @@ endw $totalTblNum = $tbNum * 2 sleep 1000 sql show tables -print ====> expect $totalTblNum and infinsert $rows in fact +print ====> expect $totalTblNum and insert $rows in fact if $rows != $totalTblNum then return -1 endi diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index c4ab9bd4bcbb4c513359d58da05e62a064a6037f..a9858acbfb220ab71d030aea3804ade785499bf5 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -202,7 +202,7 @@ else endi vg_ready: -print ====> create stable/child table +print ====> create stable stb /child table ctb and general table ntb sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) sql show stables @@ -225,7 +225,7 @@ endw $totalTblNum = $tbNum * 2 sleep 1000 sql show tables -print ====> expect $totalTblNum and infinsert $rows in fact +print ====> expect $totalTblNum and insert $rows in fact if $rows != $totalTblNum then return -1 endi @@ -237,6 +237,7 @@ sql show vgroups $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId +print ====> switch_leader_to_offine_loop switch_leader_to_offine_loop: print $dnodeId @@ -271,22 +272,7 @@ system sh/exec.sh -n $dnodeId -s start $switch_loop_cnt = $switch_loop_cnt + 1 print $switch_loop_cnt -if $switch_loop_cnt == 1 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 2 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 3 then - sql show vgroups - $dnodeId = $data[0][3] - $dnodeId = dnode . $dnodeId - goto switch_leader_to_offine_loop -elif $switch_loop_cnt == 4 then +if $switch_loop_cnt <= 4 then sql show vgroups $dnodeId = $data[0][3] $dnodeId = dnode . $dnodeId diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 702037f0a6ce7b1bb490d1dde82566e84ceac3eb..6e456148bff1f5d2f3fc80c357bef2f16f0c8130 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -78,7 +78,7 @@ sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), sql select * from tb1 where tbcol not in (1,2,3,null); sql select * from tb1 where tbcol + 3 <> null; sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) -#sql select tbcol5 - tbcol3 from tb1 +sql select tbcol5 - tbcol3 from tb1 print =============== step4: stb sql select avg(tbcol) as c from stb @@ -109,7 +109,7 @@ sql select * from stb where tbcol + 3 <> null; sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb where tbcol = 1 and tbcol2 = 1 and tbcol3 = 1 partition by tgcol interval(1d) sql select _wstart, count(*) from tb1 session(ts, 1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) -#sql select tbcol5 - tbcol3 from stb +sql select tbcol5 - tbcol3 from stb print =============== step5: explain sql explain analyze select ts from stb where -2; diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index ce8ac6941b1464d8bf05ee516c3a57d97403414a..3be59f0adf691f9479cce2c927c9161741bc8130 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -1,15 +1,16 @@ -from tabnanny import check import taos import time import inspect import traceback import socket from dataclasses import dataclass +from datetime import datetime from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from util.common import * PRIVILEGES_ALL = "ALL" PRIVILEGES_READ = "READ" @@ -21,17 +22,40 @@ WEIGHT_WRITE = 3 PRIMARY_COL = "ts" -INT_COL = "c1" -BINT_COL = "c2" -SINT_COL = "c3" -TINT_COL = "c4" -FLOAT_COL = "c5" -DOUBLE_COL = "c6" -BOOL_COL = "c7" - -BINARY_COL = "c8" -NCHAR_COL = "c9" -TS_COL = "c10" +INT_COL = "c_int" +BINT_COL = "c_bint" +SINT_COL = "c_sint" +TINT_COL = "c_tint" +FLOAT_COL = "c_float" +DOUBLE_COL = "c_double" +BOOL_COL = "c_bool" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" + +NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [BOOL_COL, ] +TS_TYPE_COL = [TS_COL, ] + +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] + +# insert data args： +TIME_STEP = 10000 +NOW = int(datetime.timestamp(datetime.now()) * 1000) + +# init db/table +DBNAME = "db" +STBNAME = "stb1" +CTBNAME = "ct1" +NTBNAME = "nt1" class TDconnect: def __init__(self, @@ -247,25 +271,26 @@ class TDTestCase: with taos_connect(user=user.name, passwd=user.passwd) as use: time.sleep(2) if check_priv == PRIVILEGES_ALL: - use.query("use db") - use.query("show tables") - use.query("select * from ct1") - use.query("insert into t1 (ts) values (now())") + use.query(f"use {DBNAME}") + use.query(f"show {DBNAME}.tables") + use.query(f"select * from {DBNAME}.{CTBNAME}") + use.query(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())") elif check_priv == PRIVILEGES_READ: - use.query("use db") - use.query("show tables") - use.query("select * from ct1") - use.error("insert into t1 (ts) values (now())") + use.query(f"use {DBNAME}") + use.query(f"show {DBNAME}.tables") + use.query(f"select * from {DBNAME}.{CTBNAME}") + use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())") elif check_priv == PRIVILEGES_WRITE: - use.query("use db") - use.query("show tables") - use.error("select * from ct1") - use.query("insert into t1 (ts) values (now())") + use.query(f"use {DBNAME}") + use.query(f"show {DBNAME}.tables") + use.error(f"select * from {DBNAME}.{CTBNAME}") + use.query(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())") elif check_priv is None: - use.error("use db") - use.error("show tables") - use.error("select * from db.ct1") - use.error("insert into db.t1 (ts) values (now())") + use.error(f"use {DBNAME}") + # use.error(f"show {DBNAME}.tables") + use.error(f"show tables") + use.error(f"select * from {DBNAME}.{CTBNAME}") + use.error(f"insert into {DBNAME}.{CTBNAME} (ts) values (now())") def __change_user_priv(self, user: User, pre_priv, invoke=False): if user.priv == pre_priv and invoke : @@ -418,7 +443,7 @@ class TDTestCase: self.__grant_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__grant_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"GRANT {self.__privilege[0]} ON * TO {self.__user_list[0]}" , - f"GRANT {self.__privilege[0]} ON db.t1 TO {self.__user_list[0]}" , + f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" , ] def __revoke_err(self): @@ -430,7 +455,7 @@ class TDTestCase: self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" , - f"REVOKE {self.__privilege[0]} ON db.t1 FROM {self.__user_list[0]}" , + f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" , ] def test_grant_err(self): @@ -505,101 +530,48 @@ class TDTestCase: self.drop_user_error() self.drop_user_current() - def __create_tb(self): - - tdLog.printNoPrefix("==========step1:create table") - create_stb_sql = f'''create table stb1( - ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, - {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp - ) tags (t1 int) - ''' - create_ntb_sql = f'''create table t1( - ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, - {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp - ) + def __create_tb(self, stb=STBNAME, ctb_num=20, ntbnum=1, dbname=DBNAME): + tdLog.printNoPrefix("==========step: create table") + create_stb_sql = f'''create table {dbname}.{stb}( + {PRIMARY_COL} timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) tags ({INT_TAG} int) ''' tdSql.execute(create_stb_sql) - tdSql.execute(create_ntb_sql) - - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2} - - def __insert_data(self, rows): - now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - for i in range(rows): - tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" - ) - tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" - ) - tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" - ) - tdSql.execute( - f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) - ''' - ) - tdSql.execute( - f'''insert into ct4 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( - { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + for i in range(ntbnum): + create_ntb_sql = f'''create table {dbname}.nt{i+1}( + {PRIMARY_COL} timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned ) - ( - { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} - ) - ''' - ) + ''' + tdSql.execute(create_ntb_sql) - tdSql.execute( - f'''insert into ct2 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( - { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } - ) - ( - { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } - ) - ''' - ) + for i in range(ctb_num): + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.{stb} tags ( {i+1} )') + + def __insert_data(self, rows, ctb_num=20, dbname=DBNAME, star_time=NOW): + tdLog.printNoPrefix("==========step: start inser data into tables now.....") + # from ...pytest.util.common import DataSet + data = DataSet() + data.get_order_set(rows) for i in range(rows): - insert_data = f'''insert into t1 values - ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) - ''' - tdSql.execute(insert_data) - tdSql.execute( - f'''insert into t1 values - ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } - ) - ( - { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } - ) + row_data = f''' + {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.utint_data[i]}, + {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]} ''' - ) + tdSql.execute( f"insert into {dbname}.{NTBNAME} values ( {star_time - i * int(TIME_STEP * 1.2)}, {row_data} )" ) + + for j in range(ctb_num): + tdSql.execute( f"insert into {dbname}.ct{j+1} values ( {star_time - j * i * TIME_STEP}, {row_data} )" ) def run(self): tdSql.prepare() @@ -656,27 +628,81 @@ class TDTestCase: with taos_connect(user=self.__user_list[0], passwd=f"new{self.__passwd_list[0]}") as user: # user = conn # 不能创建用户 - tdLog.printNoPrefix("==========step5: normal user can not create user") + tdLog.printNoPrefix("==========step4.1: normal user can not create user") user.error("create use utest1 pass 'utest1pass'") # 可以查看用户 - tdLog.printNoPrefix("==========step6: normal user can show user") + tdLog.printNoPrefix("==========step4.2: normal user can show user") user.query("show users") assert user.queryRows == self.users_count + 1 # 不可以修改其他用户的密码 - tdLog.printNoPrefix("==========step7: normal user can not alter other user pass") + tdLog.printNoPrefix("==========step4.3: normal user can not alter other user pass") user.error(self.__alter_pass_sql(self.__user_list[1], self.__passwd_list[1] )) user.error(self.__alter_pass_sql("root", "taosdata_root" )) # 可以修改自己的密码 - tdLog.printNoPrefix("==========step8: normal user can alter owner pass") + tdLog.printNoPrefix("==========step4.4: normal user can alter owner pass") user.query(self.__alter_pass_sql(self.__user_list[0], self.__passwd_list[0])) # 不可以删除用户，包括自己 - tdLog.printNoPrefix("==========step9: normal user can not drop any user ") + tdLog.printNoPrefix("==========step4.5: normal user can not drop any user ") user.error(f"drop user {self.__user_list[0]}") user.error(f"drop user {self.__user_list[1]}") user.error("drop user root") + tdLog.printNoPrefix("==========step5: enable info") + taos1_conn = taos.connect(user=self.__user_list[1], password=f"new{self.__passwd_list[1]}") + taos1_conn.query(f"show databases") + tdSql.execute(f"alter user {self.__user_list[1]} enable 0") + tdSql.execute(f"alter user {self.__user_list[2]} enable 0") + taos1_except = True + try: + taos1_conn.query("show databases") + except BaseException: + taos1_except = False + if taos1_except: + tdLog.exit("taos 1 connect except error not occured, when enable == 0, should not r/w ") + else: + tdLog.info("taos 1 connect except error occured, enable == 0") + + taos2_except = True + try: + taos.connect(user=self.__user_list[2], password=f"new{self.__passwd_list[2]}") + except BaseException: + taos2_except = False + if taos2_except: + tdLog.exit("taos 2 connect except error not occured, when enable == 0, should not connect") + else: + tdLog.info("taos 2 connect except error occured, enable == 0, can not login") + + tdLog.printNoPrefix("==========step6: sysinfo info") + taos3_conn = taos.connect(user=self.__user_list[3], password=f"new{self.__passwd_list[3]}") + taos3_conn.query(f"show dnodes") + taos3_conn.query(f"show {DBNAME}.vgroups") + tdSql.execute(f"alter user {self.__user_list[3]} sysinfo 0") + tdSql.execute(f"alter user {self.__user_list[4]} sysinfo 0") + taos3_except = True + try: + taos3_conn.query(f"show dnodes") + taos3_conn.query(f"show {DBNAME}.vgroups") + except BaseException: + taos3_except = False + if taos3_except: + tdLog.exit("taos 3 query except error not occured, when sysinfo == 0, should not show info:dnode/monde/qnode ") + else: + tdLog.info("taos 3 query except error occured, sysinfo == 0, can not show dnode/vgroups") + + taos4_conn = taos.connect(user=self.__user_list[4], password=f"new{self.__passwd_list[4]}") + taos4_except = True + try: + taos4_conn.query(f"show mnodes") + taos4_conn.query(f"show {DBNAME}.vgroups") + except BaseException: + taos4_except = False + if taos4_except: + tdLog.exit("taos 4 query except error not occured, when sysinfo == 0, when enable == 0, should not show info:dnode/monde/qnode") + else: + tdLog.info("taos 4 query except error occured, sysinfo == 0, can not show dnode/vgroups") + # root删除用户测试 - tdLog.printNoPrefix("==========step10: super user drop normal user") + tdLog.printNoPrefix("==========step7: super user drop normal user") self.test_drop_user() tdSql.query("show users") diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 20ac3205523af96cec2e7c646c6245c53a55c7e8..3b0fa96b08f73e26e11c35c89d6673268f764ddc 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/system-test/1-insert/performanceInsert.json b/tests/system-test/1-insert/performanceInsert.json index de410c30f2fa1846d0318def447d1d09aff2cfea..1e83217e71719ae75047979ed6e3a506d88bed1b 100644 --- a/tests/system-test/1-insert/performanceInsert.json +++ b/tests/system-test/1-insert/performanceInsert.json @@ -1,23 +1,23 @@ { "filetype": "insert", "cfgdir": "/etc/taos/", - "host": "test216", + "host": "localhost", "port": 6030, "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, - "interlace_rows": 1000, + "interlace_rows": 0, "num_of_records_per_req": 100000, "databases": [ { "dbinfo": { "name": "db", "drop": "yes", - "vgroups": 24 + "vgroups": 8 }, "super_tables": [ { @@ -29,8 +29,8 @@ "batch_create_tbl_num": 50000, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 5, - "interlace_rows": 100000, + "insert_rows": 1000, + "interlace_rows": 0, "insert_interval": 0, "max_sql_len": 10000000, "disorder_ratio": 0, diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index e65dded6013fdecf3075e9fbbbb8e0321e28cb7b..e66f5560dfc2c077bf318934d4c2cad072fb00f7 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime import time from dataclasses import dataclass @@ -8,6 +8,7 @@ from util.sql import * from util.cases import * from util.dnodes import * from util.constant import * +from util.common import * PRIMARY_COL = "ts" @@ -38,7 +39,7 @@ TAG_COL = [INT_TAG] # insert data args： TIME_STEP = 10000 -NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +NOW = int(datetime.timestamp(datetime.now()) * 1000) # init db/table DBNAME = "db" @@ -47,40 +48,6 @@ CTBNAME = "ct1" NTBNAME = "nt1" -@dataclass -class DataSet: - ts_data : List[int] = None - int_data : List[int] = None - bint_data : List[int] = None - sint_data : List[int] = None - tint_data : List[int] = None - int_un_data : List[int] = None - bint_un_data: List[int] = None - sint_un_data: List[int] = None - tint_un_data: List[int] = None - float_data : List[float] = None - double_data : List[float] = None - bool_data : List[int] = None - binary_data : List[str] = None - nchar_data : List[str] = None - - def __post_init__(self): - self.ts_data = [] - self.int_data = [] - self.bint_data = [] - self.sint_data = [] - self.tint_data = [] - self.int_un_data = [] - self.bint_un_data = [] - self.sint_un_data = [] - self.tint_un_data = [] - self.float_data = [] - self.double_data = [] - self.bool_data = [] - self.binary_data = [] - self.nchar_data = [] - - @dataclass class SMAschema: creation : str = "CREATE" @@ -164,10 +131,6 @@ class SMAschema: del self.other[k] - -# from ...pytest.util.sql import * -# from ...pytest.util.constant import * - class TDTestCase: updatecfgDict = {"querySmaOptimize": 1} @@ -469,14 +432,12 @@ class TDTestCase: err_sqls.append( SMAschema(index_flag="SMA INDEX ,", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) err_sqls.append( SMAschema(index_name="tbname", tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) - # current_set cur_sqls.append( SMAschema(max_delay="",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) cur_sqls.append( SMAschema(watermark="",index_name="sma_index_2",tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) cur_sqls.append( SMAschema(sliding="",index_name='sma_index_3',tbname=STBNAME, func=(f"min({INT_COL})",f"max({INT_COL})") ) ) - return err_sqls, cur_sqls def test_create_sma(self): @@ -512,102 +473,48 @@ class TDTestCase: self.test_create_sma() self.test_drop_sma() - pass - - def __create_tb(self): + def __create_tb(self, stb=STBNAME, ctb_num=20, ntbnum=1, dbname=DBNAME): tdLog.printNoPrefix("==========step: create table") - create_stb_sql = f'''create table {STBNAME}( - ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + create_stb_sql = f'''create table {dbname}.{stb}( + {PRIMARY_COL} timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned ) tags ({INT_TAG} int) ''' - create_ntb_sql = f'''create table {NTBNAME}( - ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, - {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, - {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, - {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned - ) - ''' tdSql.execute(create_stb_sql) - tdSql.execute(create_ntb_sql) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - - def __data_set(self, rows): - data_set = DataSet() - - for i in range(rows): - data_set.ts_data.append(NOW + 1 * (rows - i)) - data_set.int_data.append(rows - i) - data_set.bint_data.append(11111 * (rows - i)) - data_set.sint_data.append(111 * (rows - i) % 32767) - data_set.tint_data.append(11 * (rows - i) % 127) - data_set.int_un_data.append(rows - i) - data_set.bint_un_data.append(11111 * (rows - i)) - data_set.sint_un_data.append(111 * (rows - i) % 32767) - data_set.tint_un_data.append(11 * (rows - i) % 127) - data_set.float_data.append(1.11 * (rows - i)) - data_set.double_data.append(1100.0011 * (rows - i)) - data_set.bool_data.append((rows - i) % 2) - data_set.binary_data.append(f'binary{(rows - i)}') - data_set.nchar_data.append(f'nchar_测试_{(rows - i)}') - - return data_set - - def __insert_data(self): + for i in range(ntbnum): + create_ntb_sql = f'''create table {dbname}.nt{i+1}( + {PRIMARY_COL} timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) + ''' + tdSql.execute(create_ntb_sql) + + for i in range(ctb_num): + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.{stb} tags ( {i+1} )') + + def __insert_data(self, rows, ctb_num=20, dbname=DBNAME, star_time=NOW): tdLog.printNoPrefix("==========step: start inser data into tables now.....") - data = self.__data_set(rows=self.rows) + # from ...pytest.util.common import DataSet + data = DataSet() + data.get_order_set(rows, bint_step=2) - # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - null_data = '''null, null, null, null, null, null, null, null, null, null, null, null, null, null''' - zero_data = "0, 0, 0, 0, 0, 0, 0, 'binary_0', 'nchar_0', 0, 0, 0, 0, 0" - - for i in range(self.rows): + for i in range(rows): row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, - {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, - {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} - ''' - neg_row_data = f''' - {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, - {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, - {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.utint_data[i]}, + {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]} ''' + tdSql.execute( f"insert into {dbname}.{NTBNAME} values ( {star_time - i * int(TIME_STEP * 1.2)}, {row_data} )" ) - tdSql.execute( - f"insert into ct1 values ( {NOW - i * TIME_STEP}, {row_data} )") - tdSql.execute( - f"insert into ct2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") - tdSql.execute( - f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") - tdSql.execute( - f"insert into {NTBNAME} values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") - - tdSql.execute( - f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") - tdSql.execute( - f"insert into ct2 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.6)}, {null_data} )") - tdSql.execute( - f"insert into ct2 values ( {NOW - self.rows * int(TIME_STEP * 0.29) }, {null_data} )") - - tdSql.execute( - f"insert into ct4 values ( {NOW + int(TIME_STEP * 0.8)}, {null_data} )") - tdSql.execute( - f"insert into ct4 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.8)}, {null_data} )") - tdSql.execute( - f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") - - tdSql.execute( - f"insert into {NTBNAME} values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") - tdSql.execute( - f"insert into {NTBNAME} values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") - tdSql.execute( - f"insert into {NTBNAME} values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + for j in range(ctb_num): + tdSql.execute( f"insert into {dbname}.ct{j+1} values ( {star_time - j * i * TIME_STEP}, {row_data} )" ) def run(self): self.rows = 10 @@ -616,14 +523,60 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:create table in normal database") tdSql.prepare() - self.__create_tb() - self.__insert_data() + self.__create_tb(dbname=DBNAME) + self.__insert_data(rows=self.rows) self.all_test() + # # from ...pytest.util.sql import * + # drop databases, create same name db、stb and sma index tdSql.prepare() - self.__create_tb() - self.__insert_data() + self.__create_tb(dbname=DBNAME) + self.__insert_data(rows=self.rows,star_time=NOW + self.rows * 2 * TIME_STEP) + tdLog.printNoPrefix("==========step1.1 : create a tsma index and checkdata") + tdSql.execute(f"create sma index {DBNAME}.sma_index_name1 on {DBNAME}.{STBNAME} function(max({INT_COL}),max({BINT_COL}),min({INT_COL})) interval(6m,10s) sliding(6m)") + self.__insert_data(rows=self.rows) + tdSql.query(f"select max({INT_COL}), max({BINT_COL}), min({INT_COL}) from {DBNAME}.{STBNAME} interval(6m,10s) sliding(6m)") + tdSql.checkData(0, 0, self.rows - 1) + tdSql.checkData(0, 1, (self.rows - 1) * 2 ) + tdSql.checkData(tdSql.queryRows - 1, 2, 0) + # tdSql.checkData(0, 2, 0) + + tdLog.printNoPrefix("==========step1.2 : alter table schema, drop col without index") + tdSql.execute(f"alter stable {DBNAME}.{STBNAME} drop column {BINARY_COL}") + tdSql.query(f"select max({INT_COL}), max({BINT_COL}), min({INT_COL}) from {DBNAME}.{STBNAME} interval(6m,10s) sliding(6m)") + tdSql.checkData(0, 0, self.rows - 1) + tdSql.checkData(0, 1, (self.rows - 1) * 2 ) + tdSql.checkData(tdSql.queryRows - 1, 2, 0) + + tdLog.printNoPrefix("==========step1.3 : alter table schema, drop col with index") + # TODO: TD-18047, can not drop col, when col in tsma-index and tsma-index is not dropped. + tdSql.error(f"alter stable {DBNAME}.stb1 drop column {BINT_COL}") + + tdLog.printNoPrefix("==========step1.4 : alter table schema, add col") + tdSql.execute(f"alter stable {DBNAME}.{STBNAME} add column {BINT_COL}_1 bigint") + tdSql.execute(f"insert into {DBNAME}.{CTBNAME} ({PRIMARY_COL}, {BINT_COL}_1) values(now(), 111)") + tdSql.query(f"select max({INT_COL}), max({BINT_COL}), min({INT_COL}) from {DBNAME}.{STBNAME} interval(6m,10s) sliding(6m)") + tdSql.checkData(0, 0, self.rows - 1) + tdSql.checkData(0, 1, (self.rows - 1) * 2 ) + tdSql.checkData(tdSql.queryRows - 1, 2, 0) + # tdSql.checkData(0, 2, 0) + tdSql.query(f"select max({BINT_COL}_1) from {DBNAME}.{STBNAME} ") + tdSql.checkData(0, 0 , 111) + + tdSql.execute(f"flush database {DBNAME}") + + tdLog.printNoPrefix("==========step1.5 : drop child table") + tdSql.execute(f"drop table {CTBNAME}") + tdSql.query(f"select max({INT_COL}), max({BINT_COL}), min({INT_COL}) from {DBNAME}.{STBNAME} interval(6m,10s) sliding(6m)") + tdSql.checkData(0, 0, self.rows - 1) + tdSql.checkData(0, 1, (self.rows - 1) * 2 ) + tdSql.checkData(tdSql.queryRows - 1, 2, 0) + + tdLog.printNoPrefix("==========step1.6 : drop stable") + tdSql.execute(f"drop table {STBNAME}") + tdSql.error(f"select * from {DBNAME}.{STBNAME}") + self.all_test() tdLog.printNoPrefix("==========step2:create table in rollup database") @@ -640,7 +593,6 @@ class TDTestCase: tdSql.execute("flush database db ") - tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index ea7c3329eaf0f3f888589d8b26d66eba522f0474..2afcc29ac892e9cd184a41a139209e1a3bcf2b54 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -295,7 +295,7 @@ class TDTestCase: tdSql.checkData(0, 0, 4.500000000) tdSql.query(f" select avg(c1) from {dbname}.stb1 where c1 is null ") - tdSql.checkRows(0) + tdSql.checkRows(1) def avg_func_filter(self, dbname="db"): diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index ad754ad805192b2f8d45d729614cd2522027a3f7..34164acfa4631868250242f8873fcc50e166e5bb 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -86,7 +86,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2.800000000) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index a7b31a2084af6665a83fac722c8ca1cb653d5dd9..f67088d9e6f1d9091fce226b19d89d35b8f9e23b 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -168,7 +168,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select max(c1) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select max(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,10) diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index cc50092451167bcf72ad57480b03cae370a7b297..29e7c5f519fd4ca7ad11c75083d3ec422c0b5b72 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -167,7 +167,7 @@ class TDTestCase: def distribute_agg_query(self, dbname="testdb"): # basic filter tdSql.query(f"select min(c1) from {dbname}.stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(f"select min(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2) diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index 842a74628d7dbc05f10a99ba49364d8ddc7c2c5d..135a61d1944d72820dd763bc36d0fbccb945fe1b 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -195,7 +195,7 @@ class TDTestCase: def distribute_agg_query(self): # basic filter tdSql.query("select spread(c1) from stb1 where c1 is null") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query("select spread(c1) from stb1 where t1=1") tdSql.checkData(0,0,8.000000000) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index b6bdd8926e4368543595302fd6ccee9f148ef31f..f09265c3009e9bb0a99275bb55ff42ae02a5642a 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -743,7 +743,7 @@ class TDTestCase: # filter data tdSql.query(" select sample(c1, 20 ) from t1 where c1 is null ") - tdSql.checkRows(0) + tdSql.checkRows(1) tdSql.query(" select sample(c1, 20 ) from t1 where c1 =6 ") tdSql.checkRows(1) @@ -891,4 +891,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 75b33f1a5e4561995ee02e70405c6ac88ba91beb..617f7e74643c9b1dbb24834e3535b4bac669e4bb 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -4,6 +4,8 @@ import sys import datetime import inspect import random +from util.dnodes import TDDnode +from util.dnodes import tdDnodes from util.log import * from util.sql import * @@ -30,10 +32,10 @@ class TDTestCase: for i in range(ctbNum): tagValue = 'beijing' if (i % 10 == 0): - sql += " %s%d using %s (name,fleet,driver,device_version) tags('truck_%d', 'South%d','Trish%d','v2.%d')"%(ctbPrefix,i,stbName,i,i,i,i) + sql += " %s%d using %s (name,fleet,driver,device_version,load_capacity,fuel_capacity,nominal_fuel_consumption) tags('truck_%d', 'South%d','Trish%d','v2.%d', 1500+%d*20, 150+%d*2, 5+%d)"%(ctbPrefix,i,stbName,i,i,i,i,(1500+i*20),(150+i*2),(5+i)) else: model = 'H-%d'%i - sql += " %s%d using %s tags('truck_%d', 'South%d','Trish%d','%s','v2.%d')"%(ctbPrefix,i,stbName,i,i,i,model,i) + sql += " %s%d using %s tags('truck_%d', 'South%d','Trish%d','%s','v2.%d', %d, %d,%d)"%(ctbPrefix,i,stbName,i,i,i,model,i,(1500+i*20),(150+i*2),(5+i)) if (i > 0) and (i%1000 == 0): tsql.execute(sql) sql = pre_create @@ -55,10 +57,10 @@ class TDTestCase: sql += " %s%d values "%(ctbPrefix,i) for j in range(rowsPerTbl): if(ctbPrefix=="rct"): - sql += f"({startTs+j*60000}, {80+j}, {90+j}, {85+j}, {30+j*10}, {1.2*j}, {221+j*2}, {20+j*0.2}, {1500+j*20}, {150+j*2},{5+j}) " + sql += f"({startTs+j*60000}, {80+j}, {90+j}, {85+j}, {30+j*10}, {1.2*j}, {221+j*2}, {20+j*0.2}) " elif ( ctbPrefix=="dct"): status= random.randint(0,1) - sql += f"( {startTs+j*60000}, {1+j*0.1},{1400+j*15}, {status},{1500+j*20}, {150+j*2},{5+j} ) " + sql += f"( {startTs+j*60000}, {1+j*0.1},{1400+j*15}, {status} ) " # tdLog.debug("1insert sql:%s"%sql) if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): # tdLog.debug("2insert sql:%s"%sql) @@ -79,22 +81,22 @@ class TDTestCase: stabname2="diagnostics" ctbnamePre1="rct" ctbnamePre2="dct" - ctbNums=40 + ctbNums=50 self.ctbNums=ctbNums - rowNUms=200 + rowNUms=5000 ts=1451606400000 tdSql.execute(f"create database {dbname};") tdSql.execute(f"use {dbname} ") tdSql.execute(f''' - create table {stabname1} (ts timestamp,latitude double,longitude double,elevation double,velocity double,heading double,grade double,fuel_consumption double,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)); + create table {stabname1} (ts timestamp,latitude double,longitude double,elevation double,velocity double,heading double,grade double,fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30),load_capacity double,fuel_capacity double,nominal_fuel_consumption double); ''') tdSql.execute(f''' - create table {stabname2} (ts timestamp,fuel_state double,current_load double,status bigint,load_capacity double,fuel_capacity double,nominal_fuel_consumption double) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30)) ; + create table {stabname2} (ts timestamp,fuel_state double,current_load double,status bigint) tags (name binary(30),fleet binary(30),driver binary(30),model binary(30),device_version binary(30),load_capacity double,fuel_capacity double,nominal_fuel_consumption double) ; ''') self.create_ctable(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums) self.create_ctable(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums) - self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=1000) - self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=1000) + self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname1,ctbPrefix=ctbnamePre1,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=10000) + self.insertData(tsql=tdSql,dbName=dbname,stbName=stabname2,ctbPrefix=ctbnamePre2,ctbNum=ctbNums,rowsPerTbl=rowNUms,startTs=ts,batchNum=10000) # for i in range(ctbNum): # if i %10 == 0 : # # tdLog.debug(f"create table rct{i} using readings (name,fleet,driver,model,device_version) tags ('truck_{i}','South{i}','Trish{i}', NULL,'v2.3')") @@ -131,7 +133,7 @@ class TDTestCase: # tdLog.info("avg value check pass , it work as expected ,sql is \"%s\" "%check_query ) - def tsbsIotQuery(self): + def tsbsIotQuery(self,insertinto=True): tdSql.execute("use db_tsbs") @@ -143,10 +145,11 @@ class TDTestCase: # test insert into - tdSql.execute("create table testsnode (ts timestamp, c1 float,c2 binary(30),c3 binary(30),c4 binary(30)) ;") - tdSql.query("insert into testsnode SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") - - tdSql.query("insert into testsnode(ts,c1,c2,c3,c4) SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") + if insertinto == True : + tdSql.execute("create table testsnode (ts timestamp, c1 float,c2 binary(30),c3 binary(30),c4 binary(30)) ;") + tdSql.query("insert into testsnode SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") + + tdSql.query("insert into testsnode(ts,c1,c2,c3,c4) SELECT ts,avg(velocity) as mean_velocity,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet,ts interval(10m);") # test paitition interval fill @@ -182,8 +185,7 @@ class TDTestCase: # # 6. avg-daily-driving-session # #taosc core dumped - # tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") - # tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query(" SELECT _wstart as ts,name,floor(avg(velocity)/5) AS mv FROM readings WHERE name is not null AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0);") # tdSql.query("select name,diff(mv) AS difka FROM (SELECT ts,name,mv FROM (SELECT _wstart as ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0))) group BY name ;") # tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") @@ -234,7 +236,9 @@ class TDTestCase: tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() self.tsbsIotQuery() - + tdDnodes.stop(1) + tdDnodes.start(1) + self.tsbsIotQuery(False) def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 77484e715617c8a3e6269556f048fcbbaf13e7f7..0428014291204d5314ec5386503e22c5a1fba170 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -196,15 +196,15 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 # vnode case -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py -N 4 -M 1 -# python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 +python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py -N 4 -M 1 diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4526ff2230a8b05e72c06bcdd60f0c98c2dbaa9a..f0bda821725bc00ca49c77ab43c4aca4a8d89d5b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,7 +22,8 @@ static bool shellIsEmptyCommand(const char *cmd); static int32_t shellRunSingleCommand(char *command); -static int32_t shellRunCommand(char *command); +static void shellRecordCommandToHistory(char *command); +static int32_t shellRunCommand(char *command, bool recordHistory); static void shellRunSingleCommandImp(char *command); static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); @@ -101,11 +102,7 @@ int32_t shellRunSingleCommand(char *command) { return 0; } -int32_t shellRunCommand(char *command) { - if (shellIsEmptyCommand(command)) { - return 0; - } - +void shellRecordCommandToHistory(char *command) { SShellHistory *pHistory = &shell.history; if (pHistory->hstart == pHistory->hend || pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || @@ -120,6 +117,14 @@ int32_t shellRunCommand(char *command) { pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } +} + +int32_t shellRunCommand(char *command, bool recordHistory) { + if (shellIsEmptyCommand(command)) { + return 0; + } + + if (recordHistory) shellRecordCommandToHistory(command); char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { @@ -826,11 +831,15 @@ void shellSourceFile(const char *file) { size_t cmd_len = 0; char *line = NULL; char fullname[PATH_MAX] = {0}; + char sourceFileCommand[PATH_MAX + 8] = {0}; if (taosExpandDir(file, fullname, PATH_MAX) != 0) { tstrncpy(fullname, file, PATH_MAX); } + sprintf(sourceFileCommand, "source %s;",fullname); + shellRecordCommandToHistory(sourceFileCommand); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "failed to open file %s\r\n", fullname); @@ -853,9 +862,13 @@ void shellSourceFile(const char *file) { continue; } + if (line[read_len - 1] == '\r') { + line[read_len - 1] = ' '; + } + memcpy(cmd + cmd_len, line, read_len); printf("%s%s\r\n", shell.info.promptHeader, cmd); - shellRunCommand(cmd); + shellRunCommand(cmd, false); memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } @@ -977,7 +990,7 @@ void *shellThreadLoop(void *arg) { } taosResetTerminalMode(); - } while (shellRunCommand(command) == 0); + } while (shellRunCommand(command, true) == 0); taosMemoryFreeClear(command); shellWriteHistory(); @@ -1019,7 +1032,7 @@ int32_t shellExecute() { if (pArgs->commands != NULL) { printf("%s%s\r\n", shell.info.promptHeader, pArgs->commands); char *cmd = strdup(pArgs->commands); - shellRunCommand(cmd); + shellRunCommand(cmd, true); taosMemoryFree(cmd); }