Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/TD-14761

.clang-format

@@ -88,4 +88,3 @@ Standard:        Auto
 TabWidth:        8
 UseTab:          Never
 ...
-

.gitattributes

+*.py linguist-detectable=false

.gitignore

 build/
 compile_commands.json
+CMakeSettings.json
 .cache
 .ycm_extra_conf.py
 .tasks

CMakeSettings.json

-{
-  "configurations": [
-    {
-      "name": "WSL-GCC-Debug",
-      "generator": "Unix Makefiles",
-      "configurationType": "Debug",
-      "buildRoot": "${projectDir}\\build\\",
-      "installRoot": "${projectDir}\\build\\",
-      "cmakeExecutable": "/usr/bin/cmake",
-      "cmakeCommandArgs": "",
-      "buildCommandArgs": "",
-      "ctestCommandArgs": "",
-      "inheritEnvironments": [ "linux_x64" ],
-      "wslPath": "${defaultWSLPath}",
-      "addressSanitizerRuntimeFlags": "detect_leaks=0",
-      "variables": [
-        {
-          "name": "CMAKE_INSTALL_PREFIX",
-          "value": "/mnt/d/TDengine/TDengine/build",
-          "type": "PATH"
-        }
-      ]
-    }
-  ]
-}
\ No newline at end of file

cmake/cmake.platform

@@ -46,7 +46,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
 
         MESSAGE("Current system processor is ${CMAKE_SYSTEM_PROCESSOR}.")
         IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
-            MESSAGE("Current system arch is arm64")
+            MESSAGE("Current system arch is 64")
             SET(TD_DARWIN_64 TRUE)
             ADD_DEFINITIONS("-D_TD_DARWIN_64")
         ENDIF ()

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG aa45ad4
+        GIT_TAG 9cb965f
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/01-index.md

@@ -4,25 +4,24 @@ sidebar_label: Documentation Home
 slug: /
 ---
 
-
-TDengine is an [open source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud native](https://tdengine.com/tdengine/cloud-native-time-series-database/) time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors.  This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design and other topics. It’s written mainly for architects, developers and system administrators.
+TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) time-series database optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators.
 
 To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section.
 
-TDengine greatly improves the efficiency of data ingestion, querying and storage by exploiting the characteristics of time series data, introducing the novel concepts of "one table for one data collection point" and "super table", and designing an innovative storage engine. To understand the new concepts in TDengine and make full use of the features and capabilities of TDengine, please read [“Concepts”](./concept) thoroughly.
+TDengine greatly improves the efficiency of data ingestion, querying, and storage by exploiting the characteristics of time series data, introducing the novel concepts of "one table for one data collection point" and "super table", and designing an innovative storage engine. To understand the new concepts in TDengine and make full use of the features and capabilities of TDengine, please read [Concepts](./concept) thoroughly.
 
-If you are a developer, please read the [“Developer Guide”](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, make a few changes to accommodate your application, and it will work.
+If you are a developer, please read the [Developer Guide](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, and make a few changes to accommodate your application, and it will work.
 
-We live in the era of big data, and scale-up is unable to meet the growing needs of business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to ["cluster deployment"](../deployment).
+We live in the era of big data, and scale-up is unable to meet the growing needs of the business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to [Cluster Deployment](../deployment).
 
-TDengine uses ubiquitious SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll up, interpolation and time weighted average, among many others. The ["SQL Reference"](./taos-sql) chapter describes the SQL syntax in detail, and lists the various supported commands and functions.
+TDengine uses ubiquitous SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll-up, interpolation, and time-weighted average, among many others. The [SQL Reference](./taos-sql) chapter describes the SQL syntax in detail and lists the various supported commands and functions.
 
-If you are a system administrator who cares about installation, upgrade, fault tolerance, disaster recovery, data import, data export, system configuration, how to monitor whether TDengine is running healthily, and how to improve system performance, please refer to, and thoroughly read the ["Administration"](./operation) section.
+If you are a system administrator who cares about installation, upgrade, fault tolerance, disaster recovery, data import, data export, system configuration, how to monitor whether TDengine is running healthily, and how to improve system performance, please refer to, and thoroughly read the [Administration](./operation) section.
 
-If you want to know more about TDengine tools, the REST API, and connectors for various programming languages, please see the ["Reference"](./reference) chapter.
+If you want to know more about TDengine tools, the REST API, and connectors for various programming languages, please see the [Reference](./reference) chapter.
 
-If you are very interested in the internal design of TDengine, please read the chapter ["Inside TDengine”](./tdinternal), which introduces the cluster design, data partitioning, sharding, writing, and reading processes in detail. If you want to study TDengine code or even contribute code, please read this chapter carefully.
+If you are very interested in the internal design of TDengine, please read the chapter [Inside TDengine](./tdinternal), which introduces the cluster design, data partitioning, sharding, writing, and reading processes in detail. If you want to study TDengine code or even contribute code, please read this chapter carefully.
 
-TDengine is an open source database, and we would love for you to be a part of TDengine. If you find any errors in the documentation, or see parts where more clarity or elaboration is needed, please click "Edit this page" at the bottom of each page to edit it directly.
+TDengine is an open-source database, and we would love for you to be a part of TDengine. If you find any errors in the documentation or see parts where more clarity or elaboration is needed, please click "Edit this page" at the bottom of each page to edit it directly.
 
-Together, we make a difference.
+Together, we make a difference!

docs/en/07-develop/01-connect/index.md

 ---
-title: Connect
-description: "This document explains how to establish connections to TDengine and how to install and use TDengine connectors."
+sidebar_label: Connect
+title: Connect to TDengine
+description: "How to establish connections to TDengine and how to install and use TDengine connectors."
 ---
 
 import Tabs from "@theme/Tabs";

docs/en/07-develop/03-insert-data/05-high-volume.md

@@ -16,7 +16,7 @@ To achieve high performance writing, there are a few aspects to consider. In the
 
 From the perspective of application program, you need to consider:
 
-1. The data size of each single write, also known as batch size. Generally speaking, higher batch size generates better writing performance. However, once the batch size is over a specific value, you will not get any additional benefit anymore. When using SQL to write into TDengine, it's better to put as much as possible data in single SQL. The maximum SQL length supported by TDengine is 1,048,576 bytes, i.e. 1 MB. It can be configured by parameter `maxSQLLength` on client side, and the default value is 65,480.
+1. The data size of each single write, also known as batch size. Generally speaking, higher batch size generates better writing performance. However, once the batch size is over a specific value, you will not get any additional benefit anymore. When using SQL to write into TDengine, it's better to put as much as possible data in single SQL. The maximum SQL length supported by TDengine is 1,048,576 bytes, i.e. 1 MB. 
 
 2. The number of concurrent connections. Normally more connections can get better result. However, once the number of connections exceeds the processing ability of the server side, the performance may downgrade.
 
@@ -46,12 +46,9 @@ If the data source is Kafka, then the appication program is a consumer of Kafka,
 
 ### Tune TDengine
 
-TDengine is a distributed and high performance time series database, there are also some ways to tune TDengine to get better writing performance.
+On the server side, database configuration parameter `vgroups` needs to be set carefully to maximize the system performance. If it's set too low, the system capability can't be utilized fully; if it's set too big, unnecessary resource competition may be produced. A normal recommendation for `vgroups` parameter is 2 times of the number of CPU cores. However, depending on the actual system resources, it may still need to tuned.
 
-1. Set proper number of `vgroups` according to available CPU cores. Normally, we recommend 2 \* number_of_cores as a starting point. If the verification result shows this is not enough to utilize CPU resources, you can use a higher value.
-2. Set proper `minTablesPerVnode`, `tableIncStepPerVnode`, and `maxVgroupsPerDb` according to the number of tables so that tables are distributed even across vgroups. The purpose is to balance the workload among all vnodes so that system resources can be utilized better to get higher performance.
-
-For more performance tuning parameters, please refer to [Configuration Parameters](../../../reference/config).
+For more configuration parameters, please refer to [Database Configuration](../../../taos-sql/database) and [Server Configuration](../../../reference/config)。
 
 ## Sample Programs
 
@@ -359,7 +356,7 @@ Writing process tries to read as much as possible data from message queue and wr
 
 <details>
 
-SQLWriter class encapsulates the logic of composing SQL and writing data. Please be noted that the tables have not been created before writing, but are created automatically when catching the exception of table doesn't exist. For other exceptions caught, the SQL which caused the exception are logged for you to debug. This class also checks the SQL length, if the SQL length is closed to `maxSQLLength` the SQL will be executed immediately. To improve writing efficiency, it's better to increase `maxSQLLength` properly.
+SQLWriter class encapsulates the logic of composing SQL and writing data. Please be noted that the tables have not been created before writing, but are created automatically when catching the exception of table doesn't exist. For other exceptions caught, the SQL which caused the exception are logged for you to debug. This class also checks the SQL length, and passes the maximum SQL length by parameter maxSQLLength according to actual TDengine limit. 
 
 <summary>SQLWriter</summary>
 

docs/en/12-taos-sql/14-stream.md

@@ -50,7 +50,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
 ## Delete a Stream
 
 ```sql
-DROP STREAM [IF NOT EXISTS] stream_name
+DROP STREAM [IF EXISTS] stream_name
 ```
 
 This statement deletes the stream processing service only. The data generated by the stream is retained.

docs/en/12-taos-sql/22-meta.md

@@ -245,3 +245,35 @@ Provides dnode configuration information.
 | 1   | dnode_id | INT          | Dnode ID |
 | 2   |   name   | BINARY(32)   | Parameter   |
 | 3   |  value   | BINARY(64)   | Value |
+
+## INS_TOPICS
+
+| #   |    **Column**    | **Data Type** | **Description**                  |
+| --- | :---------: | ------------ | ------------------------------ |
+| 1   | topic_name  | BINARY(192)  | Topic name                     |
+| 2   |   db_name   | BINARY(64)   | Database for the topic                |
+| 3   | create_time | TIMESTAMP    | Creation time              |
+| 4   |     sql     | BINARY(1024) | SQL statement used to create the topic |
+
+## INS_SUBSCRIPTIONS
+
+| #   |    **Column**    | **Data Type** | **Description**                  |
+| --- | :------------: | ------------ | ------------------------ |
+| 1   |   topic_name   | BINARY(204)  | Subscribed topic           |
+| 2   | consumer_group | BINARY(193)  | Subscribed consumer group         |
+| 3   |   vgroup_id    | INT          | Vgroup ID for the consumer |
+| 4   |  consumer_id   | BIGINT       | Consumer ID          |
+
+## INS_STREAMS
+
+| #   |    **Column**    | **Data Type** | **Description**                  |
+| --- | :----------: | ------------ | --------------------------------------- |
+| 1   | stream_name  | BINARY(64)   | Stream name                              |
+| 2   | create_time  | TIMESTAMP    | Creation time                                |
+| 3   |     sql      | BINARY(1024) | SQL statement used to create the stream             |
+| 4   |    status    | BIANRY(20)   | Current status                              |
+| 5   |  source_db   | BINARY(64)   | Source database                                |
+| 6   |  target_db   | BIANRY(64)   | Target database                              |
+| 7   | target_table | BINARY(192)  | Target table                      |
+| 8   |  watermark   | BIGINT       | Watermark (see stream processing documentation)        |
+| 9   |   trigger    | INT          | Method of triggering the result push (see stream processing documentation) |

docs/en/12-taos-sql/23-perf.md

@@ -61,15 +61,6 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE
 | 12  |  sub_status  | BINARY(1000) | Subquery status                   |
 | 13  |     sql      | BINARY(1024) | SQL statement                     |
 
-## PERF_TOPICS
-
-| #   |    **Column**    | **Data Type** | **Description**                  |
-| --- | :---------: | ------------ | ------------------------------ |
-| 1   | topic_name  | BINARY(192)  | Topic name                     |
-| 2   |   db_name   | BINARY(64)   | Database for the topic                |
-| 3   | create_time | TIMESTAMP    | Creation time              |
-| 4   |     sql     | BINARY(1024) | SQL statement used to create the topic |
-
 ## PERF_CONSUMERS
 
 | #   |    **Column**    | **Data Type** | **Description**                  |
@@ -83,15 +74,6 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE
 | 7   | subscribe_time | TIMESTAMP    | Time of first subscription                                        |
 | 8   | rebalance_time | TIMESTAMP    | Time of first rebalance triggering                                 |
 
-## PERF_SUBSCRIPTIONS
-
-| #   |    **Column**    | **Data Type** | **Description**                  |
-| --- | :------------: | ------------ | ------------------------ |
-| 1   |   topic_name   | BINARY(204)  | Subscribed topic           |
-| 2   | consumer_group | BINARY(193)  | Subscribed consumer group         |
-| 3   |   vgroup_id    | INT          | Vgroup ID for the consumer |
-| 4   |  consumer_id   | BIGINT       | Consumer ID          |
-
 ## PERF_TRANS
 
 | #   |    **Column**    | **Data Type** | **Description**                  |
@@ -113,17 +95,3 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE
 | 2   | create_time | TIMESTAMP    | Creation time                                |
 | 3   | stable_name | BINARY(192)  | Supertable name                        |
 | 4   |  vgroup_id  | INT          | Dedicated vgroup name                      |
-
-## PERF_STREAMS
-
-| #   |    **Column**    | **Data Type** | **Description**                  |
-| --- | :----------: | ------------ | --------------------------------------- |
-| 1   | stream_name  | BINARY(64)   | Stream name                              |
-| 2   | create_time  | TIMESTAMP    | Creation time                                |
-| 3   |     sql      | BINARY(1024) | SQL statement used to create the stream             |
-| 4   |    status    | BIANRY(20)   | Current status                              |
-| 5   |  source_db   | BINARY(64)   | Source database                                |
-| 6   |  target_db   | BIANRY(64)   | Target database                              |
-| 7   | target_table | BINARY(192)  | Target table                      |
-| 8   |  watermark   | BIGINT       | Watermark (see stream processing documentation)        |
-| 9   |   trigger    | INT          | Method of triggering the result push (see stream processing documentation) |

docs/en/12-taos-sql/25-grant.md

 ---
-sidebar_label: Permissions Management
-title: Permissions Management
+sidebar_label: Access Control
+title: User and Access Control
+description: Manage user and user's permission
 ---
 
 This document describes how to manage permissions in TDengine.

docs/en/14-reference/03-connector/05-go.mdx

@@ -7,7 +7,7 @@ title: TDengine Go Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import GoInsert from "../../07-develop/03-insert-data/_go_sql.mdx"
 import GoInfluxLine from "../../07-develop/03-insert-data/_go_line.mdx"
 import GoOpenTSDBTelnet from "../../07-develop/03-insert-data/_go_opts_telnet.mdx"

docs/en/14-reference/03-connector/06-rust.mdx

@@ -7,7 +7,7 @@ title: TDengine Rust Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import RustInsert from "../../07-develop/03-insert-data/_rust_sql.mdx"
 import RustBind from "../../07-develop/03-insert-data/_rust_stmt.mdx"
 import RustQuery from "../../07-develop/04-query-data/_rust.mdx"

docs/en/14-reference/03-connector/07-python.mdx

@@ -7,7 +7,7 @@ description: "taospy is the official Python connector for TDengine. taospy provi
 import Tabs from "@theme/Tabs";
 import TabItem from "@theme/TabItem";
 
-`taospy is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. `taospy` wraps both the [native interface](/reference/connector/cpp) and [REST interface](/reference/rest-api) of TDengine, which correspond to the `taos` and `taosrest` modules of the `taospy` package, respectively.
+`taospy` is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. `taospy` wraps both the [native interface](/reference/connector/cpp) and [REST interface](/reference/rest-api) of TDengine, which correspond to the `taos` and `taosrest` modules of the `taospy` package, respectively.
 In addition to wrapping the native and REST interfaces, `taospy` also provides a set of programming interfaces that conforms to the [Python Data Access Specification (PEP 249)](https://peps.python.org/pep-0249/). It is easy to integrate `taospy` with many third-party tools, such as [SQLAlchemy](https://www.sqlalchemy.org/) and [pandas](https://pandas.pydata.org/).
 
 The direct connection to the server using the native interface provided by the client driver is referred to hereinafter as a "native connection"; the connection to the server using the REST interface provided by taosAdapter is referred to hereinafter as a "REST connection".

docs/en/14-reference/03-connector/08-node.mdx

@@ -7,7 +7,7 @@ title: TDengine Node.js Connector
 import Tabs from "@theme/Tabs";
 import TabItem from "@theme/TabItem";
 
-import Preparition from "./_preparition.mdx";
+import Preparition from "./_preparation.mdx";
 import NodeInsert from "../../07-develop/03-insert-data/_js_sql.mdx";
 import NodeInfluxLine from "../../07-develop/03-insert-data/_js_line.mdx";
 import NodeOpenTSDBTelnet from "../../07-develop/03-insert-data/_js_opts_telnet.mdx";

docs/en/14-reference/03-connector/09-csharp.mdx

@@ -7,7 +7,7 @@ title: C# Connector
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-import Preparition from "./_preparition.mdx"
+import Preparition from "./_preparation.mdx"
 import CSInsert from "../../07-develop/03-insert-data/_cs_sql.mdx"
 import CSInfluxLine from "../../07-develop/03-insert-data/_cs_line.mdx"
 import CSOpenTSDBTelnet from "../../07-develop/03-insert-data/_cs_opts_telnet.mdx"

docs/en/14-reference/03-connector/_preparition.mdx

-- 已安装客户端驱动（使用原生连接必须安装，使用 REST 连接无需安装）
-
-:::info
-
-由于 TDengine 的客户端驱动使用 C 语言编写，使用原生连接时需要加载系统对应安装在本地的客户端驱动共享库文件，通常包含在 TDengine 安装包。TDengine Linux 服务端安装包附带了 TDengine 客户端，也可以单独安装 [Linux 客户端](/get-started/) 。在 Windows 环境开发时需要安装 TDengine 对应的 [Windows 客户端](https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client) 。
-
-- libtaos.so: 在 Linux 系统中成功安装 TDengine 后，依赖的 Linux 版客户端驱动 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
-- taos.dll: 在 Windows 系统中安装完客户端之后，依赖的 Windows 版客户端驱动 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
-
-:::

docs/en/14-reference/14-taosKeeper.md

 ---
 sidebar_label: taosKeeper
 title: taosKeeper
-description: Instructions and tips for using taosKeeper
+description: exports TDengine monitoring metrics.
 ---
 
 ## Introduction
@@ -22,26 +22,35 @@ You can compile taosKeeper separately and install it. Please refer to the [taosK
 
 ### Configuration and running methods
 
-<!-- taosKeeper needs to be executed on the terminal of the operating system, it supports two configuration methods: [Command-line arguments](#command-line-arguments-in-detail) and [configuration file](#configuration-file-parameters-in-detail). Command-line arguments take precedence over values in the configuration file. -->
-taosKeeper needs to be executed on the terminal of the operating system. To run taosKeeper, see [configuration file](#configuration-file-parameters-in-detail).
+taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: [Command-line arguments](#command-line-arguments-in-detail), [environment variable](#environment-variable-in-detail) and [configuration file](#configuration-file-parameters-in-detail). The precedence of those is Command-line, environment variable and configuration file.
 
 **Make sure that the TDengine cluster is running correctly before running taosKeeper. ** Ensure that the monitoring service in TDengine has been started. For more information, see [TDengine Monitoring Configuration](../config/#monitoring).
 
-<!--
 ### Command-Line Parameters
 
-You can use command-line parameters to run taosBenchmark and control its behavior:
+You can use command-line parameters to run taosKeeper and control its behavior:
 
 ```shell
-taosKeeper
+$ taosKeeper
 ```
--->
+### Environment variable
+
+You can use Environment variable to run taosKeeper and control its behavior:
+
+```shell
+$ export TAOS_KEEPER_TDENGINE_HOST=192.168.64.3
+ 
+$ taoskeeper
+```
+
+you can run `taoskeeper -h` for more detail.
+
 ### Configuration File
 
 You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. 
 
 ```shell
-taoskeeper -c <keeper config file>
+$ taoskeeper -c <keeper config file>
 ```
 
 **Sample configuration files**
@@ -110,7 +119,7 @@ Query OK, 1 rows in database (0.036162s)
 #### Export Monitoring Metrics
 
 ```shell
-curl http://127.0.0.1:6043/metrics
+$ curl http://127.0.0.1:6043/metrics
 ```
 
 Sample result set (excerpt):

docs/en/20-third-party/12-google-data-studio.md

+---
+sidebar_label: Google Data Studio
+title: Use Google Data Studio to access TDengine
+---
+
+Data Studio is a powerful tool for reporting and visualization, offering a wide variety of charts and connectors and making it easy to generate reports based on predefined templates. Its ease of use and robust ecosystem have made it one of the first choices for people working in data analysis.
+
+TDengine is a high-performance, scalable time-series database that supports SQL. Many businesses and developers in fields spanning from IoT and Industry Internet to IT and finance are using TDengine as their time-series database management solution.
+
+The TDengine team immediately saw the benefits of using TDengine to process time-series data with Data Studio to analyze it, and they got to work to create a connector for Data Studio.
+
+With the release of the TDengine connector in Data Studio, you can now get even more out of your data. To obtain the connector, first go to the Data Studio Connector Gallery, click Connect to Data, and search for “TDengine”.
+
+![02](gds/gds-02.png.webp)
+
+Select the TDengine connector and click Authorize.
+
+![03](gds/gds-03.png.webp)
+
+Then sign in to your Google Account and click Allow to enable the connection to TDengine.
+
+![04](gds/gds-04.png.webp)
+
+In the Enter URL field, type the hostname and port of the server running the TDengine REST service. In the following fields, type your username, password, database name, table name, and the start and end times of your query range. Then, click Connect.
+
+![05](gds/gds-05.png.webp)
+
+After the connection is established, you can use Data Studio to process your data and create reports.
+
+![06](gds/gds-06.png.webp)
+
+In Data Studio, TDengine timestamps and tags are considered dimensions, and all other items are considered metrics. You can create all kinds of custom charts with your data – some examples are shown below.
+
+![07](gds/gds-07.png.webp)
+
+With the ability to process petabytes of data per day and provide monitoring and alerting in real time, TDengine is a great solution for time-series data management. Now, with the Data Studio connector, we’re sure you’ll be able to gain new insights and obtain even more value from your data.

docs/zh/01-index.md

@@ -4,22 +4,22 @@ sidebar_label: 文档首页
 slug: /
 ---
 
-TDengine是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、工业互联网、金融等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一极简的时序数据处理平台。本文档是 TDengine 用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发者与系统管理员的。
+TDengine 是一款[开源](https://www.taosdata.com/tdengine/open_source_time-series_database)、[高性能](https://www.taosdata.com/fast)、[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)的<a href="https://www.taosdata.com/" data-internallinksmanager029f6b8e52c="2" title="时序数据库" target="_blank" rel="noopener">时序数据库</a>（<a href="https://www.taosdata.com/time-series-database" data-internallinksmanager029f6b8e52c="9" title="Time Series DataBase" target="_blank" rel="noopener">Time Series Database</a>, <a href="https://www.taosdata.com/tsdb" data-internallinksmanager029f6b8e52c="8" title="TSDB" target="_blank" rel="noopener">TSDB</a>）, 它专为物联网、车联网、工业互联网、金融、IT 运维等场景优化设计。同时它还带有内建的缓存、流式计算、数据订阅等系统功能，能大幅减少系统设计的复杂度，降低研发和运营成本，是一款极简的时序数据处理平台。本文档是 TDengine 的用户手册，主要是介绍 TDengine 的基本概念、安装、使用、功能、开发接口、运营维护、TDengine 内核设计等等，它主要是面向架构师、开发工程师与系统管理员的。
 
-TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用TDengine, 无论如何，请您仔细阅读[基本概念](./concept)一章。
+TDengine 充分利用了时序数据的特点，提出了“一个数据采集点一张表”与“超级表”的概念，设计了创新的存储引擎，让数据的写入、查询和存储效率都得到极大的提升。为正确理解并使用 TDengine，无论如何，请您仔细阅读[基本概念](./concept)一章。
 
-如果你是开发者，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要把示例代码拷贝粘贴，针对自己的应用稍作改动，就能跑起来。
+如果你是开发工程师，请一定仔细阅读[开发指南](./develop)一章，该部分对数据库连接、建模、插入数据、查询、流式计算、缓存、数据订阅、用户自定义函数等功能都做了详细介绍，并配有各种编程语言的示例代码。大部分情况下，你只要复制粘贴示例代码，针对自己的应用稍作改动，就能跑起来。
 
-我们已经生活在大数据的时代，纵向扩展已经无法满足日益增长的业务需求，任何系统都必须具有水平扩展的能力，集群成为大数据以及 database 系统的不可缺失功能。TDengine 团队不仅实现了集群功能，而且将这一重要核心功能开源。怎么部署、管理和维护 TDengine 集群，请参考[部署集群](./deployment)一章。
+我们已经生活在大数据时代，纵向扩展已经无法满足日益增长的业务需求，任何系统都必须具有水平扩展的能力，集群成为大数据以及 Database 系统的不可缺失功能。TDengine 团队不仅实现了集群功能，而且将这一重要核心功能开源。怎么部署、管理和维护 TDengine 集群，请仔细参考[部署集群](./deployment)一章。
 
-TDengine 采用 SQL 作为其查询语言，大大降低学习成本、降低迁移成本，但同时针对时序数据场景，又做了一些扩展，以支持插值、降采样、时间加权平均等操作。[SQL 手册](./taos-sql)一章详细描述了 SQL 语法、详细列出了各种支持的命令和函数。
+TDengine 采用 SQL 作为查询语言，大大降低学习成本、降低迁移成本，但同时针对时序数据场景，又做了一些扩展，以支持插值、降采样、时间加权平均等操作。[SQL 手册](./taos-sql)一章详细描述了 SQL 语法、详细列出了各种支持的命令和函数。
 
-如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出，配置参数，怎么监测 TDengine 是否健康运行，怎么提升系统运行的性能，那么请仔细参考[运维指南](./operation)一章。
+如果你是系统管理员，关心安装、升级、容错灾备、关心数据导入、导出、配置参数，如何监测 TDengine 是否健康运行，如何提升系统运行的性能，请仔细参考[运维指南](./operation)一章。
 
-如果你对 TDengine 外围工具，REST API, 各种编程语言的连接器想做更多详细了解，请看[参考指南](./reference)一章。
+如果你对 TDengine 的外围工具、REST API、各种编程语言的连接器（Connector）想做更多详细了解，请看[参考指南](./reference)一章。
 
-如果你对 TDengine 内部的架构设计很有兴趣，欢迎仔细阅读[技术内幕](./tdinternal)一章，里面对集群的设计、数据分区、分片、写入、读出、查询、聚合查询的流程都做了详细的介绍。如果你想研读 TDengine 代码甚至贡献代码，请一定仔细读完这一章。
+如果你对 TDengine 的内部架构设计很有兴趣，欢迎仔细阅读[技术内幕](./tdinternal)一章，里面对集群的设计、数据分区、分片、写入、读出、查询、聚合查询的流程都做了详细的介绍。如果你想研读 TDengine 代码甚至贡献代码，请一定仔细读完这一章。
 
-最后，作为一个开源软件，欢迎大家的参与。如果发现文档的任何错误，描述不清晰的地方，都请在每个页面的最下方，点击“编辑本文档“直接进行修改。
+最后，作为一个开源软件，欢迎大家的参与。如果发现文档有任何错误、描述不清晰的地方，请在每个页面的最下方，点击“编辑本文档”直接进行修改。
 
 Together, we make a difference!

docs/zh/02-intro.md

@@ -4,53 +4,53 @@ description: 简要介绍 TDengine 的主要功能
 toc_max_heading_level: 2
 ---
 
-TDengine 是一款开源、高性能、云原生的[时序数据库](https://tdengine.com/tsdb/)，且针对物联网、车联网以及工业互联网进行了优化。TDengine 的代码，包括其集群功能，都在 GNU AGPL v3.0 下开源。除核心的时序数据库功能外，TDengine 还提供[缓存](../develop/cache/)、[数据订阅](../develop/tmq)、[流式计算](../develop/stream)等其它功能以降低系统复杂度及研发和运维成本。
+TDengine 是一款开源、高性能、云原生的[时序数据库](https://tdengine.com/tsdb/)，且针对物联网、车联网、工业互联网、金融、IT 运维等场景进行了优化。TDengine 的代码，包括集群功能，都在 GNU AGPL v3.0 下开源。除核心的时序数据库功能外，TDengine 还提供[缓存](../develop/cache/)、[数据订阅](../develop/tmq)、[流式计算](../develop/stream)等其它功能以降低系统复杂度及研发和运维成本。
 
-本章节介绍TDengine的主要功能、竞争优势、适用场景、与其他数据库的对比测试等等，让大家对TDengine有个整体的了解。
+本章节介绍 TDengine 的主要功能、竞争优势、适用场景、与其他数据库的对比测试等等，让大家对 TDengine 有个整体的了解。
 
 ## 主要功能
 
-TDengine的主要功能如下：
+TDengine 的主要功能如下：
 
 1. 写入数据，支持
-    - [SQL 写入](../develop/insert-data/sql-writing)
-    - [Schemaless 写入](../reference/schemaless/)，支持多种标准写入协议
-        - [InfluxDB LINE 协议](../develop/insert-data/influxdb-line)
-        - [OpenTSDB Telnet 协议](../develop/insert-data/opentsdb-telnet)
-        - [OpenTSDB JSON 协议](../develop/insert-data/opentsdb-json)
-    - 与多种第三方工具的无缝集成，它们都可以仅通过配置而无需任何代码即可将数据写入 TDengine
-        - [Telegraf](../third-party/telegraf)
-        - [Prometheus](../third-party/prometheus)
-        - [StatsD](../third-party/statsd)
-        - [collectd](../third-party/collectd)
-        - [icinga2](../third-party/icinga2)
-        - [TCollector](../third-party/tcollector)
-        - [EMQ](../third-party/emq-broker)
-        - [HiveMQ](../third-party/hive-mq-broker) ；
+   - [SQL 写入](../develop/insert-data/sql-writing)
+   - [无模式（Schemaless）写入](../reference/schemaless/)，支持多种标准写入协议
+     - [InfluxDB Line 协议](../develop/insert-data/influxdb-line)
+     - [OpenTSDB Telnet 协议](../develop/insert-data/opentsdb-telnet)
+     - [OpenTSDB JSON 协议](../develop/insert-data/opentsdb-json)
+   - 与多种第三方工具的无缝集成，它们都可以仅通过配置而无需任何代码即可将数据写入 TDengine
+     - [Telegraf](../third-party/telegraf)
+     - [Prometheus](../third-party/prometheus)
+     - [StatsD](../third-party/statsd)
+     - [collectd](../third-party/collectd)
+     - [Icinga2](../third-party/icinga2)
+     - [TCollector](../third-party/tcollector)
+     - [EMQX](../third-party/emq-broker)
+     - [HiveMQ](../third-party/hive-mq-broker)
 2. 查询数据，支持
-    - [标准SQL](../taos-sql)，含嵌套查询
-    - [时序数据特色函数](../taos-sql/function/#time-series-extensions)
-    - [时序顺序特色查询](../taos-sql/distinguished)，例如降采样、插值、累加和、时间加权平均、状态窗口、会话窗口等
-    - [用户自定义函数](../taos-sql/udf)
+   - [标准 SQL](../taos-sql)，含嵌套查询
+   - [时序数据特色函数](../taos-sql/function/#time-series-extensions)
+   - [时序数据特色查询](../taos-sql/distinguished)，例如降采样、插值、累加和、时间加权平均、状态窗口、会话窗口等
+   - [用户自定义函数（UDF）](../taos-sql/udf)
 3. [缓存](../develop/cache)，将每张表的最后一条记录缓存起来，这样无需 Redis 就能对时序数据进行高效处理
-4. [流式计算](../develop/stream)(Stream Processing)，TDengine 不仅支持连续查询，还支持事件驱动的流式计算，这样在处理时序数据时就无需 Flink 或 Spark 这样流计算组件
-5. [数据订阅](../develop/tmq)，应用程序可以订阅一张表或一组表的数据，API 与 Kafka 相同，而且可以指定过滤条件
+4. [流式计算（Stream Processing）](../develop/stream)，TDengine 不仅支持连续查询，还支持事件驱动的流式计算，这样在处理时序数据时就无需 Flink 或 Spark 这样流式计算组件
+5. [数据订阅](../develop/tmq)，应用程序可以订阅一张表或一组表的数据，提供与 Kafka 相同的 API，而且可以指定过滤条件
 6. 可视化
-    - 支持与 [Grafana](../third-party/grafana/) 的无缝集成
-    - 支持与 Google Data Studio 的无缝集成
+   - 支持与 [Grafana](../third-party/grafana/) 的无缝集成
+   - 支持与 Google Data Studio 的无缝集成
 7. 集群
-    - 集群部署(../deployment/)，可以通过增加节点进行水平扩展以提升处理能力
-    - 可以通过 [Kubernets 部署 TDengine](../deployment/k8s/)
-    - 通过多副本提供高可用能力
+   - [集群部署](../deployment/)，可以通过增加节点进行水平扩展以提升处理能力
+   - 可以通过 [Kubernetes 部署 TDengine](../deployment/k8s/)
+   - 通过多副本提供高可用能力
 8. 管理
-    - [监控](../operation/monitor)运行中的 TDengine 实例
-    - 多种[数据导入](../operation/import)方式
-    - 多种[数据导出](../operation/export)方式
+   - [监控](../operation/monitor)运行中的 TDengine 实例
+   - 多种[数据导入](../operation/import)方式
+   - 多种[数据导出](../operation/export)方式
 9. 工具
-    - 提供交互式[命令行程序](../reference/taos-shell)，便于管理集群，检查系统状态，做即席查询
-    - 提供压力测试工具[taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
+   - 提供[交互式命令行程序（CLI）](../reference/taos-shell)，便于管理集群，检查系统状态，做即席查询
+   - 提供压力测试工具[taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能
 10. 编程
-    - 提供各种语言的[连接器](../connector): 如 [C/C++](../connector/cpp), [Java](../connector/java), [Go](../connector/go), [Node.JS](../connector/node), [Rust](../connector/rust), [Python](../connector/python), [C#](../connector/csharp) 等
+    - 提供各种语言的[连接器（Connector）](../connector): 如 [C/C++](../connector/cpp)、[Java](../connector/java)、[Go](../connector/go)、[Node.js](../connector/node)、[Rust](../connector/rust)、[Python](../connector/python)、[C#](../connector/csharp) 等
     - 支持 [REST 接口](../connector/rest-api/)
 
 更多细节功能，请阅读整个文档。
@@ -63,36 +63,36 @@ TDengine的主要功能如下：
 
 - **[极简时序数据平台](https://www.taosdata.com/tdengine/simplified_solution_for_time-series_data_processing)**：TDengine 内建缓存、流式计算和数据订阅等功能，为时序数据的处理提供了极简的解决方案，从而大幅降低了业务系统的设计复杂度和运维成本。
 
-- **[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)**：通过原生的分布式设计、数据分片和分区、存算分离、RAFT 协议、Kubernets 部署和完整的可观测性，TDengine 是一款云原生时序数据库并且能够部署在公有云、私有云和混合云上。
+- **[云原生](https://www.taosdata.com/tdengine/cloud_native_time-series_database)**：通过原生的分布式设计、数据分片和分区、存算分离、RAFT 协议、Kubernetes 部署和完整的可观测性，TDengine 是一款云原生时序数据库并且能够部署在公有云、私有云和混合云上。
 
-- **[简单易用](https://www.taosdata.com/tdengine/ease_of_use)**：对系统管理员来说，TDengine 大幅降低了管理和维护的代价。对开发者来说， TDengine 提供了简单的接口、极简的解决方案和与第三方工具的无缝集成。对数据分析专家来说，TDengine 提供了便捷的数据访问。
+- **[简单易用](https://www.taosdata.com/tdengine/ease_of_use)**：对系统管理员来说，TDengine 大幅降低了管理和维护的代价。对开发者来说， TDengine 提供了简单的接口、极简的解决方案和与第三方工具的无缝集成。对数据分析专家来说，TDengine 提供了便捷的数据访问能力。
 
 - **[分析能力](https://www.taosdata.com/tdengine/easy_data_analytics)**：通过超级表、存储计算分离、分区分片、预计算和其它技术，TDengine 能够高效地浏览、格式化和访问数据。
 
-- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 19k，且拥有一个活跃的开发者社区。 
+- **[核心开源](https://www.taosdata.com/tdengine/open_source_time-series_database)**：TDengine 的核心代码包括集群功能全部在开源协议下公开。全球超过 140k 个运行实例，GitHub Star 19k，且拥有一个活跃的开发者社区。
 
 采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面：
 
-1. 由于其超强性能，它能将系统需要的计算资源和存储资源大幅降低
+1. 由于其超强性能，它能将系统所需的计算资源和存储资源大幅降低
 2. 因为支持 SQL，能与众多第三方软件无缝集成，学习迁移成本大幅下降
-3. 因为是一极简的时序数据平台，系统复杂度、研发和运营成本大幅降低
+3. 因为是一款极简的时序数据平台，系统复杂度、研发和运营成本大幅降低
 
 ## 技术生态
 
-在整个时序大数据平台中，TDengine 在其中扮演的角色如下：
+在整个时序大数据平台中，TDengine 扮演的角色如下：
 
 <figure>
 
 ![TDengine Database 技术生态图](eco_system.webp)
 
+<center><figcaption>图 1. TDengine 技术生态图</figcaption></center>
 </figure>
-<center>图 1. TDengine技术生态图</center>
 
-上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka, 他们的数据将被源源不断的写入到 TDengine。右侧则是可视化、BI 工具、组态软件、应用程序。下侧则是 TDengine 自身提供的命令行程序 (CLI) 以及可视化管理管理。
+上图中，左侧是各种数据采集或消息队列，包括 OPC-UA、MQTT、Telegraf、也包括 Kafka，他们的数据将被源源不断的写入到 TDengine。右侧则是可视化、BI 工具、组态软件、应用程序。下侧则是 TDengine 自身提供的命令行程序（CLI）以及可视化管理工具。
 
 ## 典型适用场景
 
-作为一个高性能、分布式、支持 SQL 的时序数据库 （Database)，TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。本文对适用场景做更多详细的分析。
+作为一个高性能、分布式、支持 SQL 的时序数据库（Database），TDengine 的典型适用场景包括但不限于 IoT、工业互联网、车联网、IT 运维、能源、金融证券等领域。需要指出的是，TDengine 是针对时序数据场景设计的专用数据库和专用大数据处理工具，因其充分利用了时序大数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM 等通用型数据。下面本文将对适用场景做更多详细的分析。
 
 ### 数据源特点和需求
 
@@ -114,18 +114,18 @@ TDengine的主要功能如下：
 
 ### 系统功能需求
 
-| 系统功能需求               | 不适用 | 可能适用 | 非常适用 | 简单说明                                                                                                              |
-| -------------------------- | ------ | -------- | -------- | --------------------------------------------------------------------------------------------------------------------- |
-| 要求完整的内置数据处理算法 |        | √        |          | TDengine 的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。 |
-| 需要大量的交叉查询处理     |        | √        |          | 这种类型的处理更多应该用关系型数据系统处理，或者应该考虑 TDengine 和关系型数据系统配合实现系统功能。                  |
+| 系统功能需求               | 不适用 | 可能适用 | 非常适用 | 简单说明                                                                                                                  |
+| -------------------------- | ------ | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------- |
+| 要求完整的内置数据处理算法 |        | √        |          | TDengine 实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有需求，因此特殊类型的处理需求还需要在应用层面解决。 |
+| 需要大量的交叉查询处理     |        | √        |          | 这种类型的处理更多应该用关系型数据库处理，或者应该考虑 TDengine 和关系型数据库配合实现系统功能。                          |
 
 ### 系统性能需求
 
-| 系统性能需求           | 不适用 | 可能适用 | 非常适用 | 简单说明                                                                                               |
-| ---------------------- | ------ | -------- | -------- | ------------------------------------------------------------------------------------------------------ |
-| 要求较大的总体处理能力 |        |          | √        | TDengine 的集群功能可以轻松地让多服务器配合达成处理能力的提升。                                        |
-| 要求高速处理数据       |        |          | √        | TDengine 的专门为 IoT 优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。 |
-| 要求快速处理小粒度数据 |        |          | √        | 这方面 TDengine 性能可以完全对标关系型和 NoSQL 型数据处理系统。                                        |
+| 系统性能需求           | 不适用 | 可能适用 | 非常适用 | 简单说明                                                                                           |
+| ---------------------- | ------ | -------- | -------- | -------------------------------------------------------------------------------------------------- |
+| 要求较大的总体处理能力 |        |          | √        | TDengine 的集群功能可以轻松地让多服务器配合达成处理能力的提升。                                    |
+| 要求高速处理数据       |        |          | √        | TDengine 专门为 IoT 优化的存储和数据处理设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。 |
+| 要求快速处理小粒度数据 |        |          | √        | 这方面 TDengine 性能可以完全对标关系型和 NoSQL 型数据处理系统。                                    |
 
 ### 系统维护需求
 

docs/zh/07-develop/03-insert-data/05-high-volume.md

@@ -11,7 +11,7 @@ import TabItem from "@theme/TabItem";
 
 从客户端程序的角度来说，高效写入数据要考虑以下几个因素：
 
-1. 单次写入的数据量。一般来讲，每批次写入的数据量越大越高效（但超过一定阈值其优势会消失）。使用 SQL 写入 TDengine 时，尽量在一条 SQL 中拼接更多数据。目前，TDengine 支持的一条 SQL 的最大长度为 1,048,576（1M）个字符。可通过配置客户端参数 maxSQLLength（默认值为 65480）进行修改。
+1. 单次写入的数据量。一般来讲，每批次写入的数据量越大越高效（但超过一定阈值其优势会消失）。使用 SQL 写入 TDengine 时，尽量在一条 SQL 中拼接更多数据。目前，TDengine 支持的一条 SQL 的最大长度为 1,048,576（1M）个字符。
 2. 并发连接数。一般来讲，同时写入数据的并发连接数越多写入越高效（但超过一定阈值反而会下降，取决于服务端处理能力）。
 3. 数据在不同表（或子表）之间的分布，即要写入数据的相邻性。一般来说，每批次只向同一张表（或子表）写入数据比向多张表（或子表）写入数据要更高效；
 4. 写入方式。一般来讲：
@@ -38,13 +38,9 @@ import TabItem from "@theme/TabItem";
 
 ### 服务器配置的角度 {#setting-view}
 
-从服务器配置的角度来说，也有很多优化写入性能的方法。
+从服务端配置的角度，要根据系统中磁盘的数量，磁盘的 I/O 能力，以及处理器能力在创建数据库时设置适当的 vgroups 数量以充分发挥系统性能。如果 vgroups 过少，则系统性能无法发挥；如果 vgroups 过多，会造成无谓的资源竞争。常规推荐 vgroups 数量为 CPU 核数的 2 倍，但仍然要结合具体的系统资源配置进行调优。
 
-如果总表数不多(远小于核数乘以1000), 且无论怎么调节客户端程序，taosd 进程的 CPU 使用率都很低，那么很可能是因为表在各个 vgroup 分布不均。比如：数据库总表数是 1000 且 minTablesPerVnode 设置的也是 1000，那么所有的表都会分布在 1 个 vgroup 上。此时如果将 minTablesPerVnode 和 tablelncStepPerVnode 都设置成 100， 则可将表分布至 10 个 vgroup。（假设 maxVgroupsPerDb 大于等于 10）。
-
-如果总表数比较大（比如大于500万），适当增加 maxVgroupsPerDb 也能显著提高建表的速度。maxVgroupsPerDb 默认值为 0， 自动配置为 CPU 的核数。 如果表的数量巨大，也建议调节 maxTablesPerVnode 参数，以免超过单个 vnode 建表的上限。
-
-更多调优参数，请参考 [配置参考](../../../reference/config)部分。
+更多调优参数，请参考 [数据库管理](../../../taos-sql/database) 和 [服务端配置](../../../reference/config)。
 
 ## 高效写入示例 {#sample-code}
 
@@ -352,7 +348,7 @@ main 函数可以接收 5 个启动参数，依次是：
 
 <details>
 
-SQLWriter 类封装了拼 SQL 和写数据的逻辑。所有的表都没有提前创建，而是在发生表不存在错误的时候，再以超级表为模板批量建表，然后重新执行 INSERT 语句。对于其它错误会记录当时执行的 SQL， 以便排查错误和故障恢复。这个类也对 SQL 是否超过最大长度限制做了检查，如果接近 SQL 最大长度限制（maxSQLLength），将会立即执行 SQL。为了减少 SQL 此时，建议将 maxSQLLength 适当调大。
+SQLWriter 类封装了拼 SQL 和写数据的逻辑。所有的表都没有提前创建，而是在发生表不存在错误的时候，再以超级表为模板批量建表，然后重新执行 INSERT 语句。对于其它错误会记录当时执行的 SQL， 以便排查错误和故障恢复。这个类也对 SQL 是否超过最大长度限制做了检查，根据 TDengine 3.0 的限制由输入参数 maxSQLLength 传入了支持的最大 SQL 长度，即 1048576 。
 
 <summary>SQLWriter</summary>
 

docs/zh/12-taos-sql/10-function.md

@@ -917,7 +917,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
 
 **返回数据类型**：同应用的字段。
 
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
 
 **适用于**：表和超级表。
 
@@ -932,7 +932,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
 
 **返回数据类型**：同应用的字段。
 
-**适用数据类型**：数值类型。
+**适用数据类型**：数值类型，时间戳类型。
 
 **适用于**：表和超级表。
 

docs/zh/12-taos-sql/14-stream.md

@@ -58,7 +58,7 @@ SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVA
 ## 删除流式计算
 
 ```sql
-DROP STREAM [IF NOT EXISTS] stream_name;
+DROP STREAM [IF EXISTS] stream_name;
 ```
 
 仅删除流式计算任务，由流式计算写入的数据不会被删除。

docs/zh/12-taos-sql/22-meta.md

@@ -246,3 +246,35 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用，所以它们
 | 1   | dnode_id | INT          | dnode 的 ID  |
 | 2   |   name   | BINARY(32)   | 配置项名称   |
 | 3   |  value   | BINARY(64)   | 该配置项的值 |
+
+## INS_TOPICS
+
+| #   |  **列名**   | **数据类型** | **说明**                       |
+| --- | :---------: | ------------ | ------------------------------ |
+| 1   | topic_name  | BINARY(192)  | topic 名称                     |
+| 2   |   db_name   | BINARY(64)   | topic 相关的 DB                |
+| 3   | create_time | TIMESTAMP    | topic 的 创建时间              |
+| 4   |     sql     | BINARY(1024) | 创建该 topic 时所用的 SQL 语句 |
+
+## INS_SUBSCRIPTIONS
+
+| #   |    **列名**    | **数据类型** | **说明**                 |
+| --- | :------------: | ------------ | ------------------------ |
+| 1   |   topic_name   | BINARY(204)  | 被订阅的 topic           |
+| 2   | consumer_group | BINARY(193)  | 订阅者的消费者组         |
+| 3   |   vgroup_id    | INT          | 消费者被分配的 vgroup id |
+| 4   |  consumer_id   | BIGINT       | 消费者的唯一 id          |
+
+## INS_STREAMS
+
+| #   |   **列名**   | **数据类型** | **说明**                                |
+| --- | :----------: | ------------ | --------------------------------------- |
+| 1   | stream_name  | BINARY(64)   | 流计算名称                              |
+| 2   | create_time  | TIMESTAMP    | 创建时间                                |
+| 3   |     sql      | BINARY(1024) | 创建流计算时提供的 SQL 语句             |
+| 4   |    status    | BIANRY(20)   | 流当前状态                              |
+| 5   |  source_db   | BINARY(64)   | 源数据库                                |
+| 6   |  target_db   | BIANRY(64)   | 目的数据库                              |
+| 7   | target_table | BINARY(192)  | 流计算写入的目标表                      |
+| 8   |  watermark   | BIGINT       | watermark，详见 SQL 手册流式计算        |
+| 9   |   trigger    | INT          | 计算结果推送模式，详见 SQL 手册流式计算 |

docs/zh/12-taos-sql/23-perf.md

@@ -62,15 +62,6 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其
 | 12  |  sub_status  | BINARY(1000) | 子查询状态                   |
 | 13  |     sql      | BINARY(1024) | SQL 语句                     |
 
-## PERF_TOPICS
-
-| #   |  **列名**   | **数据类型** | **说明**                       |
-| --- | :---------: | ------------ | ------------------------------ |
-| 1   | topic_name  | BINARY(192)  | topic 名称                     |
-| 2   |   db_name   | BINARY(64)   | topic 相关的 DB                |
-| 3   | create_time | TIMESTAMP    | topic 的 创建时间              |
-| 4   |     sql     | BINARY(1024) | 创建该 topic 时所用的 SQL 语句 |
-
 ## PERF_CONSUMERS
 
 | #   |    **列名**    | **数据类型** | **说明**                                                    |
@@ -84,15 +75,6 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其
 | 7   | subscribe_time | TIMESTAMP    | 上一次发起订阅的时间                                        |
 | 8   | rebalance_time | TIMESTAMP    | 上一次触发 rebalance 的时间                                 |
 
-## PERF_SUBSCRIPTIONS
-
-| #   |    **列名**    | **数据类型** | **说明**                 |
-| --- | :------------: | ------------ | ------------------------ |
-| 1   |   topic_name   | BINARY(204)  | 被订阅的 topic           |
-| 2   | consumer_group | BINARY(193)  | 订阅者的消费者组         |
-| 3   |   vgroup_id    | INT          | 消费者被分配的 vgroup id |
-| 4   |  consumer_id   | BIGINT       | 消费者的唯一 id          |
-
 ## PERF_TRANS
 
 | #   |     **列名**     | **数据类型** | **说明**                                                       |
@@ -114,17 +96,3 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其
 | 2   | create_time | TIMESTAMP    | sma 创建时间                                |
 | 3   | stable_name | BINARY(192)  | sma 所属的超级表名称                        |
 | 4   |  vgroup_id  | INT          | sma 专属的 vgroup 名称                      |
-
-## PERF_STREAMS
-
-| #   |   **列名**   | **数据类型** | **说明**                                |
-| --- | :----------: | ------------ | --------------------------------------- |
-| 1   | stream_name  | BINARY(64)   | 流计算名称                              |
-| 2   | create_time  | TIMESTAMP    | 创建时间                                |
-| 3   |     sql      | BINARY(1024) | 创建流计算时提供的 SQL 语句             |
-| 4   |    status    | BIANRY(20)   | 流当前状态                              |
-| 5   |  source_db   | BINARY(64)   | 源数据库                                |
-| 6   |  target_db   | BIANRY(64)   | 目的数据库                              |
-| 7   | target_table | BINARY(192)  | 流计算写入的目标表                      |
-| 8   |  watermark   | BIGINT       | watermark，详见 SQL 手册流式计算        |
-| 9   |   trigger    | INT          | 计算结果推送模式，详见 SQL 手册流式计算 |

docs/zh/14-reference/05-taosbenchmark.md

@@ -87,7 +87,7 @@ taosBenchmark -f <json file>
 <summary>subscribe.json</summary>
 
 ```json
-{{#include /taos-tools/example/subscribe.json}}
+{{#include /taos-tools/example/tmq.json}}
 ```
 
 </details>
@@ -405,37 +405,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 
 订阅子表或者普通表的配置参数在 `specified_table_query` 中设置。
 
-- **threads** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
+- **threads/concurrent** : 执行 SQL 的线程数，默认为 1。
 
 - **sqls** ：
   - **sql** : 执行的 SQL 命令，必填。
-  - **result** : 保存查询结果的文件，未指定则不保存。
-
-#### 订阅超级表的配置参数
-
-订阅超级表的配置参数在 `super_table_query` 中设置。
-
-- **stblname** : 要订阅的超级表名称，必填。
-
-- **threads** : 执行 SQL 的线程数，默认为 1。
-
-- **interval** : 执行订阅的时间间隔，单位为秒，默认为 0。
-
-- **restart** : "yes" 表示开始新的订阅，"no" 表示继续之前的订阅，默认值为 "no"。
-
-- **keepProgress** : "yes" 表示保留订阅进度，"no" 表示不保留，默认值为 "no"。
-
-- **resubAfterConsume** : "yes" 表示取消之前的订阅然后再次订阅， "no" 表示继续之前的订阅，默认值为 "no"。
-
-- **sqls** ：
-  - **sql** : 执行的 SQL 命令，必填；对于超级表的查询 SQL，在 SQL 命令中保留 "xxxx"，程序会自动将其替换为超级表的所有子表名。
-    替换为超级表中所有的子表名。
-  - **result** : 保存查询结果的文件，未指定则不保存。

docs/zh/14-reference/11-docker/index.md

@@ -119,7 +119,7 @@ taos -h tdengine -P 6030
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
 ENV TDENGINE_VERSION=3.0.0.0
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.tdengine.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -234,7 +234,7 @@ go mod tidy
 ```dockerfile
 FROM golang:1.19.0-buster as builder
 ENV TDENGINE_VERSION=3.0.0.0
-RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -250,7 +250,7 @@ RUN go build
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
 ENV TDENGINE_VERSION=3.0.0.0
-RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.tdengine.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \

docs/zh/14-reference/14-taosKeeper.md

 ---
 sidebar_label: taosKeeper
 title: taosKeeper
-description: TDengine taosKeeper 使用说明
+description: TDengine 3.0 版本监控指标的导出工具
 ---
 
 ## 简介
@@ -22,26 +22,36 @@ taosKeeper 安装方式：
 
 ### 配置和运行方式
 
-<!-- taosKeeper 需要在操作系统终端执行，该工具支持两种配置方式：[命令行参数](#命令行参数启动) 和 [配置文件](#配置文件启动)。命令行参数优先级高于配置文件参数。-->
-taosKeeper 需要在操作系统终端执行，该工具支持 [配置文件启动](#配置文件启动)。
+taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：[命令行参数](#命令行参数启动)、[环境变量](#环境变量启动) 和 [配置文件](#配置文件启动)。优先级为：命令行参数、环境变量、配置文件参数。
 
 **在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** 并且 TDengine 已经开启监控服务，具体请参考：[TDengine 监控配置](../config/#监控相关)。
 
-<!--
 ### 命令行参数启动
 
-在使用命令行参数运行 taosBenchmark 并控制其行为。
+在使用命令行参数运行 taosKeeper 并控制其行为。
 
 ```shell
-taosKeeper
+$ taosKeeper
 ```
--->
+
+### 环境变量启动
+
+通过设置环境变量达到控制启动参数的目的，通常在容器中运行时使用。
+
+```shell
+$ export TAOS_KEEPER_TDENGINE_HOST=192.168.64.3
+ 
+$ taoskeeper
+```
+
+具体参数列表请参照 `taoskeeper -h` 输入结果。
+
 ### 配置文件启动
 
 执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 
 
 ```shell
-taoskeeper -c <keeper config file>
+$ taoskeeper -c <keeper config file>
 ```
 
 **下面是配置文件的示例：**
@@ -110,7 +120,7 @@ Query OK, 1 rows in database (0.036162s)
 #### 导出监控指标
 
 ```shell
-curl http://127.0.0.1:6043/metrics
+$ curl http://127.0.0.1:6043/metrics
 ```
 
 部分结果集：

docs/zh/20-third-party/12-google-data-studio.md

+---
+sidebar_label: Google Data Studio
+title: TDengine Google Data Studio Connector
+description: 使用 Google Data Studio 存取 TDengine 数据的详细指南
+---
+
+Google Data Studio 是一个强大的报表可视化工具，它提供了丰富的数据图表和数据连接，可以非常方便地按照既定模板生成报表。因其简便易用和生态丰富而在数据分析领域得到一众数据科学家的青睐。
+
+Data Studio 可以支持多种数据来源，除了诸如 Google Analytics、Google AdWords、Search Console、BigQuery 等 Google 自己的服务之外，用户也可以直接将离线文件上传至 Google Cloud Storage，或是通过连接器来接入其它数据源。
+
+![01](gds/gds-01.webp)
+
+目前 TDengine 连接器已经发布到 Google Data Studio 应用商店，你可以在 “Connect to Data” 页面下直接搜索 TDengine，将其选作数据源。
+
+![02](gds/gds-02.png.webp)
+
+接下来选择 AUTHORIZE 按钮。
+
+![03](gds/gds-03.png.webp)
+
+设置允许连接自己的账号到外部服务。
+
+![04](gds/gds-04.png.webp)
+
+在接下来的页面选择运行 TDengine REST 服务的 URL，并输入用户名、密码、数据库名称、表名称以及查询时间范围，并点击右上角的 CONNECT 按钮。
+
+![05](gds/gds-05.png.webp)
+
+连接成功后，就可以使用 GDS 方便地进行数据处理并创建报表了。
+
+![06](gds/gds-06.png.webp)
+
+目前的维度和指标规则是：timestamp 类型的字段和 tag 字段会被连接器定义为维度，而其他类型的字段是指标。用户还可以根据自己的需求创建不同的表。
+
+![07](gds/gds-07.png.webp)
+![08](gds/gds-08.png.webp)
+![09](gds/gds-09.png.webp)
+![10](gds/gds-10.png.webp)
+![11](gds/gds-11.png.webp)

include/client/taos.h

@@ -254,6 +254,7 @@ enum tmq_res_t {
   TMQ_RES_INVALID = -1,
   TMQ_RES_DATA = 1,
   TMQ_RES_TABLE_META = 2,
+  TMQ_RES_TAOSX = 3,
 };
 
 typedef struct tmq_raw_data {

include/common/systable.h

@@ -43,17 +43,17 @@ extern "C" {
 #define TSDB_INS_TABLE_VNODES            "ins_vnodes"
 #define TSDB_INS_TABLE_CONFIGS           "ins_configs"
 #define TSDB_INS_TABLE_DNODE_VARIABLES   "ins_dnode_variables"
+#define TSDB_INS_TABLE_SUBSCRIPTIONS     "ins_subscriptions"
+#define TSDB_INS_TABLE_TOPICS            "ins_topics"
+#define TSDB_INS_TABLE_STREAMS           "ins_streams"
 
 #define TSDB_PERFORMANCE_SCHEMA_DB     "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS          "perf_smas"
 #define TSDB_PERFS_TABLE_CONNECTIONS   "perf_connections"
 #define TSDB_PERFS_TABLE_QUERIES       "perf_queries"
-#define TSDB_PERFS_TABLE_TOPICS        "perf_topics"
 #define TSDB_PERFS_TABLE_CONSUMERS     "perf_consumers"
-#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "perf_subscriptions"
 #define TSDB_PERFS_TABLE_OFFSETS       "perf_offsets"
 #define TSDB_PERFS_TABLE_TRANS         "perf_trans"
-#define TSDB_PERFS_TABLE_STREAMS       "perf_streams"
 #define TSDB_PERFS_TABLE_APPS          "perf_apps"
 
 typedef struct SSysDbTableSchema {

include/common/tcommon.h

@@ -73,6 +73,7 @@ enum {
   TMQ_MSG_TYPE__POLL_RSP,
   TMQ_MSG_TYPE__POLL_META_RSP,
   TMQ_MSG_TYPE__EP_RSP,
+  TMQ_MSG_TYPE__TAOSX_RSP,
   TMQ_MSG_TYPE__END_RSP,
 };
 
@@ -129,7 +130,6 @@ typedef struct SDataBlockInfo {
   uint32_t    capacity;
   // TODO: optimize and remove following
   int64_t     version;    // used for stream, and need serialization
-  int64_t     ts;         // used for stream, and need serialization
   int32_t     childId;    // used for stream, do not serialize
   EStreamType type;       // used for stream, do not serialize
   STimeWindow calWin;     // used for stream, do not serialize

include/common/tglobal.h

@@ -144,8 +144,8 @@ void    taosCfgDynamicOptions(const char *option, const char *value);
 
 struct SConfig *taosGetCfg();
 
-void    taosSetAllDebugFlag(int32_t flag);
-void    taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal);
+void    taosSetAllDebugFlag(int32_t flag, bool rewrite);
+void    taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite);
 int32_t taosSetCfg(SConfig *pCfg, char *name);
 void    taosLocalCfgForbiddenToChange(char* name, bool* forbidden);
 

include/common/tmsg.h

@@ -276,7 +276,6 @@ struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 };
 
-
 typedef struct {
   char     tbName[TSDB_TABLE_NAME_LEN];
   char     stbName[TSDB_TABLE_NAME_LEN];
@@ -295,17 +294,15 @@ typedef struct {
   SSchema* pSchemas;
 } STableMetaRsp;
 
-
-
 typedef struct {
-  int32_t code;
-  int8_t  hashMeta;
-  int64_t uid;
-  char*   tblFName;
-  int32_t numOfRows;
-  int32_t affectedRows;
-  int64_t sver;
-  STableMetaRsp* pMeta;  
+  int32_t        code;
+  int8_t         hashMeta;
+  int64_t        uid;
+  char*          tblFName;
+  int32_t        numOfRows;
+  int32_t        affectedRows;
+  int64_t        sver;
+  STableMetaRsp* pMeta;
 } SSubmitBlkRsp;
 
 typedef struct {
@@ -320,7 +317,7 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitBlkRsp(void* param);
+void    tFreeSSubmitBlkRsp(void* param);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON     ((int8_t)0x1)
@@ -2049,8 +2046,8 @@ typedef struct {
   STableMetaRsp* pMeta;
 } SVCreateTbRsp, SVUpdateTbRsp;
 
-int tEncodeSVCreateTbRsp(SEncoder* pCoder, const SVCreateTbRsp* pRsp);
-int tDecodeSVCreateTbRsp(SDecoder* pCoder, SVCreateTbRsp* pRsp);
+int  tEncodeSVCreateTbRsp(SEncoder* pCoder, const SVCreateTbRsp* pRsp);
+int  tDecodeSVCreateTbRsp(SDecoder* pCoder, SVCreateTbRsp* pRsp);
 void tFreeSVCreateTbRsp(void* param);
 
 int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
@@ -2630,6 +2627,22 @@ typedef struct {
   };
 } STqOffsetVal;
 
+static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) {
+  pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
+  pOffsetVal->uid = uid;
+  pOffsetVal->ts = ts;
+}
+
+static FORCE_INLINE void tqOffsetResetToMeta(STqOffsetVal* pOffsetVal, int64_t uid) {
+  pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_META;
+  pOffsetVal->uid = uid;
+}
+
+static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) {
+  pOffsetVal->type = TMQ_OFFSET__LOG;
+  pOffsetVal->version = ver;
+}
+
 int32_t tEncodeSTqOffsetVal(SEncoder* pEncoder, const STqOffsetVal* pOffsetVal);
 int32_t tDecodeSTqOffsetVal(SDecoder* pDecoder, STqOffsetVal* pOffsetVal);
 int32_t tFormatOffset(char* buf, int32_t maxLen, const STqOffsetVal* pVal);
@@ -2961,6 +2974,26 @@ typedef struct {
 
 int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);
 int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp);
+void    tDeleteSMqDataRsp(SMqDataRsp* pRsp);
+
+typedef struct {
+  SMqRspHead   head;
+  STqOffsetVal reqOffset;
+  STqOffsetVal rspOffset;
+  int32_t      blockNum;
+  int8_t       withTbName;
+  int8_t       withSchema;
+  SArray*      blockDataLen;
+  SArray*      blockData;
+  SArray*      blockTbName;
+  SArray*      blockSchema;
+  int32_t      createTableNum;
+  SArray*      createTableLen;
+  SArray*      createTableReq;
+} STaosxRsp;
+
+int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const STaosxRsp* pRsp);
+int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, STaosxRsp* pRsp);
 
 typedef struct {
   SMqRspHead head;

include/common/ttypes.h

@@ -49,9 +49,6 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t) \
-  (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
-#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -268,11 +265,16 @@ typedef struct {
 #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
 #define IS_FLOAT_TYPE(_t)            ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)
 #define IS_INTEGER_TYPE(_t)          ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
+#define IS_TIMESTAMP_TYPE(_t)        ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
 
 #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
 #define IS_MATHABLE_TYPE(_t) \
   (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
 
+#define IS_VAR_DATA_TYPE(t) \
+  (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
+#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)

include/libs/function/function.h

@@ -92,6 +92,8 @@ struct SResultRowEntryInfo;
 //for selectivity query, the corresponding tag value is assigned if the data is qualified
 typedef struct SSubsidiaryResInfo {
   int16_t num;
+  int32_t rowLen;
+  char*   buf;         // serialize data buffer
   struct SqlFunctionCtx **pCtx;
 } SSubsidiaryResInfo;
 
@@ -118,6 +120,11 @@ typedef struct SInputColumnInfoData {
   uint64_t          uid;            // table uid, used to set the tag value when building the final query result for selectivity functions.
 } SInputColumnInfoData;
 
+typedef struct SSerializeDataHandle {
+  struct SDiskbasedBuf* pBuf;
+  int32_t               currentPage;
+} SSerializeDataHandle;
+
 // sql function runtime context
 typedef struct SqlFunctionCtx {
   SInputColumnInfoData   input;
@@ -137,10 +144,9 @@ typedef struct SqlFunctionCtx {
   SFuncExecFuncs         fpSet;
   SScalarFuncExecFuncs   sfp;
   struct SExprInfo      *pExpr;
-  struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   struct SSDataBlock    *pDstBlock; // used by indefinite rows function to set selectivity
-  int32_t                curBufPage;
+  SSerializeDataHandle   saveHandle;
   bool                   isStream;
 
   char                   udfName[TSDB_FUNC_NAME_LEN];

include/libs/function/functionMgt.h

@@ -176,7 +176,8 @@ int32_t fmGetFuncInfo(SFunctionNode* pFunc, char* pMsg, int32_t msgLen);
 
 EFuncReturnRows fmGetFuncReturnRows(SFunctionNode* pFunc);
 
-bool fmIsBuiltinFunc(const char* pFunc);
+bool          fmIsBuiltinFunc(const char* pFunc);
+EFunctionType fmGetFuncType(const char* pFunc);
 
 bool fmIsAggFunc(int32_t funcId);
 bool fmIsScalarFunc(int32_t funcId);

include/util/tencode.h

@@ -264,12 +264,14 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
 
 static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
   if (tEncodeU32v(pCoder, len) < 0) return -1;
-  if (pCoder->data) {
-    if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1;
-    memcpy(TD_CODER_CURRENT(pCoder), val, len);
-  }
+  if (len) {
+    if (pCoder->data) {
+      if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1;
+      memcpy(TD_CODER_CURRENT(pCoder), val, len);
+    }
 
-  TD_CODER_MOVE_POS(pCoder, len);
+    TD_CODER_MOVE_POS(pCoder, len);
+  }
   return 0;
 }
 
@@ -414,14 +416,18 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
 static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
   uint64_t length = 0;
   if (tDecodeU64v(pCoder, &length) < 0) return -1;
-  if (len) *len = length;
+  if (length) {
+    if (len) *len = length;
 
-  if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
-  *val = taosMemoryMalloc(length);
-  if (*val == NULL) return -1;
-  memcpy(*val, TD_CODER_CURRENT(pCoder), length);
+    if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1;
+    *val = taosMemoryMalloc(length);
+    if (*val == NULL) return -1;
+    memcpy(*val, TD_CODER_CURRENT(pCoder), length);
 
-  TD_CODER_MOVE_POS(pCoder, length);
+    TD_CODER_MOVE_POS(pCoder, length);
+  } else {
+    *val = NULL;
+  }
   return 0;
 }
 

include/util/tpagedbuf.h

@@ -58,11 +58,10 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
 /**
  *
  * @param pBuf
- * @param groupId
  * @param pageId
  * @return
  */
-void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId);
+void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId);
 
 /**
  *

packaging/MPtestJenkinsfile

@@ -64,6 +64,11 @@ pipeline {
             defaultValue:'2.1.2', 
             description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1'
         )
+        string (
+            name:'nasPassword', 
+            defaultValue:'password', 
+            description: 'the pasword of the NAS server which has installPackage-192.168.1.131'
+        )
     }
     environment{
         WORK_DIR = '/var/lib/jenkins/workspace'
@@ -107,21 +112,21 @@ pipeline {
                 stage('ubuntu16') {
                     agent{label " ubuntu16 "}
                     steps {
-                        timeout(time: 10, unit: 'MINUTES'){
+                        timeout(time: 30, unit: 'MINUTES'){
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -130,21 +135,26 @@ pipeline {
                 stage('ubuntu18') {
                     agent{label " ubuntu18 "}
                     steps {
-                        timeout(time: 10, unit: 'MINUTES'){
+                        timeout(time: 30, unit: 'MINUTES'){
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_DEB} ${baseVersion} server
+                            bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -153,21 +163,21 @@ pipeline {
                 stage('centos7') {
                     agent{label " centos7_9 "}
                     steps {
-                        timeout(time: 10, unit: 'MINUTES'){
+                        timeout(time: 30, unit: 'MINUTES'){
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -176,22 +186,45 @@ pipeline {
                 stage('centos8') {
                     agent{label " centos8_3 "}
                     steps {
-                        timeout(time: 10, unit: 'MINUTES'){
+                        timeout(time: 30, unit: 'MINUTES'){
                             sync_source("${BRANCH_NAME}")
                             sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging 
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
+                            python3 checkPackageRuning.py  
+                            '''
+                            sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
-                            rmtaos
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            python3 checkPackageRuning.py  
+                            '''            
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${nasPassword}
+                            python3 checkPackageRuning.py  
+                            '''         
+                        }
+                    }
+                }
+                                    
+                stage('arm64') {
+                    agent{label 'linux_arm64'}
+                    steps {
+                        timeout(time: 30, unit: 'MINUTES'){
+                            sync_source("${BRANCH_NAME}")
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server
+                            bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client  ${nasPassword}
                             python3 checkPackageRuning.py  
                             '''                     
                         }

packaging/debRpmAutoInstall.sh

+#!/usr/bin/expect 
+set packgeName [lindex $argv 0]
+set packageSuffix [lindex $argv 1]
+set timeout 3 
+if { ${packageSuffix} == "deb" } {
+    spawn  dpkg -i ${packgeName}
+} elseif { ${packageSuffix} == "rpm"} {
+    spawn rpm -ivh ${packgeName}
+}
+expect "*one:"
+send  "\r"
+expect "*skip:"
+send  "\r" 

packaging/docker/README.md

@@ -158,7 +158,7 @@ When you build your application with docker, you should add the TDengine client
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
 ENV TDENGINE_VERSION=2.4.0.0
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -265,7 +265,7 @@ Full version of dockerfile could be:
 ```dockerfile
 FROM golang:1.17.6-buster as builder
 ENV TDENGINE_VERSION=2.4.0.0
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \
@@ -279,7 +279,7 @@ RUN go env && go mod tidy && go build
 FROM ubuntu:20.04
 RUN apt-get update && apt-get install -y wget
 ENV TDENGINE_VERSION=2.4.0.0
-RUN wget -c https://www.taosdata.com/assets-download/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
+RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \
    && cd TDengine-client-${TDENGINE_VERSION} \
    && ./install_client.sh \

packaging/testpackage.sh

 #!/bin/sh
-
-
+#parameter
+scriptDir=$(dirname $(readlink -f $0))
 packgeName=$1
 version=$2
 originPackageName=$3
 originversion=$4
 testFile=$5
 subFile="taos.tar.gz"
+password=$6
+
+# Color setting
+RED='\033[41;30m'
+GREEN='\033[1;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[1;34m'
+GREEN_DARK='\033[0;32m'
+YELLOW_DARK='\033[0;33m'
+BLUE_DARK='\033[0;34m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
 
 if [ ${testFile} = "server" ];then
     tdPath="TDengine-server-${version}"
@@ -23,109 +35,211 @@ elif [ ${testFile} = "tools" ];then
 fi
 
 function cmdInstall {
-comd=$1
-if command -v ${comd} ;then
-    echo "${comd} is already installed" 
+command=$1
+if command -v ${command} ;then
+    echoColor YD  "${command} is already installed" 
 else 
     if command -v apt ;then
-        apt-get install ${comd} -y 
+        apt-get install ${command} -y 
     elif command -v yum ;then
-        yum -y install ${comd} 
-        echo "you should install ${comd} manually"
+        yum -y install ${command} 
+        echoColor YD "you should install ${command} manually"
     fi
 fi
 }
 
+function echoColor {
+color=$1    
+command=$2
+
+if [ ${color} = 'Y' ];then
+    echo -e  "${YELLOW}${command}${NC}" 
+elif [ ${color} = 'YD' ];then
+    echo -e  "${YELLOW_DARK}${command}${NC}" 
+elif [ ${color} = 'R' ];then
+    echo -e  "${RED}${command}${NC}" 
+elif [ ${color} = 'G' ];then
+    echo  -e  "${GREEN}${command}${NC}\r\n" 
+elif [ ${color} = 'B' ];then
+    echo  -e  "${BLUE}${command}${NC}" 
+elif [ ${color} = 'BD' ];then
+    echo  -e  "${BLUE_DARK}${command}${NC}" 
+fi
+}
+
 
-echo "Uninstall all components of TDeingne"
+echoColor G "===== install basesoft ====="
+
+cmdInstall tree
+cmdInstall wget
+cmdInstall sshpass
+
+echoColor G "===== Uninstall all components of TDeingne ====="
 
 if command -v rmtaos ;then
-    echo "uninstall all components of TDeingne:rmtaos"
+    echoColor YD "uninstall all components of TDeingne:rmtaos"
     rmtaos 
 else 
-    echo "os doesn't include TDengine "
+     echoColor YD "os doesn't include TDengine"
 fi
 
 if command -v rmtaostools ;then
-    echo "uninstall all components of TDeingne:rmtaostools"
+    echoColor YD "uninstall all components of TDeingne:rmtaostools"
     rmtaostools
 else 
-    echo "os doesn't include rmtaostools "
+    echoColor YD "os doesn't include rmtaostools "
 fi
 
 
-cmdInstall tree
-cmdInstall wget
 
-echo "new workroom path"
+
+echoColor G "===== new workroom path ====="
 installPath="/usr/local/src/packageTest"
 oriInstallPath="/usr/local/src/packageTest/3.1"
 
 if [ ! -d ${installPath} ] ;then
+    echoColor BD "mkdir -p ${installPath}"
     mkdir -p ${installPath}
 else  
-    echo "${installPath} already exists"
+    echoColor YD "${installPath} already exists"
 fi
 
+if [ -d ${installPath}/${tdPath} ] ;then
+    echoColor BD "rm -rf ${installPath}/${tdPath}/*"
+    rm -rf ${installPath}/${tdPath}/*
+fi
 
 if [ ! -d ${oriInstallPath} ] ;then
+    echoColor BD "mkdir -p ${oriInstallPath}"
     mkdir -p ${oriInstallPath}
 else  
-    echo "${oriInstallPath} already exists"
+    echoColor YD "${oriInstallPath} already exists"
+fi
+
+if [ -d ${oriInstallPath}/${originTdpPath} ] ;then
+    echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*"
+    rm -rf  ${oriInstallPath}/${originTdpPath}/*  
 fi
 
-echo "decompress installPackage"
+
+echoColor G "===== download  installPackage ====="
+# cd ${installPath}
+# wget https://www.taosdata.com/assets-download/3.0/${packgeName}
+# cd  ${oriInstallPath}
+# wget https://www.taosdata.com/assets-download/3.0/${originPackageName}
 
 cd ${installPath}
-wget https://www.taosdata.com/assets-download/3.0/${packgeName}
-cd  ${oriInstallPath}
-wget https://www.taosdata.com/assets-download/3.0/${originPackageName}
+cp -r ${scriptDir}/debRpmAutoInstall.sh   . 
+
+if [ ! -f  {packgeName}  ];then
+    echoColor  BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  ."
+    sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  .
+fi
+
+packageSuffix=$(echo ${packgeName}  | awk -F '.' '{print $NF}')
+
+
+if [ ! -f  debRpmAutoInstall.sh  ];then
+    echo '#!/usr/bin/expect ' >  debRpmAutoInstall.sh
+    echo 'set packgeName [lindex $argv 0]' >>  debRpmAutoInstall.sh
+    echo 'set packageSuffix [lindex $argv 1]' >>  debRpmAutoInstall.sh
+    echo 'set timeout 3 ' >>  debRpmAutoInstall.sh
+    echo 'if { ${packageSuffix} == "deb" } {' >>  debRpmAutoInstall.sh
+    echo '    spawn  dpkg -i ${packgeName} '  >>  debRpmAutoInstall.sh
+    echo '} elseif { ${packageSuffix} == "rpm"} {' >>  debRpmAutoInstall.sh
+    echo '    spawn rpm -ivh ${packgeName}'  >>  debRpmAutoInstall.sh
+    echo '}' >>  debRpmAutoInstall.sh
+    echo 'expect "*one:"' >>  debRpmAutoInstall.sh
+    echo 'send  "\r"' >>  debRpmAutoInstall.sh
+    echo 'expect "*skip:"' >>  debRpmAutoInstall.sh
+    echo 'send  "\r" ' >>  debRpmAutoInstall.sh
+fi
+
 
+echoColor G "===== instal Package ====="
 
 if [[ ${packgeName} =~ "deb" ]];then
-    echo "dpkg ${packgeName}" &&  dpkg -i ${packgeName}
+    cd ${installPath}
+    dpkg -r taostools
+    dpkg -r tdengine
+    if [[ ${packgeName} =~ "TDengine" ]];then
+        echoColor BD "./debRpmAutoInstall.sh ${packgeName}  ${packageSuffix}" &&   chmod 755 debRpmAutoInstall.sh &&  ./debRpmAutoInstall.sh  ${packgeName}  ${packageSuffix}
+    else
+        echoColor BD "dpkg  -i ${packgeName}" &&   dpkg  -i ${packgeName}
+    fi
 elif [[ ${packgeName} =~ "rpm" ]];then
-    echo "rpm ${packgeName}"  && rpm -ivh ${packgeName}
+    cd ${installPath}
+    sudo rpm -e tdengine
+    sudo rpm -e taostools
+    if [[ ${packgeName} =~ "TDengine" ]];then
+        echoColor BD "./debRpmAutoInstall.sh ${packgeName}  ${packageSuffix}" &&   chmod 755 debRpmAutoInstall.sh &&  ./debRpmAutoInstall.sh  ${packgeName}  ${packageSuffix}
+    else
+        echoColor BD "rpm  -ivh ${packgeName}" &&   rpm  -ivh ${packgeName}
+    fi
 elif [[ ${packgeName} =~ "tar" ]];then
-    echo "tar ${packgeName}" && tar -xvf ${packgeName} 
-    cd ${oriInstallPath}
-    echo "tar -xvf ${originPackageName}" && tar -xvf ${originPackageName} 
+    echoColor G "===== check installPackage File of tar ====="
+    cd  ${oriInstallPath}
+    if [ ! -f  {originPackageName}  ];then
+        echoColor YD "download  base installPackage"
+        echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} ."
+        sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} .
+    fi
+    echoColor YD "unzip the base installation package" 
+    echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} 
     cd ${installPath} 
-    echo "tar -xvf ${packgeName}" && tar -xvf ${packgeName} 
-
+    echoColor YD "unzip the new installation package" 
+    echoColor BD "tar -xf ${packgeName}" && tar -xf ${packgeName} 
 
     if [ ${testFile} != "tools" ] ;then
-        cd ${installPath}/${tdPath} && tar vxf ${subFile}
-        cd  ${oriInstallPath}/${originTdpPath}  && tar vxf ${subFile}
+        cd ${installPath}/${tdPath} && tar xf ${subFile}
+        cd  ${oriInstallPath}/${originTdpPath}  && tar xf ${subFile}
     fi
 
-    echo "check installPackage File"
-
+    cd  ${oriInstallPath}/${originTdpPath} && tree  >  ${installPath}/base_${originversion}_checkfile
+    cd ${installPath}/${tdPath}   && tree > ${installPath}/now_${version}_checkfile
+    
     cd ${installPath} 
-
-    tree ${oriInstallPath}/${originTdpPath} > ${originPackageName}_checkfile
-    tree ${installPath}/${tdPath} > ${packgeName}_checkfile
-
-    diff  ${packgeName}_checkfile  ${originPackageName}_checkfile  > ${installPath}/diffFile.log
+    diff  ${installPath}/base_${originversion}_checkfile   ${installPath}/now_${version}_checkfile  > ${installPath}/diffFile.log
     diffNumbers=`cat ${installPath}/diffFile.log |wc -l `
+
     if [ ${diffNumbers} != 0 ];then
-        echo "The number and names of files have changed from the previous installation package"
-        echo `cat ${installPath}/diffFile.log`
+        echoColor R "The number and names of files is different from the previous installation package"
+        echoColor Y `cat ${installPath}/diffFile.log`
         exit -1
+    else 
+        echoColor G "The number and names of files are the same as previous installation packages"
     fi
-
+    echoColor YD  "===== install Package of tar ====="
     cd ${installPath}/${tdPath}
     if [ ${testFile} = "server" ];then
+        echoColor BD "bash ${installCmd}  -e no  "
         bash ${installCmd}  -e no  
     else
+        echoColor BD "bash ${installCmd} "
         bash ${installCmd} 
     fi
-    if [[ ${packgeName} =~ "Lite" ]];then
-        cd ${installPath}
-        wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz
-        tar xvf taosTools-2.1.2-Linux-x64.tar.gz
-        cd taosTools-2.1.2 && bash install-taostools.sh
-    fi
-
 fi  
 
+cd ${installPath}
+
+if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
+    echoColor G "===== install taos-tools when package is lite or client ====="
+    cd ${installPath}
+    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
+    # wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz
+    tar xf taosTools-2.1.2-Linux-x64.tar.gz
+    cd taosTools-2.1.2 && bash install-taostools.sh
+elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "deb" ]] ;then
+    echoColor G "===== install taos-tools when package is lite or client ====="
+    cd ${installPath}
+    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
+    tar xf taosTools-2.1.2-Linux-x64.tar.gz
+    cd taosTools-2.1.2 && bash install-taostools.sh
+elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "rpm" ]]  ;then
+    echoColor G "===== install taos-tools when package is lite or client ====="
+    cd ${installPath}
+    sshpass -p ${password}   scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
+    tar xf taosTools-2.1.2-Linux-x64.tar.gz
+    cd taosTools-2.1.2 && bash install-taostools.sh
+fi
+

source/client/inc/clientInt.h

@@ -52,15 +52,17 @@ enum {
   RES_TYPE__QUERY = 1,
   RES_TYPE__TMQ,
   RES_TYPE__TMQ_META,
+  RES_TYPE__TAOSX,
 };
 
 #define SHOW_VARIABLES_RESULT_COLS       2
 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 
-#define TD_RES_QUERY(res)    (*(int8_t*)res == RES_TYPE__QUERY)
-#define TD_RES_TMQ(res)      (*(int8_t*)res == RES_TYPE__TMQ)
-#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
+#define TD_RES_QUERY(res)     (*(int8_t*)res == RES_TYPE__QUERY)
+#define TD_RES_TMQ(res)       (*(int8_t*)res == RES_TYPE__TMQ || *(int8_t*)res == RES_TYPE__TAOSX)
+#define TD_RES_TMQ_META(res)  (*(int8_t*)res == RES_TYPE__TMQ_META)
+#define TD_RES_TMQ_TAOSX(res) (*(int8_t*)res == RES_TYPE__TAOSX)
 
 typedef struct SAppInstInfo SAppInstInfo;
 
@@ -101,6 +103,8 @@ typedef struct SQueryExecMetric {
   int64_t ctgStart;     // start to parse, us
   int64_t ctgEnd;       // end to parse, us
   int64_t semanticEnd;
+  int64_t planEnd;
+  int64_t resultReady;
   int64_t execEnd;
   int64_t send;  // start to send to server, us
   int64_t rsp;   // receive response from server, us
@@ -198,8 +202,8 @@ typedef struct {
   int32_t        vgId;
   SSchemaWrapper schema;
   int32_t        resIter;
-  SMqDataRsp     rsp;
   SReqResultInfo resInfo;
+  SMqDataRsp     rsp;
 } SMqRspObj;
 
 typedef struct {
@@ -210,6 +214,17 @@ typedef struct {
   SMqMetaRsp metaRsp;
 } SMqMetaRspObj;
 
+typedef struct {
+  int8_t         resType;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
+  char           db[TSDB_DB_FNAME_LEN];
+  int32_t        vgId;
+  SSchemaWrapper schema;
+  int32_t        resIter;
+  SReqResultInfo resInfo;
+  STaosxRsp      rsp;
+} SMqTaosxRspObj;
+
 typedef struct SRequestObj {
   int8_t               resType;  // query or tmq
   uint64_t             requestId;
@@ -369,7 +384,7 @@ void         launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData*
 int32_t      refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
 int32_t      updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
 void         doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
-int32_t      removeMeta(STscObj* pTscObj, SArray* tbList);  
+int32_t      removeMeta(STscObj* pTscObj, SArray* tbList);
 int32_t      handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
 int32_t      handleCreateTbExecRes(void* res, SCatalog* pCatalog);
 bool         qnodeRequired(SRequestObj* pRequest);

source/client/src/TMQConnector.c

@@ -42,6 +42,7 @@ void commit_cb(tmq_t *tmq, int32_t code, void *param) {
 
 JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_tmqConfNewImp(JNIEnv *env, jobject jobj) {
   tmq_conf_t *conf = tmq_conf_new();
+  jniGetGlobalMethod(env);
   return (jlong)conf;
 }
 

source/client/src/clientEnv.c

@@ -76,18 +76,19 @@ static void deregisterRequest(SRequestObj *pRequest) {
            pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
 
   if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) {
-    tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 "us, exec:%" PRId64 "us", 
-            duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, 
-            pRequest->metric.ctgEnd - pRequest->metric.ctgStart,
-            pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
+    tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
+            "us, exec:%" PRId64 "us",
+            duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
+            pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
             pRequest->metric.execEnd - pRequest->metric.semanticEnd);
     atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
   } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
-    tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 "us, exec:%" PRId64 "us", 
-            duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, 
-            pRequest->metric.ctgEnd - pRequest->metric.ctgStart,
-            pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
-            pRequest->metric.execEnd - pRequest->metric.semanticEnd);
+    tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
+            "us, planner:%" PRId64 "us, exec:%" PRId64 "us",
+            duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
+            pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
+            pRequest->metric.planEnd - pRequest->metric.semanticEnd,
+            pRequest->metric.resultReady - pRequest->metric.planEnd);
     atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration);
   }
 

source/client/src/clientHb.c

@@ -145,7 +145,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
 }
 
 static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
-  SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey));
+  SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == pReq) {
     tscWarn("pReq to get activeInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid,
             pRsp->connKey.connType);
@@ -260,6 +260,8 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
     }
   }
 
+  taosHashRelease(pAppHbMgr->activeInfo, pReq);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -914,10 +916,11 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in
 }
 
 void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) {
-  SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
+  SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
   if (pReq) {
     tFreeClientHbReq(pReq);
     taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey));
+    taosHashRelease(pAppHbMgr->activeInfo, pReq);
   }
 
   if (NULL == pReq) {

source/client/src/clientImpl.c

@@ -728,7 +728,7 @@ int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog
       tFreeSTableMetaRsp(blk->pMeta);
       taosMemoryFreeClear(blk->pMeta);
     }
-    
+
     if (NULL == blk->tblFName || 0 == blk->tblFName[0]) {
       continue;
     }
@@ -851,6 +851,8 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
   SRequestObj* pRequest = (SRequestObj*)param;
   pRequest->code = code;
 
+  pRequest->metric.resultReady = taosGetTimestampUs();
+
   if (pResult) {
     memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult));
   }
@@ -1030,6 +1032,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
         pRequest->body.subplanNum = pDag->numOfSubplans;
       }
 
+      pRequest->metric.planEnd = taosGetTimestampUs();
+
       if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
         SArray* pNodeList = NULL;
         buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);

source/client/src/clientMain.c

@@ -184,6 +184,19 @@ void taos_free_result(TAOS_RES *res) {
     SRequestObj *pRequest = (SRequestObj *)res;
     tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId);
     destroyRequest(pRequest);
+  } else if (TD_RES_TMQ_TAOSX(res)) {
+    SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res;
+    if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
+    if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
+    if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
+    if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+    // taosx
+    taosArrayDestroy(pRsp->rsp.createTableLen);
+    taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree);
+
+    pRsp->resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&pRsp->resInfo);
+    taosMemoryFree(pRsp);
   } else if (TD_RES_TMQ(res)) {
     SMqRspObj *pRsp = (SMqRspObj *)res;
     if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);

source/client/src/tmq.c

@@ -164,6 +164,7 @@ typedef struct {
   union {
     SMqDataRsp dataRsp;
     SMqMetaRsp metaRsp;
+    STaosxRsp  taosxRsp;
   };
 } SMqPollRspWrapper;
 
@@ -810,8 +811,19 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
 }
 
 int32_t tmq_unsubscribe(tmq_t* tmq) {
+  int32_t     rsp;
+  int32_t     retryCnt = 0;
   tmq_list_t* lst = tmq_list_new();
-  int32_t     rsp = tmq_subscribe(tmq, lst);
+  while (1) {
+    rsp = tmq_subscribe(tmq, lst);
+    if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
+      break;
+    } else {
+      retryCnt++;
+      taosMsleep(500);
+    }
+  }
+
   tmq_list_destroy(lst);
   return rsp;
 }
@@ -1130,21 +1142,29 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
     tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp);
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
-  } else {
-    ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP);
+
+    tscDebug("consumer:%" PRId64 ", recv poll: vgId:%d, req offset %" PRId64 ", rsp offset %" PRId64 " type %d",
+             tmq->consumerId, pVg->vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version,
+             rspType);
+
+  } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
     SDecoder decoder;
     tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
     tDecodeSMqMetaRsp(&decoder, &pRspWrapper->metaRsp);
     tDecoderClear(&decoder);
     memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead));
+  } else if (rspType == TMQ_MSG_TYPE__TAOSX_RSP) {
+    SDecoder decoder;
+    tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead));
+    tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp);
+    tDecoderClear(&decoder);
+    memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
+  } else {
+    ASSERT(0);
   }
 
   taosMemoryFree(pMsg->pData);
 
-  tscDebug("consumer:%" PRId64 ", recv poll: vgId:%d, req offset %" PRId64 ", rsp offset %" PRId64 " type %d",
-           tmq->consumerId, pVg->vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version,
-           rspType);
-
   taosWriteQitem(tmq->mqueue, pRspWrapper);
   tsem_post(&tmq->rspSem);
 
@@ -1443,6 +1463,24 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
+SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) {
+  SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
+  pRspObj->resType = RES_TYPE__TAOSX;
+  tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
+  tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
+  pRspObj->vgId = pWrapper->vgHandle->vgId;
+  pRspObj->resIter = -1;
+  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqTaosxRspObj));
+
+  pRspObj->resInfo.totalRows = 0;
+  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
+  if (!pWrapper->dataRsp.withSchema) {
+    setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
+  }
+
+  return pRspObj;
+}
+
 int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
   /*tscDebug("call poll");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
@@ -1595,6 +1633,30 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
                  pollRspWrapper->metaRsp.head.epoch, consumerEpoch);
         taosFreeQitem(pollRspWrapper);
       }
+    } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) {
+      SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
+      /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
+      int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
+      if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) {
+        SMqClientVg* pVg = pollRspWrapper->vgHandle;
+        /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset,
+         * rspMsg->msg.rspOffset);*/
+        pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset;
+        atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        if (pollRspWrapper->taosxRsp.blockNum == 0) {
+          taosFreeQitem(pollRspWrapper);
+          rspWrapper = NULL;
+          continue;
+        }
+        // build rsp
+        SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
+        taosFreeQitem(pollRspWrapper);
+        return pRsp;
+      } else {
+        tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n",
+                 pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
+        taosFreeQitem(pollRspWrapper);
+      }
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
@@ -1707,9 +1769,11 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) {
   } else if (TD_RES_TMQ_META(res)) {
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
     if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) {
-      return TMQ_RES_DATA;
+      return TMQ_RES_TAOSX;
     }
     return TMQ_RES_TABLE_META;
+  } else if (TD_RES_TMQ_TAOSX(res)) {
+    return TMQ_RES_TAOSX;
   } else {
     return TMQ_RES_INVALID;
   }

source/common/src/systable.c

@@ -240,6 +240,22 @@ static const SSysDbTableSchema variablesSchema[] = {
     {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
 };
 
+static const SSysDbTableSchema topicSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    // TODO config
+};
+
+
+static const SSysDbTableSchema subscriptionSchema[] = {
+    {.name = "topic_name", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "consumer_group", .bytes = TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
+};
+
 static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
     {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@@ -260,6 +276,9 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema), true},
     {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema), true},
     {TSDB_INS_TABLE_DNODE_VARIABLES, variablesSchema, tListLen(variablesSchema), true},
+    {TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema), false},
+    {TSDB_INS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema), false},
+    {TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {
@@ -272,13 +291,6 @@ static const SSysDbTableSchema connectionsSchema[] = {
     {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
 };
 
-static const SSysDbTableSchema topicSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
-    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
-    // TODO config
-};
 
 static const SSysDbTableSchema consumerSchema[] = {
     {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
@@ -292,13 +304,6 @@ static const SSysDbTableSchema consumerSchema[] = {
     {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
 };
 
-static const SSysDbTableSchema subscriptionSchema[] = {
-    {.name = "topic_name", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
-    {.name = "consumer_group", .bytes = TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
-    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
-};
-
 static const SSysDbTableSchema offsetSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY, .sysInfo = false},
@@ -345,13 +350,10 @@ static const SSysDbTableSchema appSchema[] = {
 static const SSysTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema), false},
     {TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema), false},
-    {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema), false},
     {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema), false},
-    {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema), false},
     // {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
     {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema), false},
     // {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema), false},
-    {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
     {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema), false}};
 // clang-format on
 

source/common/src/tglobal.c

@@ -427,6 +427,152 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   return 0;
 }
 
+static int32_t taosUpdateServerCfg(SConfig *pCfg) {
+  SConfigItem *pItem;
+  ECfgSrcType stype;
+  int32_t numOfCores;
+  int64_t totalMemoryKB;
+
+  pItem = cfgGetItem(tsCfg, "numOfCores");
+  if (pItem == NULL) {
+    return -1;
+  } else {
+    stype = pItem->stype;
+    numOfCores = pItem->fval;
+  }
+
+  pItem = cfgGetItem(tsCfg, "supportVnodes");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfSupportVnodes = numOfCores * 2;
+    tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
+    pItem->i32 = tsNumOfSupportVnodes;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfRpcThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfRpcThreads = numOfCores / 2;
+    tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 1, 4);
+    pItem->i32 = tsNumOfRpcThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfCommitThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfCommitThreads = numOfCores / 2;
+    tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
+    pItem->i32 = tsNumOfCommitThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfMnodeReadThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfMnodeReadThreads = numOfCores / 8;
+    tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4);
+    pItem->i32 = tsNumOfMnodeReadThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeQueryThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeQueryThreads = numOfCores * 2;
+    tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4);
+    pItem->i32 = tsNumOfVnodeQueryThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeStreamThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeStreamThreads = numOfCores / 4;
+    tsNumOfVnodeStreamThreads = TMAX(tsNumOfVnodeStreamThreads, 4);
+    pItem->i32 = tsNumOfVnodeStreamThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeFetchThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeFetchThreads = numOfCores / 4;
+    tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
+    pItem->i32 = tsNumOfVnodeFetchThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeWriteThreads = numOfCores;
+    tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
+    pItem->i32 = tsNumOfVnodeWriteThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeSyncThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeSyncThreads = numOfCores * 2;
+    tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16);
+    pItem->i32 = tsNumOfVnodeSyncThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfVnodeRsmaThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfVnodeRsmaThreads = numOfCores;
+    tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4);
+    pItem->i32 = tsNumOfVnodeRsmaThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfQnodeQueryThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfQnodeQueryThreads = numOfCores * 2;
+    tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4);
+    pItem->i32 = tsNumOfQnodeQueryThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfQnodeFetchThreads = numOfCores / 2;
+    tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4);
+    pItem->i32 = tsNumOfQnodeFetchThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfSnodeSharedThreads = numOfCores / 4;
+    tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4);
+    pItem->i32 = tsNumOfSnodeSharedThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "numOfSnodeUniqueThreads");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsNumOfSnodeUniqueThreads = numOfCores / 4;
+    tsNumOfSnodeUniqueThreads = TRANGE(tsNumOfSnodeUniqueThreads, 2, 4);
+    pItem->i32 = tsNumOfSnodeUniqueThreads;
+    pItem->stype = stype;
+  }
+
+  pItem = cfgGetItem(tsCfg, "totalMemoryKB");
+  if (pItem == NULL) {
+    return -1;
+  } else {
+    stype = pItem->stype;
+    totalMemoryKB = pItem->i64;
+  }
+
+  pItem = cfgGetItem(tsCfg, "rpcQueueMemoryAllowed");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsRpcQueueMemoryAllowed = totalMemoryKB * 1024 * 0.1;
+    tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
+    pItem->i64 = tsRpcQueueMemoryAllowed;
+    pItem->stype = stype;
+  }
+
+  return 0;
+}
+
+
 static void taosSetClientLogCfg(SConfig *pCfg) {
   SConfigItem *pItem = cfgGetItem(pCfg, "logDir");
   tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX);
@@ -981,7 +1127,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
     taosSetServerLogCfg(pCfg);
   }
 
-  taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32);
+  taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false);
 
   if (taosMulMkDir(tsLogDir) != 0) {
     uError("failed to create dir:%s since %s", tsLogDir, terrstr());
@@ -1048,6 +1194,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
     if (taosSetClientCfg(tsCfg)) return -1;
   } else {
     if (taosSetClientCfg(tsCfg)) return -1;
+    if (taosUpdateServerCfg(tsCfg)) return -1;
     if (taosSetServerCfg(tsCfg)) return -1;
     if (taosSetTfsCfg(tsCfg) != 0) return -1;
   }
@@ -1072,7 +1219,7 @@ void taosCleanupCfg() {
 void taosCfgDynamicOptions(const char *option, const char *value) {
   if (strncasecmp(option, "debugFlag", 9) == 0) {
     int32_t flag = atoi(value);
-    taosSetAllDebugFlag(flag);
+    taosSetAllDebugFlag(flag, true);
     return;
   }
 
@@ -1097,11 +1244,13 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
       "dDebugFlag",   "vDebugFlag",  "mDebugFlag",   "wDebugFlag",   "sDebugFlag",   "tsdbDebugFlag",
       "tqDebugFlag",  "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag",
       "tmrDebugFlag", "uDebugFlag",  "smaDebugFlag", "rpcDebugFlag", "qDebugFlag",   "metaDebugFlag",
+      "jniDebugFlag",
   };
   int32_t *optionVars[] = {
       &dDebugFlag,   &vDebugFlag,  &mDebugFlag,   &wDebugFlag,   &sDebugFlag,   &tsdbDebugFlag,
       &tqDebugFlag,  &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag,
       &tmrDebugFlag, &uDebugFlag,  &smaDebugFlag, &rpcDebugFlag, &qDebugFlag,   &metaDebugFlag,
+      &jniDebugFlag,
   };
 
   int32_t optionSize = tListLen(options);
@@ -1113,41 +1262,42 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
     int32_t flag = atoi(value);
     uInfo("%s set from %d to %d", optName, *optionVars[d], flag);
     *optionVars[d] = flag;
-    taosSetDebugFlag(optionVars[d], optName, flag);
+    taosSetDebugFlag(optionVars[d], optName, flag, true);
     return;
   }
 
   uError("failed to cfg dynamic option:%s value:%s", option, value);
 }
 
-void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal) {
+void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite) {
   SConfigItem *pItem = cfgGetItem(tsCfg, flagName);
-  if (pItem != NULL) {
+  if (pItem != NULL && (rewrite || pItem->i32 == 0)) {
     pItem->i32 = flagVal;
   }
   *pFlagPtr = flagVal;
 }
 
-void taosSetAllDebugFlag(int32_t flag) {
+void taosSetAllDebugFlag(int32_t flag, bool rewrite) {
   if (flag <= 0) return;
 
-  taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag);
-  taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag);
-  taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag);
-  taosSetDebugFlag(&qDebugFlag, "qDebugFlag", flag);
-  taosSetDebugFlag(&cDebugFlag, "cDebugFlag", flag);
-  taosSetDebugFlag(&dDebugFlag, "dDebugFlag", flag);
-  taosSetDebugFlag(&vDebugFlag, "vDebugFlag", flag);
-  taosSetDebugFlag(&mDebugFlag, "mDebugFlag", flag);
-  taosSetDebugFlag(&wDebugFlag, "wDebugFlag", flag);
-  taosSetDebugFlag(&sDebugFlag, "sDebugFlag", flag);
-  taosSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag);
-  taosSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag);
-  taosSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag);
-  taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag);
-  taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag);
-  taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag);
-  taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag);
-  taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag);
+  taosSetDebugFlag(&uDebugFlag, "uDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&rpcDebugFlag, "rpcDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&jniDebugFlag, "jniDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&qDebugFlag, "qDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&cDebugFlag, "cDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&dDebugFlag, "dDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&vDebugFlag, "vDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&mDebugFlag, "mDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&wDebugFlag, "wDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&sDebugFlag, "sDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&tsdbDebugFlag, "tsdbDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&tqDebugFlag, "tqDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&fsDebugFlag, "fsDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&udfDebugFlag, "udfDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&smaDebugFlag, "smaDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&idxDebugFlag, "idxDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&metaDebugFlag, "tmrDebugFlag", flag, rewrite);
   uInfo("all debug flag are set to %d", flag);
 }

source/common/src/tmsg.c

@@ -3330,7 +3330,7 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) {
   return 0;
 }
 
-void tFreeSTableMetaRsp(void *pRsp) { taosMemoryFreeClear(((STableMetaRsp*)pRsp)->pSchemas); }
+void tFreeSTableMetaRsp(void *pRsp) { taosMemoryFreeClear(((STableMetaRsp *)pRsp)->pSchemas); }
 
 void tFreeSTableIndexRsp(void *info) {
   if (NULL == info) {
@@ -5119,17 +5119,17 @@ int tDecodeSVCreateTbRsp(SDecoder *pCoder, SVCreateTbRsp *pRsp) {
   } else {
     pRsp->pMeta = NULL;
   }
-  
+
   tEndDecode(pCoder);
   return 0;
 }
 
-void tFreeSVCreateTbRsp(void* param) {
+void tFreeSVCreateTbRsp(void *param) {
   if (NULL == param) {
     return;
   }
-  
-  SVCreateTbRsp* pRsp = (SVCreateTbRsp*)param;
+
+  SVCreateTbRsp *pRsp = (SVCreateTbRsp *)param;
   if (pRsp->pMeta) {
     taosMemoryFree(pRsp->pMeta->pSchemas);
     taosMemoryFree(pRsp->pMeta);
@@ -5347,7 +5347,7 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
   if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
   if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
   if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
-  
+
   int32_t meta = 0;
   if (tDecodeI32(pDecoder, &meta) < 0) return -1;
   if (meta) {
@@ -5395,12 +5395,12 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
   return 0;
 }
 
-void tFreeSSubmitBlkRsp(void* param) {
+void tFreeSSubmitBlkRsp(void *param) {
   if (NULL == param) {
     return;
   }
-  
-  SSubmitBlkRsp* pRsp = (SSubmitBlkRsp*)param;
+
+  SSubmitBlkRsp *pRsp = (SSubmitBlkRsp *)param;
 
   taosMemoryFree(pRsp->tblFName);
   if (pRsp->pMeta) {
@@ -5409,7 +5409,6 @@ void tFreeSSubmitBlkRsp(void* param) {
   }
 }
 
-
 void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
   if (NULL == pRsp) return;
 
@@ -5621,7 +5620,6 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
   }
 }
 
-
 int32_t tEncodeSMCreateStbRsp(SEncoder *pEncoder, const SMCreateStbRsp *pRsp) {
   if (tStartEncode(pEncoder) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->pMeta->pSchemas ? 1 : 0) < 0) return -1;
@@ -5673,8 +5671,6 @@ void tFreeSMCreateStbRsp(SMCreateStbRsp *pRsp) {
   }
 }
 
-
-
 int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) {
   if (tEncodeI8(pEncoder, pOffsetVal->type) < 0) return -1;
   if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_META) {
@@ -5692,7 +5688,7 @@ int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal)
 
 int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) {
   if (tDecodeI8(pDecoder, &pOffsetVal->type) < 0) return -1;
-  if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA  || pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_META) {
+  if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_META) {
     if (tDecodeI64(pDecoder, &pOffsetVal->uid) < 0) return -1;
     if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1;
   } else if (pOffsetVal->type == TMQ_OFFSET__LOG) {
@@ -5714,7 +5710,7 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
     snprintf(buf, maxLen, "offset(reset to latest)");
   } else if (pVal->type == TMQ_OFFSET__LOG) {
     snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version);
-  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA  || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
+  } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA || pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
     snprintf(buf, maxLen, "offset(ss data) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
   } else {
     ASSERT(0);
@@ -5815,17 +5811,17 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
   return 0;
 }
 
-int32_t tEncodeSMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp) {
+int32_t tEncodeSMqMetaRsp(SEncoder *pEncoder, const SMqMetaRsp *pRsp) {
   if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
-  if(tEncodeI16(pEncoder, pRsp->resMsgType)) return -1;
-  if(tEncodeBinary(pEncoder, pRsp->metaRsp, pRsp->metaRspLen)) return -1;
+  if (tEncodeI16(pEncoder, pRsp->resMsgType)) return -1;
+  if (tEncodeBinary(pEncoder, pRsp->metaRsp, pRsp->metaRspLen)) return -1;
   return 0;
 }
 
-int32_t tDecodeSMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp) {
+int32_t tDecodeSMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) {
   if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
   if (tDecodeI16(pDecoder, &pRsp->resMsgType) < 0) return -1;
-  if (tDecodeBinaryAlloc(pDecoder, &pRsp->metaRsp, (uint64_t*)&pRsp->metaRspLen) < 0) return -1;
+  if (tDecodeBinaryAlloc(pDecoder, &pRsp->metaRsp, (uint64_t *)&pRsp->metaRspLen) < 0) return -1;
   return 0;
 }
 
@@ -5895,6 +5891,99 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
   return 0;
 }
 
+void tDeleteSMqDataRsp(SMqDataRsp *pRsp) {
+  taosArrayDestroy(pRsp->blockDataLen);
+  taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree);
+  taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper);
+  taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree);
+}
+
+int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
+  if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1;
+  if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1;
+  if (pRsp->blockNum != 0) {
+    if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1;
+    if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1;
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
+      void   *data = taosArrayGetP(pRsp->blockData, i);
+      if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
+      if (pRsp->withSchema) {
+        SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
+        if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1;
+      }
+      if (pRsp->withTbName) {
+        char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i);
+        if (tEncodeCStr(pEncoder, tbName) < 0) return -1;
+      }
+    }
+  }
+  if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
+  if (pRsp->createTableNum) {
+    for (int32_t i = 0; i < pRsp->createTableNum; i++) {
+      void   *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
+      int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
+      if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
+    }
+  }
+  return 0;
+}
+
+int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
+  if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1;
+  if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1;
+  if (pRsp->blockNum != 0) {
+    pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t));
+    if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1;
+    if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1;
+    if (pRsp->withTbName) {
+      pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    }
+    if (pRsp->withSchema) {
+      pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *));
+    }
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      void    *data;
+      uint64_t bLen;
+      if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
+      taosArrayPush(pRsp->blockData, &data);
+      int32_t len = bLen;
+      taosArrayPush(pRsp->blockDataLen, &len);
+
+      if (pRsp->withSchema) {
+        SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
+        if (pSW == NULL) return -1;
+        if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1;
+        taosArrayPush(pRsp->blockSchema, &pSW);
+      }
+
+      if (pRsp->withTbName) {
+        char *tbName;
+        if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1;
+        taosArrayPush(pRsp->blockTbName, &tbName);
+      }
+    }
+  }
+  if (tDecodeI32(pDecoder, &pRsp->createTableNum) < 0) return -1;
+  if (pRsp->createTableNum) {
+    pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
+    pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
+    for (int32_t i = 0; i < pRsp->createTableNum; i++) {
+      void    *pCreate = NULL;
+      uint64_t len;
+      if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
+      int32_t l = (int32_t)len;
+      taosArrayPush(pRsp->createTableLen, &l);
+      taosArrayPush(pRsp->createTableReq, &pCreate);
+    }
+  }
+  return 0;
+}
 int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) {
   if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1;
   if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1;

source/common/src/tvariant.c

@@ -155,8 +155,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
 void taosVariantDestroy(SVariant *pVar) {
   if (pVar == NULL) return;
 
-  if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR
-     || pVar->nType == TSDB_DATA_TYPE_JSON) {
+  if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR ||
+      pVar->nType == TSDB_DATA_TYPE_JSON) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
@@ -185,8 +185,8 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
 
   pDst->nType = pSrc->nType;
-  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR
-      || pSrc->nType == TSDB_DATA_TYPE_JSON) {
+  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR ||
+      pSrc->nType == TSDB_DATA_TYPE_JSON) {
     int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
     char   *p = taosMemoryRealloc(pDst->pz, len);
     assert(p);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -301,7 +301,7 @@ int32_t dmInitServer(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
 
   SRpcInit rpcInit = {0};
-  strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn));
+  strncpy(rpcInit.localFqdn, tsLocalFqdn, TSDB_FQDN_LEN);
   rpcInit.localPort = tsServerPort;
   rpcInit.label = "DND-S";
   rpcInit.numOfThreads = tsNumOfRpcThreads;

source/dnode/mnode/impl/src/mndShow.c

@@ -88,7 +88,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
     type = TSDB_MGMT_TABLE_VGROUP;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONSUMERS, len) == 0) {
     type = TSDB_MGMT_TABLE_CONSUMERS;
-  } else if (strncasecmp(name, TSDB_PERFS_TABLE_SUBSCRIPTIONS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIPTIONS, len) == 0) {
     type = TSDB_MGMT_TABLE_SUBSCRIPTIONS;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_TRANS, len) == 0) {
     type = TSDB_MGMT_TABLE_TRANS;
@@ -102,9 +102,9 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
     type = TSDB_MGMT_TABLE_QUERIES;
   } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
     type = TSDB_MGMT_TABLE_VNODES;
-  } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) {
     type = TSDB_MGMT_TABLE_TOPICS;
-  } else if (strncasecmp(name, TSDB_PERFS_TABLE_STREAMS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_INS_TABLE_STREAMS, len) == 0) {
     type = TSDB_MGMT_TABLE_STREAMS;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_APPS, len) == 0) {
     type = TSDB_MGMT_TABLE_APPS;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -287,6 +287,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     if (consumerVgNum > minVgCnt) {
       if (imbCnt < imbConsumerNum) {
         if (consumerVgNum == minVgCnt + 1) {
+          imbCnt++;
           continue;
         } else {
           // pop until equal minVg + 1

source/dnode/mnode/impl/src/mndTopic.c

@@ -763,8 +763,9 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
     int32_t          cols = 0;
 
     char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
-    tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB);
-    tNameGetDbName(&n, varDataVal(topicName));
+    strcpy(varDataVal(topicName), mndGetDbStr(pTopic->name));
+    /*tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB);*/
+    /*tNameGetDbName(&n, varDataVal(topicName));*/
     varDataSetLen(topicName, strlen(varDataVal(topicName)));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);

source/dnode/vnode/inc/vnode.h

@@ -157,17 +157,17 @@ void   tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
 size_t tsdbCacheGetCapacity(SVnode *pVnode);
 
 // tq
-typedef struct SMetaTableInfo{
+typedef struct SMetaTableInfo {
   int64_t         suid;
   int64_t         uid;
   SSchemaWrapper *schema;
   char            tbName[TSDB_TABLE_NAME_LEN];
-}SMetaTableInfo;
+} SMetaTableInfo;
 
-typedef struct SIdInfo{
-  int64_t         version;
-  int32_t         index;
-}SIdInfo;
+typedef struct SIdInfo {
+  int64_t version;
+  int32_t index;
+} SIdInfo;
 
 typedef struct SSnapContext {
   SMeta    *pMeta;
@@ -180,8 +180,8 @@ typedef struct SSnapContext {
   SArray   *idList;
   int32_t   index;
   bool      withMeta;
-  bool      queryMetaOrData;    // true-get meta, false-get data
-}SSnapContext;
+  bool      queryMetaOrData;  // true-get meta, false-get data
+} SSnapContext;
 
 typedef struct STqReader {
   int64_t           ver;
@@ -232,11 +232,12 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot);
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 
-int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta, SSnapContext** ctxRet);
-int32_t getMetafromSnapShot(SSnapContext* ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
-SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx);
-int32_t setForSnapShot(SSnapContext* ctx, int64_t uid);
-int32_t destroySnapContext(SSnapContext* ctx);
+int32_t        buildSnapContext(SMeta *pMeta, int64_t snapVersion, int64_t suid, int8_t subType, bool withMeta,
+                                SSnapContext **ctxRet);
+int32_t        getMetafromSnapShot(SSnapContext *ctx, void **pBuf, int32_t *contLen, int16_t *type, int64_t *uid);
+SMetaTableInfo getUidfromSnapShot(SSnapContext *ctx);
+int32_t        setForSnapShot(SSnapContext *ctx, int64_t uid);
+int32_t        destroySnapContext(SSnapContext *ctx);
 
 // structs
 struct STsdbCfg {
@@ -259,6 +260,7 @@ typedef struct {
   int64_t numOfNTables;
   int64_t numOfNTimeSeries;
   int64_t numOfTimeSeries;
+  int64_t itvTimeSeries;
   int64_t pointsWritten;
   int64_t totalStorage;
   int64_t compStorage;

source/dnode/vnode/src/inc/sma.h

@@ -95,6 +95,7 @@ struct SRSmaStat {
   int64_t          refId;             // shared by fetch tasks
   volatile int64_t nBufItems;         // number of items in queue buffer
   SRWLatch         lock;              // r/w lock for rsma fs(e.g. qtaskinfo)
+  volatile int32_t nFetchAll;         // active number of fetch all
   int8_t           triggerStat;       // shared by fetch tasks
   int8_t           commitStat;        // 0 not in committing, 1 in committing
   SArray          *aTaskFile;         // qTaskFiles committed recently(for recovery/snapshot r/w)

source/dnode/vnode/src/inc/tq.h

@@ -67,21 +67,21 @@ typedef struct {
 // tqExec
 
 typedef struct {
-  char*       qmsg;
+  char* qmsg;
 } STqExecCol;
 
 typedef struct {
-  int64_t     suid;
+  int64_t suid;
 } STqExecTb;
 
 typedef struct {
-  SHashObj*   pFilterOutTbUid;
+  SHashObj* pFilterOutTbUid;
 } STqExecDb;
 
 typedef struct {
   int8_t subType;
 
-  STqReader* pExecReader;
+  STqReader*  pExecReader;
   qTaskInfo_t task;
   union {
     STqExecCol execCol;
@@ -144,7 +144,7 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
 
 // tqExec
-int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp);
+int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp);
 int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp);
 
 // tqMeta
@@ -175,22 +175,6 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data);
 char*   tqOffsetBuildFName(const char* path, int32_t ver);
 int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
 
-static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) {
-  pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA;
-  pOffsetVal->uid = uid;
-  pOffsetVal->ts = ts;
-}
-
-static FORCE_INLINE void tqOffsetResetToMeta(STqOffsetVal* pOffsetVal, int64_t uid) {
-  pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_META;
-  pOffsetVal->uid = uid;
-}
-
-static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) {
-  pOffsetVal->type = TMQ_OFFSET__LOG;
-  pOffsetVal->version = ver;
-}
-
 // tqStream
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask);
 

source/dnode/vnode/src/meta/metaQuery.c

@@ -615,9 +615,13 @@ int64_t metaGetTbNum(SMeta *pMeta) {
 // N.B. Called by statusReq per second
 int64_t metaGetTimeSeriesNum(SMeta *pMeta) {
   // sum of (number of columns of stable -  1) * number of ctables (excluding timestamp column)
-  int64_t num = 0;
-  vnodeGetTimeSeriesNum(pMeta->pVnode, &num);
-  pMeta->pVnode->config.vndStats.numOfTimeSeries = num;
+  if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || ++pMeta->pVnode->config.vndStats.itvTimeSeries % 60 == 0) {
+    int64_t num = 0;
+    vnodeGetTimeSeriesNum(pMeta->pVnode, &num);
+    pMeta->pVnode->config.vndStats.numOfTimeSeries = num;
+
+    pMeta->pVnode->config.vndStats.itvTimeSeries = 0;
+  }
 
   return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries;
 }
@@ -890,7 +894,7 @@ const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) {
 
 #ifdef TAG_FILTER_DEBUG
   if (IS_VAR_DATA_TYPE(val->type)) {
-    char* buf = taosMemoryCalloc(val->nData + 1, 1);
+    char *buf = taosMemoryCalloc(val->nData + 1, 1);
     memcpy(buf, val->pData, val->nData);
     metaDebug("metaTag table val varchar index:%d cid:%d type:%d value:%s", 1, val->cid, val->type, buf);
     taosMemoryFree(buf);
@@ -900,13 +904,13 @@ const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) {
     metaDebug("metaTag table val number index:%d cid:%d type:%d value:%f", 1, val->cid, val->type, dval);
   }
 
-  SArray* pTagVals = NULL;
-  tTagToValArray((STag*)pTag, &pTagVals);
+  SArray *pTagVals = NULL;
+  tTagToValArray((STag *)pTag, &pTagVals);
   for (int i = 0; i < taosArrayGetSize(pTagVals); i++) {
-    STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i);
+    STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
 
     if (IS_VAR_DATA_TYPE(pTagVal->type)) {
-      char* buf = taosMemoryCalloc(pTagVal->nData + 1, 1);
+      char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1);
       memcpy(buf, pTagVal->pData, pTagVal->nData);
       metaDebug("metaTag table varchar index:%d cid:%d type:%d value:%s", i, pTagVal->cid, pTagVal->type, buf);
       taosMemoryFree(buf);

source/dnode/vnode/src/sma/smaCommit.c

@@ -172,7 +172,7 @@ static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) {
   TdDirPtr      pDir = NULL;
   TdDirEntryPtr pDirEntry = NULL;
   char          dir[TSDB_FILENAME_LEN];
-  const char   *pattern = "v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
+  const char   *pattern = "v[0-9]+qinf\\.v([0-9]+)?$";
   regex_t       regex;
   int           code = 0;
 
@@ -312,15 +312,22 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
 
   SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
   SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat);
+  int32_t    nLoops = 0;
 
   // step 1: set rsma stat
   atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
-  atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 1);
+  while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) {
+    ++nLoops;
+    if (nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
   pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
   ASSERT(pRSmaStat->commitAppliedVer > 0);
 
   // step 2: wait for all triggered fetch tasks to finish
-  int32_t nLoops = 0;
+  nLoops = 0;
   while (1) {
     if (T_REF_VAL_GET(pStat) == 0) {
       smaDebug("vgId:%d, rsma commit, fetch tasks are all finished", SMA_VID(pSma));
@@ -344,7 +351,8 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
     return TSDB_CODE_FAILED;
   }
 
-  smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), (void*)taosGetSelfPthreadId());
+  smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma),
+          (void *)taosGetSelfPthreadId());
   nLoops = 0;
   while (atomic_load_64(&pRSmaStat->nBufItems) > 0) {
     ++nLoops;
@@ -359,7 +367,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
   }
   smaInfo("vgId:%d, rsma commit, operator state commited, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
 
-#if 0 // consuming task of qTaskInfo clone 
+#if 0  // consuming task of qTaskInfo clone 
   // step 4:  swap queue/qall and iQueue/iQall
   // lock
   // taosWLockLatch(SMA_ENV_LOCK(pEnv));

source/dnode/vnode/src/sma/smaOpen.c

@@ -16,17 +16,17 @@
 #include "sma.h"
 #include "tsdb.h"
 
-static int32_t smaEvalDays(SRetention *r, int8_t precision);
-static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
+static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration);
+static int32_t smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
 static int32_t rsmaRestore(SSma *pSma);
 
-#define SMA_SET_KEEP_CFG(l)                                                                       \
+#define SMA_SET_KEEP_CFG(v, l)                                                                    \
   do {                                                                                            \
     SRetention *r = &pCfg->retentions[l];                                                         \
     pKeepCfg->keep2 = convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE); \
     pKeepCfg->keep0 = pKeepCfg->keep2;                                                            \
     pKeepCfg->keep1 = pKeepCfg->keep2;                                                            \
-    pKeepCfg->days = smaEvalDays(r, pCfg->precision);                                             \
+    pKeepCfg->days = smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days);            \
   } while (0)
 
 #define SMA_OPEN_RSMA_IMPL(v, l)                                                   \
@@ -38,51 +38,78 @@ static int32_t rsmaRestore(SSma *pSma);
       }                                                                            \
       break;                                                                       \
     }                                                                              \
-    smaSetKeepCfg(&keepCfg, pCfg, TSDB_TYPE_RSMA_L##l);                            \
+    smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l);                         \
     if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg) < 0) { \
       goto _err;                                                                   \
     }                                                                              \
   } while (0)
 
-#define RETENTION_DAYS_SPLIT_RATIO 10
-#define RETENTION_DAYS_SPLIT_MIN   1
-#define RETENTION_DAYS_SPLIT_MAX   30
+/**
+ * @brief Evaluate days(duration) for rsma level 1/2/3.
+ *  1) level 1: duration from "create database"
+ *  2) level 2/3: duration * (freq/freqL1)
+ * @param pVnode
+ * @param r
+ * @param level
+ * @param precision
+ * @param duration
+ * @return int32_t
+ */
+static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration) {
+  int32_t freqDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE);
+  int32_t keepDuration = convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE);
+  int32_t days = duration;  // min
 
-static int32_t smaEvalDays(SRetention *r, int8_t precision) {
-  int32_t keepDays = convertTimeFromPrecisionToUnit(r->keep, precision, TIME_UNIT_DAY);
-  int32_t freqDays = convertTimeFromPrecisionToUnit(r->freq, precision, TIME_UNIT_DAY);
+  if (days < freqDuration) {
+    days = freqDuration;
+  }
 
-  int32_t days = keepDays / RETENTION_DAYS_SPLIT_RATIO;
-  if (days <= RETENTION_DAYS_SPLIT_MIN) {
-    days = RETENTION_DAYS_SPLIT_MIN;
-    if (days < freqDays) {
-      days = freqDays + 1;
-    }
-  } else {
-    if (days > RETENTION_DAYS_SPLIT_MAX) {
-      days = RETENTION_DAYS_SPLIT_MAX;
-    }
-    if (days < freqDays) {
-      days = freqDays + 1;
-    }
+  if (days > keepDuration) {
+    days = keepDuration;
+  }
+
+  if (level == TSDB_RETENTION_L0) {
+    goto end;
   }
-  return days * 1440;
+
+  ASSERT(level >= TSDB_RETENTION_L1 && level <= TSDB_RETENTION_L2);
+  
+  freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
+  keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);
+
+  int32_t nFreqTimes = (r + level)->freq / (r + TSDB_RETENTION_L0)->freq;
+  days *= (nFreqTimes > 1 ? nFreqTimes : 1);
+
+  if (days > keepDuration) {
+    days = keepDuration;
+  }
+
+  if (days > TSDB_MAX_DURATION_PER_FILE) {
+    days = TSDB_MAX_DURATION_PER_FILE;
+  }
+
+  if (days < freqDuration) {
+    days = freqDuration;
+  }
+end:
+  smaInfo("vgId:%d, evaluated duration for level %" PRIi8 " is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration);
+  return days;
 }
 
-int smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
+int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
   pKeepCfg->precision = pCfg->precision;
   switch (type) {
     case TSDB_TYPE_TSMA:
       ASSERT(0);
       break;
     case TSDB_TYPE_RSMA_L0:
-      SMA_SET_KEEP_CFG(0);
+      SMA_SET_KEEP_CFG(pVnode, 0);
       break;
     case TSDB_TYPE_RSMA_L1:
-      SMA_SET_KEEP_CFG(1);
+      SMA_SET_KEEP_CFG(pVnode, 1);
       break;
     case TSDB_TYPE_RSMA_L2:
-      SMA_SET_KEEP_CFG(2);
+      SMA_SET_KEEP_CFG(pVnode, 2);
       break;
     default:
       ASSERT(0);
@@ -148,11 +175,11 @@ int32_t smaClose(SSma *pSma) {
 
 /**
  * @brief rsma env restore
- * 
- * @param pSma 
- * @param type 
- * @param committedVer 
- * @return int32_t 
+ *
+ * @param pSma
+ * @param type
+ * @param committedVer
+ * @return int32_t
  */
 int32_t tdRsmaRestore(SSma *pSma, int8_t type, int64_t committedVer) {
   ASSERT(VND_IS_RSMA(pSma->pVnode));

source/dnode/vnode/src/sma/smaRollup.c

@@ -21,17 +21,17 @@
 #define RSMA_SUBMIT_BATCH_SIZE     (1024)                                                // cnt
 #define RSMA_FETCH_DELAY_MAX       (900000)                                              // ms
 #define RSMA_FETCH_ACTIVE_MAX      (1800)                                                // ms
+#define RSMA_FETCH_INTERVAL        (5000)                                                // ms
 
 SSmaMgmt smaMgmt = {
     .inited = 0,
     .rsetId = -1,
 };
 
-#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
-#define TD_RSMAINFO_DEL_FILE      "rsmainfo.del"
+#define TD_QTASKINFO_FNAME_PREFIX "qinf.v"
+
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
-typedef struct SRSmaExecQItem     SRSmaExecQItem;
 
 static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
@@ -82,11 +82,6 @@ struct SRSmaQTaskInfoIter {
   int32_t nBufPos;
 };
 
-struct SRSmaExecQItem {
-  void *pRSmaInfo;
-  void *qall;
-};
-
 void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) {
   tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
 }
@@ -1083,9 +1078,6 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTa
     goto _err;
   }
 
-  SSmaEnv   *pRSmaEnv = pSma->pRSmaEnv;
-  SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pRSmaEnv);
-
   SRSmaQTaskInfoIter fIter = {0};
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
@@ -1501,13 +1493,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
   switch (rsmaTriggerStat) {
     case TASK_TRIGGER_STAT_PAUSED:
     case TASK_TRIGGER_STAT_CANCELLED: {
-      tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId);
       smaDebug("vgId:%d, rsma fetch task not start for level %" PRIi8 " since stat is %" PRIi8
                ", rsetId rsetId:%" PRIi64 " refId:%d",
                SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaInfo->refId);
       if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) {
-        taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
+        taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
       }
+      tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId);
       return;
     }
     default:
@@ -1518,7 +1510,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
   switch (fetchTriggerStat) {
     case TASK_TRIGGER_STAT_ACTIVE: {
-      smaDebug("vgId:%d, rsma fetch task started for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
+      smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
                SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
       // async process
       pItem->fetchLevel = pItem->level;
@@ -1531,8 +1523,6 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       if (atomic_load_8(&pRSmaInfo->assigned) == 0) {
         tsem_post(&(pStat->notEmpty));
       }
-      smaInfo("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64, SMA_VID(pSma), pItem->level,
-              pRSmaInfo->suid);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is paused",
@@ -1715,21 +1705,36 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
                 smaDebug("vgId:%d, batchSize:%d, execType:%" PRIi8, SMA_VID(pSma), qallItemSize, type);
               }
 
-              if (type == RSMA_EXEC_OVERFLOW) {
+              int8_t oldStat = atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 2);
+              if (oldStat == 0 ||
+                  ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) {
+                atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1);
                 tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr);
+                if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) {
+                  atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
+                }
               }
 
               if (qallItemSize > 0) {
                 atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize);
                 continue;
               } else if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) {
-                continue;
+                if (atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)) == 0) {
+                  continue;
+                }
+                for (int32_t j = 0; j < TSDB_RETENTION_L2; ++j) {
+                  SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, j);
+                  if (pItem->fetchLevel) {
+                    pItem->fetchLevel = 0;
+                    taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
+                  }
+                }
               }
 
               break;
             }
           }
-          ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0));
+          atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0);
         }
       }
       if (type == RSMA_EXEC_COMMIT) {
@@ -1758,7 +1763,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
             }
 
             // tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr);
-            ASSERT(1 == atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0));
+            atomic_val_compare_exchange_8(&pInfo->assigned, 1, 0);
           }
         }
         ASSERT(taosQueueItemSize(pInfo->iQueue) == 0);
@@ -1775,7 +1780,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
       if (pEnv->flag & SMA_ENV_FLG_CLOSE) {
         break;
       }
-      
+
       tsem_wait(&pRSmaStat->notEmpty);
 
       if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) {

source/dnode/vnode/src/tq/tq.c

@@ -357,8 +357,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
               TD_VID(pTq->pVnode), formatBuf);
     } else {
       if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
-        if (pReq->useSnapshot){
-          if (pHandle->fetchMeta){
+        if (pReq->useSnapshot) {
+          if (pHandle->fetchMeta) {
             tqOffsetResetToMeta(&fetchOffsetNew, 0);
           } else {
             tqOffsetResetToData(&fetchOffsetNew, 0, 0);
@@ -373,43 +373,47 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
         if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
           code = -1;
         }
-        goto OVER;
+        tDeleteSMqDataRsp(&dataRsp);
+        return code;
       } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
         tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
                 " in vg %d, subkey %s, reset none failed",
                 pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
         terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
         code = -1;
-        goto OVER;
+        tDeleteSMqDataRsp(&dataRsp);
+        return code;
       }
     }
   }
 
-  if(pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN || fetchOffsetNew.type != TMQ_OFFSET__LOG){
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN || fetchOffsetNew.type != TMQ_OFFSET__LOG) {
     SMqMetaRsp metaRsp = {0};
     tqScan(pTq, pHandle, &dataRsp, &metaRsp, &fetchOffsetNew);
 
-    if(metaRsp.metaRspLen > 0){
+    if (metaRsp.metaRspLen > 0) {
       if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
         code = -1;
       }
-      tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send meta offset type:%d,uid:%ld,version:%ld", consumerId, pHandle->subKey,
-              TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.version);
+      tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send meta offset type:%d,uid:%ld,version:%ld", consumerId,
+              pHandle->subKey, TD_VID(pTq->pVnode), metaRsp.rspOffset.type, metaRsp.rspOffset.uid,
+              metaRsp.rspOffset.version);
       taosMemoryFree(metaRsp.metaRsp);
       goto OVER;
     }
 
-    if (dataRsp.blockNum > 0){
+    if (dataRsp.blockNum > 0) {
       if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
         code = -1;
       }
       goto OVER;
-    }else{
+    } else {
       fetchOffsetNew = dataRsp.rspOffset;
     }
 
-    tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld", consumerId, pHandle->subKey,
-            TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
+    tqDebug("tmq poll: consumer %ld, subkey %s, vg %d, send data blockNum:%d, offset type:%d,uid:%ld,version:%ld",
+            consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type,
+            dataRsp.rspOffset.uid, dataRsp.rspOffset.version);
   }
 
   if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
@@ -426,7 +430,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       consumerEpoch = atomic_load_32(&pHandle->epoch);
       if (consumerEpoch > reqEpoch) {
         tqWarn("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, vg %d offset %" PRId64
-                   ", found new consumer epoch %d, discard req epoch %d",
+               ", found new consumer epoch %d, discard req epoch %d",
                consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
         break;
       }
@@ -449,7 +453,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       if (pHead->msgType == TDMT_VND_SUBMIT) {
         SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
 
-        if (tqLogScanExec(pTq, &pHandle->execHandle, pCont, &dataRsp) < 0) {
+        if (tqLogScanExec(pTq, pHandle, pCont, &dataRsp) < 0) {
           /*ASSERT(0);*/
         }
         // TODO batch optimization:
@@ -490,18 +494,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
 
 OVER:
   if (pCkHead) taosMemoryFree(pCkHead);
-  // TODO wrap in destroy func
-  taosArrayDestroy(dataRsp.blockDataLen);
-  taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
-
-  if (dataRsp.withSchema) {
-    taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
-  }
-
-  if (dataRsp.withTbName) {
-    taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree);
-  }
-
+  tDeleteSMqDataRsp(&dataRsp);
   return code;
 }
 
@@ -629,9 +622,9 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe
       tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList);
       taosArrayDestroy(tbUidList);
 
-      buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, (SSnapContext **)(&handle.sContext));
-      pHandle->execHandle.task =
-          qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
+      buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta,
+                       (SSnapContext**)(&handle.sContext));
+      pHandle->execHandle.task = qCreateQueueExecTaskInfo(NULL, &handle, NULL, NULL);
     }
     taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
     tqDebug("try to persist handle %s consumer %" PRId64, req.subKey, pHandle->consumerId);

source/dnode/vnode/src/tq/tqExec.c

@@ -60,6 +60,46 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
   return 0;
 }
 
+int64_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+  const STqExecHandle* pExec = &pHandle->execHandle;
+  ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
+
+  qTaskInfo_t task = pExec->task;
+
+  if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
+    tqDebug("prepare scan failed, return");
+    if (pOffset->type == TMQ_OFFSET__LOG) {
+      pRsp->rspOffset = *pOffset;
+      return 0;
+    } else {
+      tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
+      if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) {
+        tqDebug("prepare scan failed, return");
+        pRsp->rspOffset = *pOffset;
+        return 0;
+      }
+    }
+  }
+
+  int32_t rowCnt = 0;
+  while (1) {
+    SSDataBlock* pDataBlock = NULL;
+    uint64_t     ts = 0;
+    tqDebug("tmq task start to execute");
+    if (qExecTask(task, &pDataBlock, &ts) < 0) {
+      ASSERT(0);
+    }
+    tqDebug("tmq task execute end, get %p", pDataBlock);
+
+    if (pDataBlock) {
+      tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
+      pRsp->blockNum++;
+    }
+  }
+
+  return 0;
+}
+
 int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) {
   const STqExecHandle* pExec = &pHandle->execHandle;
   qTaskInfo_t          task = pExec->task;
@@ -102,18 +142,18 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
           taosArrayPush(pRsp->blockTbName, &tbName);
         }
       }
-      if(pRsp->withSchema){
+      if (pRsp->withSchema) {
         if (pOffset->type == TMQ_OFFSET__LOG) {
           tqAddBlockSchemaToRsp(pExec, pRsp);
-        }else{
+        } else {
           SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
           taosArrayPush(pRsp->blockSchema, &pSW);
         }
       }
 
-      if(pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN){
+      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
         tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols);
-      }else{
+      } else {
         tqAddBlockDataToRsp(pDataBlock, pRsp, taosArrayGetSize(pDataBlock->pDataBlock));
       }
       pRsp->blockNum++;
@@ -125,17 +165,9 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
       }
     }
 
-    if(pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN){
-      if (pRsp->blockNum == 0 && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-        tqDebug("vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
-                pHandle->snapshotVer + 1);
-        tqOffsetResetToLog(pOffset, pHandle->snapshotVer);
-        qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
-        continue;
-      }
-    }else{
-      if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA){
-        if(qStreamExtractPrepareUid(task) != 0){
+    if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
+      if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+        if (qStreamExtractPrepareUid(task) != 0) {
           continue;
         }
         tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode),
@@ -143,13 +175,13 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
         break;
       }
 
-      if (pRsp->blockNum > 0){
+      if (pRsp->blockNum > 0) {
         tqDebug("tmqsnap task exec exited, get data");
         break;
       }
 
       SMqMetaRsp* tmp = qStreamExtractMetaMsg(task);
-      if(tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA){
+      if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
         tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts);
         qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType);
         tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META;
@@ -173,57 +205,8 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqMetaRsp*
   return 0;
 }
 
-#if 0
-int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) {
-  ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
-  qTaskInfo_t task = pExec->execCol.task[workerId];
-
-  if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) {
-    ASSERT(0);
-  }
-
-  int32_t rowCnt = 0;
-  while (1) {
-    SSDataBlock* pDataBlock = NULL;
-    uint64_t     ts = 0;
-    if (qExecTask(task, &pDataBlock, &ts) < 0) {
-      ASSERT(0);
-    }
-    if (pDataBlock == NULL) break;
-
-    ASSERT(pDataBlock->info.rows != 0);
-    ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0);
-
-    tqAddBlockDataToRsp(pDataBlock, pRsp);
-
-    if (pRsp->withTbName) {
-      pRsp->withTbName = 0;
-#if 0
-      int64_t uid;
-      int64_t ts;
-      if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
-        ASSERT(0);
-      }
-      tqAddTbNameToRsp(pTq, uid, pRsp);
-#endif
-    }
-    pRsp->blockNum++;
-
-    rowCnt += pDataBlock->info.rows;
-    if (rowCnt >= 4096) break;
-  }
-  int64_t uid;
-  int64_t ts;
-  if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
-    ASSERT(0);
-  }
-  tqOffsetResetToData(&pRsp->rspOffset, uid, ts);
-
-  return 0;
-}
-#endif
-
-int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp) {
+int32_t tqLogScanExec(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, SMqDataRsp* pRsp) {
+  STqExecHandle* pExec = &pHandle->execHandle;
   ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN);
 
   if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
@@ -268,6 +251,28 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       tqAddBlockSchemaToRsp(pExec, pRsp);
       pRsp->blockNum++;
     }
+#if 0
+    if (pHandle->fetchMeta && pRsp->blockNum) {
+      SSubmitMsgIter iter = {0};
+      tInitSubmitMsgIter(pReq, &iter);
+      STaosxRsp* pXrsp = (STaosxRsp*)pRsp;
+      while (1) {
+        SSubmitBlk* pBlk = NULL;
+        if (tGetSubmitMsgNext(&iter, &pBlk) < 0) return -1;
+        if (pBlk->schemaLen > 0) {
+          if (pXrsp->createTableNum == 0) {
+            pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
+            pXrsp->createTableReq = taosArrayInit(0, sizeof(void*));
+          }
+          void* createReq = taosMemoryCalloc(1, pBlk->schemaLen);
+          memcpy(createReq, pBlk->data, pBlk->schemaLen);
+          taosArrayPush(pXrsp->createTableLen, &pBlk->schemaLen);
+          taosArrayPush(pXrsp->createTableReq, &createReq);
+          pXrsp->createTableNum++;
+        }
+      }
+    }
+#endif
   }
 
   if (pRsp->blockNum == 0) {

source/libs/command/src/command.c

@@ -471,6 +471,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
     len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName);
     appendColumnFields(buf2, &len, pCfg);
     len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")");
+    appendTableOptions(buf2, &len, pDbCfg, pCfg);
   }
 
   varDataLen(buf2) = len;

source/libs/executor/inc/executorimpl.h

@@ -143,6 +143,8 @@ typedef struct {
   STqOffsetVal   prepareStatus; // for tmq
   STqOffsetVal   lastStatus;    // for tmq
   SMqMetaRsp     metaRsp;       // for tmq fetching meta
+  int64_t snapshotVer;
+
   SSchemaWrapper *schema;
   char           tbName[TSDB_TABLE_NAME_LEN];
   SSDataBlock*   pullOverBlk;   // for streaming
@@ -303,6 +305,7 @@ typedef struct SAggSupporter {
   char*          keyBuf;               // window key buffer
   SDiskbasedBuf* pResultBuf;           // query result buffer based on blocked-wised disk file
   int32_t        resultRowSize;        // the result buffer size for each result row, with the meta data size for each row
+  int32_t        currentPageId;        // current write page id
 } SAggSupporter;
 
 typedef struct {
@@ -327,7 +330,6 @@ typedef struct STableScanInfo {
   SQueryTableDataCond cond;
   int32_t         scanFlag;     // table scan flag to denote if it is a repeat/reverse/main scan
   int32_t         dataBlockLoadFlag;
-//  SInterval       interval;     // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded.
   SSampleExecInfo sample;       // sample execution info
   int32_t         currentGroupId;
   int32_t         currentTable;
@@ -431,6 +433,7 @@ typedef struct SStreamAggSupporter {
   char*          pKeyBuf;              // window key buffer
   SDiskbasedBuf* pResultBuf;           // query result buffer based on blocked-wised disk file
   int32_t        resultRowSize;        // the result buffer size for each result row, with the meta data size for each row
+  int32_t        currentPageId;        // buffer page that is active
   SSDataBlock*   pScanBlock;
 } SStreamAggSupporter;
 
@@ -485,24 +488,23 @@ typedef struct SStreamScanInfo {
   STimeWindowAggSupp     twAggSup;
   SSDataBlock*           pUpdateDataRes;
   // status for tmq
-  // SSchemaWrapper schema;
-  SNodeList*             pGroupTags;
-  SNode*                 pTagCond;
-  SNode*                 pTagIndexCond;
+  SNodeList* pGroupTags;
+  SNode*     pTagCond;
+  SNode*     pTagIndexCond;
 } SStreamScanInfo;
 
-typedef struct SStreamRawScanInfo{
-//  int8_t    subType;
-//  bool      withMeta;
-//  int64_t   suid;
-//  int64_t   snapVersion;
-//  void     *metaInfo;
-//  void     *dataInfo;
-  SVnode*         vnode;
-  SSDataBlock     pRes;         // result SSDataBlock
-  STsdbReader*    dataReader;
-  SSnapContext*   sContext;
-}SStreamRawScanInfo;
+typedef struct {
+  //  int8_t    subType;
+  //  bool      withMeta;
+  //  int64_t   suid;
+  //  int64_t   snapVersion;
+  //  void     *metaInfo;
+  //  void     *dataInfo;
+  SVnode*       vnode;
+  SSDataBlock   pRes;  // result SSDataBlock
+  STsdbReader*  dataReader;
+  SSnapContext* sContext;
+} SStreamRawScanInfo;
 
 typedef struct SSysTableScanInfo {
   SRetrieveMetaTableRsp* pRsp;
@@ -527,14 +529,14 @@ typedef struct SBlockDistInfo {
   SSDataBlock* pResBlock;
   void*        pHandle;
   SReadHandle  readHandle;
-  uint64_t     uid;        // table uid
+  uint64_t     uid;  // table uid
 } SBlockDistInfo;
 
 // todo remove this
 typedef struct SOptrBasicInfo {
-  SResultRowInfo  resultRowInfo;
-  SSDataBlock*    pRes;
-  bool            mergeResultBlock;
+  SResultRowInfo resultRowInfo;
+  SSDataBlock*   pRes;
+  bool           mergeResultBlock;
 } SOptrBasicInfo;
 
 typedef struct SIntervalAggOperatorInfo {
@@ -1009,7 +1011,7 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary
 int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput,
     int32_t size);
-SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize);
+SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize);
 SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,
     TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex);
 SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs,

source/libs/executor/src/executil.c

@@ -46,8 +46,8 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
     rowSize += pCtx[i].resDataInfo.interBufSize;
   }
 
-  rowSize +=
-      (numOfOutput * sizeof(bool));  // expand rowSize to mark if col is null for top/bottom result(doSaveTupleData)
+  rowSize += (numOfOutput * sizeof(bool));
+  // expand rowSize to mark if col is null for top/bottom result(saveTupleData)
   return rowSize;
 }
 
@@ -1178,7 +1178,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     SqlFunctionCtx* pCtx = &pFuncCtx[i];
 
     pCtx->functionId = -1;
-    pCtx->curBufPage = -1;
     pCtx->pExpr = pExpr;
 
     if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) {
@@ -1191,7 +1190,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
           fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet);
         } else {
           char* udfName = pExpr->pExpr->_function.pFunctNode->functionName;
-          strncpy(pCtx->udfName, udfName, strlen(udfName));
+          strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN);
           fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet);
         }
         pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env);
@@ -1222,6 +1221,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     pCtx->isStream = false;
 
     pCtx->param = pFunct->pParam;
+    pCtx->saveHandle.currentPage = -1;
   }
 
   for (int32_t i = 1; i < numOfOutput; ++i) {

source/libs/executor/src/executor.c

@@ -139,7 +139,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
 
 qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) {
   if (msg == NULL) {
-    // TODO create raw scan
+    // create raw scan
 
     SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo));
     if (NULL == pTaskInfo) {
@@ -151,7 +151,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
     pTaskInfo->cost.created = taosGetTimestampMs();
     pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE;
     pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo);
-    if(NULL == pTaskInfo->pRoot){
+    if (NULL == pTaskInfo->pRoot) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       taosMemoryFree(pTaskInfo);
       return NULL;
@@ -834,11 +834,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     } else {
       ASSERT(0);
     }
-  }else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA){
+  } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
     SStreamRawScanInfo* pInfo = pOperator->info;
-    SSnapContext* sContext = pInfo->sContext;
-    if(setForSnapShot(sContext, pOffset->uid) != 0) {
-      qError("setDataForSnapShot error. uid:%"PRIi64, pOffset->uid);
+    SSnapContext*       sContext = pInfo->sContext;
+    if (setForSnapShot(sContext, pOffset->uid) != 0) {
+      qError("setDataForSnapShot error. uid:%" PRIi64, pOffset->uid);
       return -1;
     }
 
@@ -847,27 +847,29 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     pInfo->dataReader = NULL;
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList);
-    if(mtInfo.uid == 0) return 0;   // no data
+    if (mtInfo.uid == 0) return 0;  // no data
 
     initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo);
     pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
     pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo));
     taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0});
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList, &pInfo->dataReader, NULL);
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pTaskInfo->tableqinfoList.pTableList,
+                   &pInfo->dataReader, NULL);
 
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
     tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema);
     pTaskInfo->streamInfo.schema = mtInfo.schema;
+
     qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts);
-  }else if(pOffset->type == TMQ_OFFSET__SNAPSHOT_META){
+  } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) {
     SStreamRawScanInfo* pInfo = pOperator->info;
-    SSnapContext* sContext = pInfo->sContext;
-    if(setForSnapShot(sContext, pOffset->uid) != 0) {
-      qError("setForSnapShot error. uid:%"PRIi64" ,version:%"PRIi64, pOffset->uid);
+    SSnapContext*       sContext = pInfo->sContext;
+    if (setForSnapShot(sContext, pOffset->uid) != 0) {
+      qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid);
       return -1;
     }
     qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid);
-  }else if (pOffset->type == TMQ_OFFSET__LOG) {
+  } else if (pOffset->type == TMQ_OFFSET__LOG) {
     SStreamRawScanInfo* pInfo = pOperator->info;
     tsdbReaderClose(pInfo->dataReader);
     pInfo->dataReader = NULL;

source/libs/executor/src/executorimpl.c

@@ -179,26 +179,23 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR
 }
 #endif
 
-SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) {
+SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) {
   SFilePage* pData = NULL;
 
   // in the first scan, new space needed for results
   int32_t pageId = -1;
-  SIDList list = getDataBufPagesIdList(pResultBuf);
-
-  if (taosArrayGetSize(list) == 0) {
-    pData = getNewBufPage(pResultBuf, tableGroupId, &pageId);
+  if (*currentPageId == -1) {
+    pData = getNewBufPage(pResultBuf, &pageId);
     pData->num = sizeof(SFilePage);
   } else {
-    SPageInfo* pi = getLastPageInfo(list);
-    pData = getBufPage(pResultBuf, getPageId(pi));
-    pageId = getPageId(pi);
+    pData = getBufPage(pResultBuf, *currentPageId);
+    pageId = *currentPageId;
 
     if (pData->num + interBufSize > getBufPageSize(pResultBuf)) {
       // release current page first, and prepare the next one
-      releaseBufPageInfo(pResultBuf, pi);
+      releaseBufPage(pResultBuf, pData);
 
-      pData = getNewBufPage(pResultBuf, tableGroupId, &pageId);
+      pData = getNewBufPage(pResultBuf, &pageId);
       if (pData != NULL) {
         pData->num = sizeof(SFilePage);
       }
@@ -215,9 +212,9 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int
   SResultRow* pResultRow = (SResultRow*)((char*)pData + pData->num);
   pResultRow->pageId = pageId;
   pResultRow->offset = (int32_t)pData->num;
+  *currentPageId = pageId;
 
   pData->num += interBufSize;
-
   return pResultRow;
 }
 
@@ -263,18 +260,15 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
 
   // allocate a new buffer page
   if (pResult == NULL) {
-#ifdef BUF_PAGE_DEBUG
-    qDebug("page_2");
-#endif
     ASSERT(pSup->resultRowSize > 0);
-    pResult = getNewResultRow(pResultBuf, groupId, pSup->resultRowSize);
+    pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
 
     initResultRow(pResult);
 
     // add a new result set for a new group
     SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
     tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos,
-                sizeof(SResultRowPosition));
+                   sizeof(SResultRowPosition));
   }
 
   // 2. set the new time window to be the new active time window
@@ -302,7 +296,7 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes
   SIDList list = getDataBufPagesIdList(pResultBuf);
 
   if (taosArrayGetSize(list) == 0) {
-    pData = getNewBufPage(pResultBuf, tid, &pageId);
+    pData = getNewBufPage(pResultBuf, &pageId);
     pData->num = sizeof(SFilePage);
   } else {
     SPageInfo* pi = getLastPageInfo(list);
@@ -313,7 +307,7 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes
       // release current page first, and prepare the next one
       releaseBufPageInfo(pResultBuf, pi);
 
-      pData = getNewBufPage(pResultBuf, tid, &pageId);
+      pData = getNewBufPage(pResultBuf, &pageId);
       if (pData != NULL) {
         pData->num = sizeof(SFilePage);
       }
@@ -2821,92 +2815,6 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
     }
   }
 }
-#if 0
-int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
-  uint8_t type = pOperator->operatorType;
-
-  pOperator->status = OP_OPENED;
-
-  if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    SStreamScanInfo* pScanInfo = pOperator->info;
-    pScanInfo->blockType = STREAM_INPUT__TABLE_SCAN;
-
-    pScanInfo->pTableScanOp->status = OP_OPENED;
-
-    STableScanInfo* pInfo = pScanInfo->pTableScanOp->info;
-    ASSERT(pInfo->scanMode == TABLE_SCAN__TABLE_ORDER);
-
-    if (uid == 0) {
-      pInfo->noTable = 1;
-      return TSDB_CODE_SUCCESS;
-    }
-
-    /*if (pSnapShotScanInfo->dataReader == NULL) {*/
-    /*pSnapShotScanInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0);*/
-    /*pSnapShotScanInfo->scanMode = TABLE_SCAN__TABLE_ORDER;*/
-    /*}*/
-
-    pInfo->noTable = 0;
-
-    if (pInfo->lastStatus.uid != uid || pInfo->lastStatus.ts != ts) {
-      SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
-
-      int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
-      bool    found = false;
-      for (int32_t i = 0; i < tableSz; i++) {
-        STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
-        if (pTableInfo->uid == uid) {
-          found = true;
-          pInfo->currentTable = i;
-        }
-      }
-      // TODO after processing drop, found can be false
-      ASSERT(found);
-
-      tsdbSetTableId(pInfo->dataReader, uid);
-      int64_t oldSkey = pInfo->cond.twindows.skey;
-      pInfo->cond.twindows.skey = ts + 1;
-      tsdbReaderReset(pInfo->dataReader, &pInfo->cond);
-      pInfo->cond.twindows.skey = oldSkey;
-      pInfo->scanTimes = 0;
-
-      qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, ts,
-             pInfo->currentTable, tableSz);
-    }
-
-    return TSDB_CODE_SUCCESS;
-
-  } else {
-    if (pOperator->numOfDownstream == 1) {
-      return doPrepareScan(pOperator->pDownstream[0], uid, ts);
-    } else if (pOperator->numOfDownstream == 0) {
-      qError("failed to find stream scan operator to set the input data block");
-      return TSDB_CODE_QRY_APP_ERROR;
-    } else {
-      qError("join not supported for stream block scan");
-      return TSDB_CODE_QRY_APP_ERROR;
-    }
-  }
-}
-
-int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
-  int32_t type = pOperator->operatorType;
-  if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-    SStreamScanInfo* pScanInfo = pOperator->info;
-    STableScanInfo*  pSnapShotScanInfo = pScanInfo->pTableScanOp->info;
-    *uid = pSnapShotScanInfo->lastStatus.uid;
-    *ts = pSnapShotScanInfo->lastStatus.ts;
-  } else {
-    if (pOperator->pDownstream[0] == NULL) {
-      return TSDB_CODE_INVALID_PARA;
-    } else {
-      doGetScanStatus(pOperator->pDownstream[0], uid, ts);
-    }
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-#endif
 
 // this is a blocking operator
 static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
@@ -3030,7 +2938,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
   SResultRow*         pRow = (SResultRow*)((char*)pPage + pos->offset);
   setBufPageDirty(pPage, true);
   releaseBufPage(pSup->pResultBuf, pPage);
-  
+
   int32_t iter = 0;
   void*   pIter = NULL;
   while ((pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter))) {
@@ -3092,7 +3000,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) {
     offset += sizeof(int32_t);
 
     uint64_t    tableGroupId = *(uint64_t*)(result + offset);
-    SResultRow* resultRow = getNewResultRow(pSup->pResultBuf, tableGroupId, pSup->resultRowSize);
+    SResultRow* resultRow = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
     if (!resultRow) {
       return TSDB_CODE_TSC_INVALID_INPUT;
     }
@@ -3440,8 +3348,10 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul
 
 int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize,
                          const char* pKey) {
+  int32_t    code = 0;
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
 
+  pAggSup->currentPageId = -1;
   pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
   pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn);
@@ -3455,18 +3365,18 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   getBufferPgSize(pAggSup->resultRowSize, &defaultPgsz, &defaultBufsz);
 
   if (!osTempSpaceAvailable()) {
-    terrno = TSDB_CODE_NO_AVAIL_DISK;
-    qError("Init stream agg supporter failed since %s", terrstr(terrno));
-    return terrno;
+    code = TSDB_CODE_NO_AVAIL_DISK;
+    qError("Init stream agg supporter failed since %s, %s", terrstr(code), pKey);
+    return code;
   }
 
-  int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir);
+  code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir);
   if (code != TSDB_CODE_SUCCESS) {
-    qError("Create agg result buf failed since %s", tstrerror(code));
+    qError("Create agg result buf failed since %s, %s", tstrerror(code), pKey);
     return code;
   }
 
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 void cleanupAggSup(SAggSupporter* pAggSup) {
@@ -3488,7 +3398,7 @@ int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf
   }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pSup->pCtx[i].pBuf = pAggSup->pResultBuf;
+    pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -3520,6 +3430,7 @@ void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
     }
 
     taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx);
+    taosMemoryFreeClear(pCtx[i].subsidiaries.buf);
     taosMemoryFree(pCtx[i].input.pData);
     taosMemoryFree(pCtx[i].input.pColumnDataAgg);
   }
@@ -4071,8 +3982,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
           STableKeyInfo* pKeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
           qDebug("creating stream task: add table %" PRId64, pKeyInfo->uid);
         }
-      }
 #endif
+      }
 
       pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan);
       pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo);
@@ -4297,42 +4208,6 @@ SArray* extractColumnInfo(SNodeList* pNodeList) {
   return pList;
 }
 
-#if 0
-STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                STableListInfo* pTableListInfo, const char* idstr) {
-  int32_t code = getTableList(pHandle->meta, pHandle->vnode, &pTableScanNode->scan, pTableListInfo);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  if (taosArrayGetSize(pTableListInfo->pTableList) == 0) {
-    code = 0;
-    qDebug("no table qualified for query, %s", idstr);
-    goto _error;
-  }
-
-  SQueryTableDataCond cond = {0};
-  code = initQueryTableDataCond(&cond, pTableScanNode);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  STsdbReader* pReader;
-  code = tsdbReaderOpen(pHandle->vnode, &cond, pTableListInfo->pTableList, &pReader, idstr);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  cleanupQueryTableDataCond(&cond);
-
-  return pReader;
-
-_error:
-  terrno = code;
-  return NULL;
-}
-#endif
-
 static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) {
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     if (pOperator->numOfDownstream == 0) {
@@ -4574,7 +4449,7 @@ _complete:
   return code;
 }
 
-static void doDestroyTableList(STableListInfo* pTableqinfoList) {
+void doDestroyTableList(STableListInfo* pTableqinfoList) {
   taosArrayDestroy(pTableqinfoList->pTableList);
   taosHashCleanup(pTableqinfoList->map);
   if (pTableqinfoList->needSortTableByGroupId) {
@@ -4678,6 +4553,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf
 
 int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput,
                                int32_t size) {
+  pSup->currentPageId = -1;
   pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pSup->keySize = sizeof(int64_t) + sizeof(TSKEY);
   pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize);
@@ -4705,7 +4581,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
   }
   int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, tsTempDir);
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    pCtx[i].pBuf = pSup->pResultBuf;
+    pCtx[i].saveHandle.pBuf = pSup->pResultBuf;
   }
+
   return code;
 }

source/libs/executor/src/groupoperator.c

@@ -547,7 +547,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
     p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len);
 
     int32_t pageId = 0;
-    pPage = getNewBufPage(pInfo->pBuf, 0, &pageId);
+    pPage = getNewBufPage(pInfo->pBuf, &pageId);
     taosArrayPush(p->pPageList, &pageId);
 
     *(int32_t *) pPage = 0;
@@ -562,7 +562,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
 
       // add a new page for current group
       int32_t pageId = 0;
-      pPage = getNewBufPage(pInfo->pBuf, 0, &pageId);
+      pPage = getNewBufPage(pInfo->pBuf, &pageId);
       taosArrayPush(p->pPageList, &pageId);
       memset(pPage, 0, getBufPageSize(pInfo->pBuf));
     }