other: merge main.

README-CN.md

@@ -175,7 +175,7 @@ cd TDengine
 ```bash
 mkdir debug
 cd debug
-cmake .. -DBUILD_TOOLS=true
+cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true
 make
 ```
 

README.md

@@ -183,7 +183,7 @@ It equals to execute following commands:
 ```bash
 mkdir debug
 cd debug
-cmake .. -DBUILD_TOOLS=true
+cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true
 make
 ```
 

build.sh

@@ -4,5 +4,5 @@ if [ ! -d debug ]; then
     mkdir debug || echo -e "failed to make directory for build"
 fi
 
-cd debug && cmake .. -DBUILD_TOOLS=true && make
+cd debug && cmake .. -DBUILD_TOOLS=true -DBUILD_CONTRIB=true && make
 

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.1.0.0.alpha")
+  SET(TD_VER_NUMBER "3.1.0.1.alpha")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

docs/en/05-get-started/01-docker.md

@@ -32,6 +32,20 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043
 
 Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
 
+If you need to persist data to a specific directory on your local machine, please run the following command:
+```shell
+docker run -d -v ~/data/taos/dnode/data:/var/lib/taos \
+  -v ~/data/taos/dnode/log:/var/log/taos \
+  -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
+```
+:::note
+
+- /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. Also you can modify ~/data/taos/dnode/data to your any local empty data directory
+- /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode/log to your any local empty log directory
+
+:::
+
+
 Run the following command to ensure that your container is running:
 
 ```shell
@@ -113,4 +127,4 @@ In the query above you are selecting the first timestamp (ts) in the interval, a
 
 ## Additional Information
 
-For more information about deploying TDengine in a Docker environment, see [Using TDengine in Docker](../../reference/docker).
+For more information about deploying TDengine in a Docker environment, see [Deploying TDengine with Docker](../../deployment/docker).

docs/en/07-develop/01-connect/index.md

@@ -83,7 +83,7 @@ If `maven` is used to manage the projects, what needs to be done is only adding
 <dependency>
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>3.2.1</version>
+  <version>3.2.4</version>
 </dependency>
 ```
 

docs/en/14-reference/11-docker/index.md → docs/en/10-deployment/02-docker.md

 ---
 title: Deploying TDengine with Docker
+sidebar_label: Docker
 description: This chapter describes how to start and access TDengine in a Docker container.
 ---
 
@@ -10,8 +11,17 @@ This chapter describes how to start the TDengine service in a container and acce
 The TDengine image starts with the HTTP service activated by default, using the following command:
 
 ```shell
-docker run -d --name tdengine -p 6041:6041 tdengine/tdengine
+docker run -d --name tdengine \
+-v ~/data/taos/dnode/data:/var/lib/taos \
+-v ~/data/taos/dnode/log:/var/log/taos \
+-p 6041:6041 tdengine/tdengine
 ```
+:::note
+
+* /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. And also you can modify ~/data/taos/dnode/data to your any other local emtpy data directory
+* /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. And also you can modify ~/data/taos/dnode/log to your any other local empty log directory
+  
+:::
 
 The above command starts a container named "tdengine" and maps the HTTP service port 6041 to the host port 6041. You can verify that the HTTP service provided in this container is available using the following command.
 
@@ -283,39 +293,38 @@ services:
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
+    ports:
+      - 6041:6041
+      - 6030:6030
     volumes:
-      - taosdata-td1:/var/lib/taos/
-      - taoslog-td1:/var/log/taos/
+      # /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. you can modify ~/data/taos/dnode1/data to your own data directory
+      - ~/data/taos/dnode1/data:/var/lib/taos
+      # /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode1/log to your own log directory
+      - ~/data/taos/dnode1/log:/var/log/taos
   td-2:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td2:/var/lib/taos/
-      - taoslog-td2:/var/log/taos/
+      - ~/data/taos/dnode2/data:/var/lib/taos
+      - ~/data/taos/dnode2/log:/var/log/taos
   td-3:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-3"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td3:/var/lib/taos/
-      - taoslog-td3:/var/log/taos/
-volumes:
-  taosdata-td1:
-  taoslog-td1:
-  taosdata-td2:
-  taoslog-td2:
-  taosdata-td3:
-  taoslog-td3:
+      - ~/data/taos/dnode3/data:/var/lib/taos
+      - ~/data/taos/dnode3/log:/var/log/taos
 ```
 
 :::note
 
 - The `VERSION` environment variable is used to set the tdengine image tag
 - `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time
-  :::
+  
+:::
 
 2. Start the cluster
 
@@ -382,24 +391,22 @@ networks:
 services:
   td-1:
     image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td1:/var/lib/taos/
-      - taoslog-td1:/var/log/taos/
+      # /var/lib/taos: TDengine's default data file directory. The location can be changed via [configuration file]. you can modify ~/data/taos/dnode1/data to your own data directory
+      - ~/data/taos/dnode1/data:/var/lib/taos
+      # /var/log/taos: TDengine's default log file directory. The location can be changed via [configure file]. you can modify ~/data/taos/dnode1/log to your own log directory
+      - ~/data/taos/dnode1/log:/var/log/taos
   td-2:
     image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td2:/var/lib/taos/
-      - taoslog-td2:/var/log/taos/
+      - ~/data/taos/dnode2/data:/var/lib/taos
+      - ~/data/taos/dnode2/log:/var/log/taos
   adapter:
     image: tdengine/tdengine:$VERSION
     entrypoint: "taosadapter"
@@ -431,11 +438,6 @@ services:
         >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
         nginx -g 'daemon off;'",
       ]
-volumes:
-  taosdata-td1:
-  taoslog-td1:
-  taosdata-td2:
-  taoslog-td2:
 ```
 
 ## Deploy with docker swarm

docs/en/10-deployment/index.md

@@ -5,7 +5,7 @@ description: This document describes how to deploy a TDengine cluster on a serve
 
 TDengine has a native distributed design and provides the ability to scale out. A few nodes can form a TDengine cluster. If you need higher processing power, you just need to add more nodes into the cluster. TDengine uses virtual node technology to virtualize a node into multiple virtual nodes to achieve load balancing. At the same time, TDengine can group virtual nodes on different nodes into virtual node groups, and use the replication mechanism to ensure the high availability of the system. The cluster feature of TDengine is completely open source.
 
-This document describes how to manually deploy a cluster on a host as well as how to deploy on Kubernetes and by using Helm.
+This document describes how to manually deploy a cluster on a host directly and deploy a cluster with Docker, Kubernetes or Helm.
 
 ```mdx-code-block
 import DocCardList from '@theme/DocCardList';

docs/en/12-taos-sql/01-data-type.md

@@ -42,7 +42,7 @@ In TDengine, the data types below can be used when specifying a column or tag.
 | 14  |       NCHAR       | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. |
 | 15  |       JSON        |              | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type.                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | 16  |      VARCHAR      | User-defined | Alias of BINARY                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-| 16  |      GEOMETRY     | User-defined | Geometry                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| 17  |      GEOMETRY     | User-defined | Geometry                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 :::note
 
 - Each row of the table cannot be longer than 48KB (64KB since version 3.0.5.0) (note that each BINARY/NCHAR/GEOMETRY column takes up an additional 2 bytes of storage space).

docs/en/12-taos-sql/10-function.md

@@ -1274,3 +1274,161 @@ SELECT SERVER_STATUS();
 ```
 
 **Description**: The server status.
+
+
+## Geometry Functions
+
+### Geometry Input Functions
+
+Geometry input functions create geometry data from WTK.
+
+#### ST_GeomFromText
+
+```sql
+ST_GeomFromText(VARCHAR WKT expr)
+```
+
+**Description**: Return a specified GEOMETRY value from Well-Known Text representation (WKT).
+
+**Return value type**: GEOMETRY
+
+**Applicable data types**: VARCHAR
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- The input can be one of WTK string, like POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON, GEOMETRYCOLLECTION.
+- The output is a GEOMETRY data type, internal defined as binary string.
+
+### Geometry Output Functions
+
+Geometry output functions convert geometry data into WTK.
+
+#### ST_AsText
+
+```sql
+ST_AsText(GEOMETRY geom)
+```
+
+**Description**: Return a specified Well-Known Text representation (WKT) value from GEOMETRY data.
+
+**Return value type**: VARCHAR
+
+**Applicable data types**: GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- The output can be one of WTK string, like POINT, LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON, GEOMETRYCOLLECTION.
+
+### Geometry Relationships Functions
+
+Geometry relationships functions determine spatial relationships between geometries.
+
+#### ST_Intersects
+
+```sql
+ST_Intersects(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Compares two geometries and returns true if they intersect.
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- Geometries intersect if they have any point in common.
+
+
+#### ST_Equals
+
+```sql
+ST_Equals(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Returns TRUE if the given geometries are "spatially equal".
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- 'Spatially equal' means ST_Contains(A,B) = true and ST_Contains(B,A) = true, and the ordering of points can be different but represent the same geometry structure.
+
+
+#### ST_Touches
+
+```sql
+ST_Touches(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Returns TRUE if A and B intersect, but their interiors do not intersect.
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- A and B have at least one point in common, and the common points lie in at least one boundary.
+- For Point/Point inputs the relationship is always FALSE, since points do not have a boundary.
+
+
+#### ST_Covers
+
+```sql
+ST_Covers(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Returns TRUE if every point in Geometry B lies inside (intersects the interior or boundary of) Geometry A.
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- A covers B means no point of B lies outside (in the exterior of) A.
+
+
+#### ST_Contains
+
+```sql
+ST_Contains(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Returns TRUE if geometry A contains geometry B.
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- A contains B if and only if all points of B lie inside (i.e. in the interior or boundary of) A (or equivalently, no points of B lie in the exterior of A), and the interiors of A and B have at least one point in common.
+
+
+#### ST_ContainsProperly
+
+```sql
+ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**Description**: Returns TRUE if every point of B lies inside A.
+
+**Return value type**: BOOL
+
+**Applicable data types**: GEOMETRY, GEOMETRY
+
+**Applicable table types**: standard tables and supertables
+
+**Explanations**：
+- There is no point of B that lies on the boundary of A or in the exterior of A.

docs/en/12-taos-sql/16-operators.md

@@ -54,7 +54,7 @@ LIKE is used together with wildcards to match strings. Its usage is described as
 MATCH and NMATCH are used together with regular expressions to match strings. Their usage is described as follows:
 
 - Use POSIX regular expression syntax. For more information, see Regular Expressions.
-- Regular expression can be used against only table names, i.e. `tbname`, and tags of binary/nchar types, but can't be used against data columns.
+- Regular expression can be used against only table names, i.e. `tbname`, and tags/columns of binary/nchar types.
 - The maximum length of regular expression string is 128 bytes. Configuration parameter `maxRegexStringLen` can be used to set the maximum allowed regular expression. It's a configuration parameter on the client side, and will take effect after restarting the client.
 
 ## Logical Operators

docs/en/12-taos-sql/22-meta.md

@@ -98,7 +98,7 @@ Provides information about user-created databases. Similar to SHOW DATABASES.
 | 21  |      cachesize       | INT              | Memory per vnode used for caching the newest data. It should be noted that `cachesize` is a TDengine keyword and needs to be escaped with ` when used as a column name.   |
 | 22  |      wal_level       | INT              | WAL level. It should be noted that `wal_level` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                      |
 | 23  |   wal_fsync_period   | INT              | Interval at which WAL is written to disk. It should be noted that `wal_fsync_period` is a TDengine keyword and needs to be escaped with ` when used as a column name.   |
-| 24  | wal_retention_period | INT              | WAL retention period. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                    |
+| 24  | wal_retention_period | INT              | WAL retention period, in second. It should be noted that `wal_retention_period` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                    |
 | 25  |  wal_retention_size  | INT              | Maximum WAL size. It should be noted that `wal_retention_size` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                    |
 | 26  |   stt_trigger   | SMALLINT | The threshold for number of files to trigger file merging. It should be noted that `stt_trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name.  |
 | 27  |   table_prefix   | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name. It should be noted that `table_prefix` is a TDengine keyword and needs to be escaped with ` when used as a column name.  |
@@ -297,3 +297,13 @@ Provides dnode configuration information.
 | 7   | target_table | BINARY(192)   | Target table                                                                                                                                                                                   |
 | 8   |  watermark   | BIGINT        | Watermark (see stream processing documentation). It should be noted that `watermark` is a TDengine keyword and needs to be escaped with ` when used as a column name.                          |
 | 9   |   trigger    | INT           | Method of triggering the result push (see stream processing documentation). It should be noted that `trigger` is a TDengine keyword and needs to be escaped with ` when used as a column name. |
+
+## INS_USER_PRIVILEGES
+
+| #   |   **Column**   | **Data Type** | **Description**                         |**                                                                               |
+| --- | :----------: | ------------ | -------------------------------------------|
+| 1   | user_name    | VARCHAR(24)       | Username                              |
+| 2   | privilege    | VARCHAR(10)       | Privilege description                 |
+| 3   | db_name      | VARCHAR(65)       | Database name                         |
+| 4   | table_name   | VARCHAR(193)      | Table name                            |
+| 5   | condition    | VARCHAR(49152)    | The privilege filter for child tables |

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -79,6 +79,12 @@ Parameter Description:
 - tz: Optional parameter that specifies the timezone of the returned time, following the IANA Time Zone rules, e.g. `America/New_York`.
 - req_id: Optional parameter that specifies the request id for tracing.
 
+:::note
+
+URL Encoding. Make sure that parameters are properly encoded. For example, when specifying a timezone you must properly encode special characters. ?tz=Etc/GMT+10 will not work because the <+> plus symbol is recognized as a space in the url. It's best practice to encode all special characters in a parameter. Instead use ?tz=Etc%2FGMT%2B10 for the parameter.
+
+:::
+
 For example, `http://h1.taos.com:6041/rest/sql/test` is a URL to `h1.taos.com:6041` and sets the default database name to `test`.
 
 TDengine supports both Basic authentication and custom authentication mechanisms, and subsequent versions will provide a standard secure digital signature mechanism for authentication.

docs/en/14-reference/03-connector/60-r-lang.mdx

+---
+toc_max_heading_level: 4
+sidebar_label: R
+title: R Language Connector
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+import Rdemo from "../../07-develop/01-connect/_connect_r.mdx"
+
+By using the RJDBC library in R, you can enable R programs to access TDengine data. Here are the installation process, configuration steps, and an example code in R.
+
+## Installation Process
+
+Before getting started, make sure you have installed the R language environment. Then, follow these steps to install and configure the RJDBC library:
+
+1. Install Java Development Kit (JDK): RJDBC library requires Java environment. Download the appropriate JDK for your operating system from the official Oracle website and follow the installation guide.
+
+2. Install the RJDBC library: Execute the following command in the R console to install the RJDBC library.
+
+```r
+install.packages("RJDBC", repos='http://cran.us.r-project.org')
+```
+
+:::note
+1. The default R language package version 4.2 which shipped with Ubuntu might lead unresponsive bug. Please install latest version of R language package from the [official website](https://www.r-project.org/).
+2. On Linux systems, installing the RJDBC package may require installing the necessary components for compilation. For example, on Ubuntu, you can execute the command ``apt install -y libbz2-dev libpcre2-dev libicu-dev`` to install the required components.
+3. On Windows systems, you need to set the **JAVA_HOME** environment variable.
+:::
+
+3. Download the TDengine JDBC driver: Visit the Maven website and download the TDengine JDBC driver (taos-jdbcdriver-X.X.X-dist.jar) to your local machine.
+
+## Configuration Process
+
+Once you have completed the installation steps, you need to do some configuration to enable the RJDBC library to connect and access the TDengine time-series database.
+
+1. Load the RJDBC library and other necessary libraries in your R script:
+
+```r
+library(DBI)
+library(rJava)
+library(RJDBC)
+```
+
+2. Set the JDBC driver and JDBC URL:
+
+```r
+# Set the JDBC driver path (specify the location on your local machine)
+driverPath <- "/path/to/taos-jdbcdriver-X.X.X-dist.jar"
+
+# Set the JDBC URL (specify the FQDN and credentials of your TDengine cluster)
+url <- "jdbc:TAOS://localhost:6030/?user=root&password=taosdata"
+```
+
+3. Load the JDBC driver:
+
+```r
+# Load the JDBC driver
+drv <- JDBC("com.taosdata.jdbc.TSDBDriver", driverPath)
+```
+
+4. Create a TDengine database connection:
+
+```r
+# Create a database connection
+conn <- dbConnect(drv, url)
+```
+
+5. Once the connection is established, you can use the ``conn`` object for various database operations such as querying data and inserting data.
+
+6. Finally, don't forget to close the database connection after you are done:
+
+```r
+# Close the database connection
+dbDisconnect(conn)
+```
+
+## Example Code Using RJDBC in R
+
+Here's an example code that uses the RJDBC library to connect to a TDengine time-series database and perform a query operation:
+
+<Rdemo/>
+
+Please modify the JDBC driver, JDBC URL, username, password, and SQL query statement according to your specific TDengine time-series database environment and requirements.
+
+By following the steps and using the provided example code, you can use the RJDBC library in the R language to access the TDengine time-series database and perform tasks such as data querying and analysis.

docs/en/14-reference/09-support-platform/index.md

@@ -7,10 +7,10 @@ description: This document describes the supported platforms for the TDengine se
 
 |              | **Windows Server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 or later** | **macOS** |
 | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | --------- |
-| X64          | ●                            | ●                 | ●                | ●                | ●         |
+| X64          | ●/E                          | ●/E               | ●                | ●                | ●         |
 | ARM64        |                              |                   | ●                |                  | ●         |
 
-Note: ● means officially tested and verified, ○ means unofficially tested and verified.
+Note: 1) ● means officially tested and verified, ○ means unofficially tested and verified, E means only supported by the enterprise edition. 2) The community edition only supports newer versions of mainstream operating systems, including Ubuntu 18+/CentOS 7+/RetHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS, etc. If you have requirements for other operating systems and editions, please contact support of the enterprise edition.
 
 ## List of supported platforms for TDengine clients and connectors
 

docs/en/14-reference/11-docker/_category_.yml

-label: TDengine Docker images
\ No newline at end of file

docs/en/14-reference/12-config/index.md

@@ -670,6 +670,15 @@ The charset that takes effect is UTF-8.
 | Value Range | 0: not consistent; 1: consistent.                                                   |
 | Default     | 0                                                                                   |
 
+### smlTsDefaultName
+
+| Attribute     | Description                                                     |
+| --------      | --------------------------------------------------------        |
+| Applicable    | Client only                                                     |
+| Meaning       | The name of the time column for schemaless automatic table creation is set through this configuration |
+| Type          | String                                                          |
+| Default Value | _ts                                                             |
+
 ## Compress Parameters
 
 ### compressMsgSize

docs/en/14-reference/13-schemaless/13-schemaless.md

@@ -34,7 +34,27 @@ In the schemaless writing data line protocol, each data item in the field_set ne
 
 - If there are English double quotes on both sides, it indicates the BINARY(32) type. For example, `"abc"`.
 - If there are double quotes on both sides and an L prefix, it means NCHAR(32) type. For example, `L"error message"`.
-- Spaces, equal signs (=), commas (,), and double quotes (") need to be escaped with a backslash (\\) in front. (All refer to the ASCII character)
+- Spaces, equals sign (=), comma (,), double quote ("), and backslash (\\) need to be escaped with a backslash (\\) in front. (All refer to the ASCII character). The rules are as follows:
+
+| **Serial number** | **Element**    | **Escape characters**   |
+| -------- | ----------- | -----------------------------       |
+| 1        | Measurement     | Comma, Space                    |
+| 2        | Tag key         | Comma, Equals Sign, Space       |
+| 3        | Tag value       | Comma, Equals Sign, Space       |
+| 4        | Field key       | Comma, Equals Sign, Space       |
+| 5        | Field value     | Double quote, Backslash         |
+
+With two contiguous backslashes, the first is interpreted as an escape character. Examples of backslash escape rules are as follows:
+
+| **Serial number**      | **Backslashes**    | **Interpreted as**            |
+| --------               | -----------        | ----------------------------- |
+| 1                      | \                  | \                             |
+| 2                      | \\\\               | \                             |
+| 3                      | \\\\\\             | \\\\                          |
+| 4                      | \\\\\\\\           | \\\\                          |
+| 5                      | \\\\\\\\\\         | \\\\\\                        |
+| 6                      | \\\\\\\\\\\\       | \\\\\\                        |
+
 - Numeric types will be distinguished from data types by the suffix.
 
 | **Serial number** | **Postfix** | **Mapping type**              | **Size (bytes)** |
@@ -88,6 +108,8 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam
 
 8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.
    Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used.
+9. Due to the fact that SQL table names do not support period (.), schemaless has also processed period (.). If there is a period (.) in the table name automatically created by schemaless, it will be automatically replaced with an underscore (\_). If you manually specify a sub table name, if there is a dot (.) in the sub table name, it will also be converted to an underscore (\_)
+10. Taos.cfg adds the configuration of smlTsDefaultName (with a string value), which only works on the client side. After configuration, the time column name of the schemaless automatic table creation can be set through this configuration. If not configured, defaults to _ts.
 
 :::tip
 All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB(64 KB since version 3.0.5.0) and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area.

docs/en/20-third-party/50-qstudio.md

+---
+sidebar_label: qStudio
+title: qStudio
+description: Step-by-Step Guide to Accessing TDengine Data with qStudio
+---
+
+qStudio is a free cross-platform SQL data analysis tool that allows easy browsing of tables, variables, functions, and configuration settings in a database. The latest version of qStudio includes built-in support for TDengine.
+
+## Prerequisites
+
+To connect TDengine using qStudio, you need to complete the following preparations:
+
+- Install qStudio: qStudio supports major operating systems, including Windows, macOS, and Linux. Please ensure you download the correct installation package for your platform from the [download page](https://www.timestored.com/qstudio/download/).
+- Set up TDengine instance: Make sure TDengine is installed and running correctly, and the taosAdapter is installed and running. For detailed information, refer to the taosAdapter User Manual.
+
+## Connecting to TDengine with qStudio
+
+1. Launch the qStudio application and select "Server" and then "Add Server..." from the menu. Choose TDengine from the Server Type dropdown.
+
+ ![qConnecting TDengine with qStudio](./qstudio/qstudio-connect-tdengine.webp)
+
+2. Configure the TDengine connection by entering the host address, port number, username, and password. If TDengine is deployed on the local machine, you can fill in the username and password only. The default username is "root," and the default password is "taosdata." Click "Test" to test the connection's availability. If the TDengine Java connector is not installed on the local machine, qStudio will prompt you to download and install it.
+
+ ![Download Java Connector](./qstudio/qstudio-jdbc-connector-download.webp)
+
+3. Once connected successfully, the screen will display as shown below. If the connection fails, check that the TDengine service and taosAdapter are running correctly, and ensure that the host address, port number, username, and password are correct.
+
+ ![Successful Connection](./qstudio/qstudio-connect-tdengine-test.webp)
+
+4. Use qStudio to select databases and tables to browse data from the TDengine server.
+
+ ![Browsing TDengine Data with qStudio](./qstudio/qstudio-browse-data.webp)
+
+5. You can also perform operations on TDengine data by executing SQL commands.
+
+ ![qStudio SQL Commands](./qstudio/qstudio-sql-execution.webp)
+
+6. qStudio supports charting functions based on the data. For more information, please refer to the [qStudio documentation](https://www.timestored.com/qstudio/help).
+
+ ![qStudio Chart](./qstudio/qstudio-chart.webp)

docs/en/25-application/_03-immigrate.md

@@ -338,7 +338,7 @@ Remark:
 Equivalent function: sum
 
 ```sql
-Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s)
+Select sum(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s)
 ```
 
 Note: This function has no interpolation requirements, so it can be directly calculated.

docs/en/28-releases/01-tdengine.md

@@ -10,6 +10,15 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
 
 import Release from "/components/ReleaseV3";
 
+## 3.1.0.0
+
+:::note IMPORTANT
+- Once you upgrade to TDengine 3.1.0.0, you cannot roll back to any previous version of TDengine. Upgrading to 3.1.0.0 will alter your data such that it cannot be read by previous versions.
+- You must remove all streams before upgrading to TDengine 3.1.0.0. If you upgrade a deployment that contains streams, the upgrade will fail and your deployment will become nonoperational.
+:::
+
+<Release type="tdengine" version="3.1.0.0" />
+
 ## 3.0.7.1
 
 <Release type="tdengine" version="3.0.7.1" />

docs/examples/R/connect_native.r

@@ -8,9 +8,13 @@ library("rJava")
 library("RJDBC")
 
 args<- commandArgs(trailingOnly = TRUE)
-driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.0.0-dist.jar"
+driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.2.4-dist.jar"
 driver = JDBC("com.taosdata.jdbc.TSDBDriver", driver_path)
 conn = dbConnect(driver, "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata")
 dbGetQuery(conn, "SELECT server_version()")
+dbSendUpdate(conn, "create database if not exists rtest")
+dbSendUpdate(conn, "create table if not exists rtest.test (ts timestamp, current float, voltage int, devname varchar(20))")
+dbSendUpdate(conn, "insert into rtest.test values (now, 1.2, 220, 'test')")
+dbGetQuery(conn, "select * from rtest.test")
 dbDisconnect(conn)
 # ANCHOR_END: demo

docs/examples/R/connect_rest.r

@@ -2,11 +2,19 @@ if (! "RJDBC" %in% installed.packages()[, "Package"]) {
   install.packages('RJDBC', repos='http://cran.us.r-project.org')
 }
 
+# ANCHOR: demo
 library("DBI")
 library("rJava")
 library("RJDBC")
-driver_path = "/home/debug/build/lib/taos-jdbcdriver-2.0.38-dist.jar"
+
+args<- commandArgs(trailingOnly = TRUE)
+driver_path = args[1] # path to jdbc-driver for example: "/root/taos-jdbcdriver-3.2.4-dist.jar"
 driver = JDBC("com.taosdata.jdbc.rs.RestfulDriver", driver_path)
 conn = dbConnect(driver, "jdbc:TAOS-RS://localhost:6041?user=root&password=taosdata")
 dbGetQuery(conn, "SELECT server_version()")
-dbDisconnect(conn)
+dbSendUpdate(conn, "create database if not exists rtest")
\ No newline at end of file
+dbSendUpdate(conn, "create table if not exists rtest.test (ts timestamp, current float, voltage int, devname varchar(20))")
+dbSendUpdate(conn, "insert into rtest.test values (now, 1.2, 220, 'test')")
+dbGetQuery(conn, "select * from rtest.test")
+dbDisconnect(conn)
+# ANCHOR_END: demo

docs/examples/R/readme.txt

+apt install -y libbz2-dev libpcre2-dev libicu-dev

docs/examples/java/pom.xml

@@ -22,7 +22,7 @@
     <dependency>
       <groupId>com.taosdata.jdbc</groupId>
       <artifactId>taos-jdbcdriver</artifactId>
-      <version>3.2.1</version>
+      <version>3.2.4</version>
     </dependency>
     <!--    ANCHOR_END: dep-->
     <dependency>
@@ -33,4 +33,4 @@
     </dependency>
   </dependencies>
 
-</project>
+</project>
\ No newline at end of file

docs/zh/05-get-started/01-docker.md

@@ -28,6 +28,21 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043
 
 注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。
 
+如果需要将数据持久化到本机的某一个文件夹，则执行下边的命令：
+
+```shell
+docker run -d -v ~/data/taos/dnode/data:/var/lib/taos \
+  -v ~/data/taos/dnode/log:/var/log/taos \
+  -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
+```
+
+:::note
+
+- /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/data为你自己的数据目录
+- /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/log为你自己的日志目录
+  
+:::
+
 确定该容器已经启动并且在正常运行。
 
 ```shell
@@ -108,4 +123,4 @@ SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(
 
 ## 其它
 
-更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。
+更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [用 Docker 部署 TDengine](../../deployment/docker)。

docs/zh/07-develop/01-connect/index.md

@@ -82,7 +82,7 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速
 <dependency>
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>3.2.1</version>
+  <version>3.2.4</version>
 </dependency>
 ```
 

docs/zh/08-connector/14-java.mdx

@@ -446,7 +446,7 @@ TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据
 **注意**：
 
 - JDBC REST 连接目前不支持参数绑定
-- 以下示例代码基于 taos-jdbcdriver-3.2.1
+- 以下示例代码基于 taos-jdbcdriver-3.2.4
 - binary 类型数据需要调用 setString 方法，nchar 类型数据需要调用 setNString 方法
 - 预处理语句中指定数据库与子表名称不要使用 `db.?`，应直接使用 `?`，然后在 setTableName 中指定数据库，如：`prepareStatement.setTableName("db.t1")`。
 

docs/zh/08-connector/43-r-lang.mdx

+---
+toc_max_heading_level: 4
+sidebar_label: R
+title: R Language Connector
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+import Rdemo from "../07-develop/01-connect/_connect_r.mdx"
+
+通过 R 语言中的 RJDBC 库可以使 R 语言程序支持访问 TDengine 数据。以下是安装过程、配置过程以及 R 语言示例代码。
+
+## 安装过程
+
+在开始之前，请确保已经安装了R语言环境。然后按照以下步骤安装和配置RJDBC库：
+
+1. 安装Java Development Kit (JDK)：RJDBC库需要依赖Java环境。请从Oracle官方网站下载适合您操作系统的JDK，并按照安装指南进行安装。
+
+2. 安装RJDBC库：在R控制台中执行以下命令来安装RJDBC库。
+
+```r
+install.packages("RJDBC", repos='http://cran.us.r-project.org')
+```
+
+:::note
+1. Ubuntu 系统自带的 R 语言软件版本 4.2 在调用 RJDBC 库会产生无响应 bug，请安装 R 语言[官网](https://www.r-project.org/)的安装包。
+2. 在 Linux 上安装 RJDBC 包可能需要安装编译需要的组件，以 Ubuntu 为例执行 `apt install -y libbz2-dev libpcre2-dev libicu-dev` 命令安装。
+3. 在 Windows 系统上需要设置 JAVA_HOME 环境变量。
+:::
+
+3. 下载 TDengine JDBC 驱动程序：访问 maven.org 网站，下载 TDengine JDBC 驱动程序（taos-jdbcdriver-X.X.X-dist.jar）。
+
+4. 将 TDengine JDBC 驱动程序放置在适当的位置：在您的计算机上选择一个合适的位置，将 TDengine JDBC 驱动程序文件（taos-jdbcdriver-X.X.X-dist.jar）保存在此处。
+
+## 配置过程
+
+完成了安装步骤后，您需要进行一些配置，以便RJDBC库能够正确连接和访问TDengine时序数据库。
+
+1. 在 R 脚本中加载 RJDBC 和其他必要的库：
+
+```r
+library(DBI)
+library(rJava)
+library(RJDBC)
+```
+
+2. 设置 JDBC 驱动程序和 JDBC URL：
+
+```r
+# 设置JDBC驱动程序路径（根据您实际保存的位置进行修改）
+driverPath <- "/path/to/taos-jdbcdriver-X.X.X-dist.jar"
+
+# 设置JDBC URL（根据您的具体环境进行修改）
+url <- "jdbc:TAOS://localhost:6030/?user=root&password=taosdata"
+```
+
+3. 加载 JDBC 驱动程序：
+
+```r
+# 加载JDBC驱动程序
+drv <- JDBC("com.taosdata.jdbc.TSDBDriver", driverPath)
+```
+
+4. 创建 TDengine 数据库连接：
+
+```r
+# 创建数据库连接
+conn <- dbConnect(drv, url)
+```
+
+5. 连接成功后，您可以使用 conn 对象进行各种数据库操作，如查询数据、插入数据等。
+
+6. 最后，不要忘记在使用完成后关闭数据库连接：
+
+```r
+# 关闭数据库连接
+dbDisconnect(conn)
+```
+
+## 使用 RJDBC 的 R 语言示例代码
+
+以下是一个使用 RJDBC 库连接 TDengine 时序数据库并执行查询操作的示例代码：
+
+<Rdemo/>
+
+请根据您的实际情况修改JDBC驱动程序、JDBC URL、用户名、密码以及SQL查询语句，以适配您的 TDengine 时序数据库环境和要求。
+
+通过以上步骤和示例代码，您可以在 R 语言环境中使用 RJDBC 库访问 TDengine 时序数据库，进行数据查询和分析等操作。

docs/zh/14-reference/11-docker/index.md → docs/zh/10-deployment/02-docker.md

 ---
 title: 用 Docker 部署 TDengine
+sidebar_label: Docker
 description: '本章主要介绍如何在容器中启动 TDengine 服务并访问它'
 ---
 
@@ -10,8 +11,17 @@ description: '本章主要介绍如何在容器中启动 TDengine 服务并访
 TDengine 镜像启动时默认激活 HTTP 服务，使用下列命令
 
 ```shell
-docker run -d --name tdengine -p 6041:6041 tdengine/tdengine
+docker run -d --name tdengine \
+-v ~/data/taos/dnode/data:/var/lib/taos \
+-v ~/data/taos/dnode/log:/var/log/taos \
+-p 6041:6041 tdengine/tdengine
 ```
+:::note
+
+- /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/data为你自己的数据目录
+- /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode/log为你自己的日志目录
+
+:::
 
 以上命令启动了一个名为“tdengine”的容器，并把其中的 HTTP 服务的端 6041 映射到了主机端口 6041。使用如下命令可以验证该容器中提供的 HTTP 服务是否可用：
 
@@ -291,38 +301,37 @@ services:
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
+    ports:
+      - 6041:6041
+      - 6030:6030
     volumes:
-      - taosdata-td1:/var/lib/taos/
-      - taoslog-td1:/var/log/taos/
+      # /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/data为你自己的数据目录
+      - ~/data/taos/dnode1/data:/var/lib/taos
+      # /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/log为你自己的日志目录
+      - ~/data/taos/dnode1/log:/var/log/taos
   td-2:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td2:/var/lib/taos/
-      - taoslog-td2:/var/log/taos/
+      - ~/data/taos/dnode2/data:/var/lib/taos
+      - ~/data/taos/dnode2/log:/var/log/taos
   td-3:
     image: tdengine/tdengine:$VERSION
     environment:
       TAOS_FQDN: "td-3"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td3:/var/lib/taos/
-      - taoslog-td3:/var/log/taos/
-volumes:
-  taosdata-td1:
-  taoslog-td1:
-  taosdata-td2:
-  taoslog-td2:
-  taosdata-td3:
-  taoslog-td3:
+      - ~/data/taos/dnode3/data:/var/lib/taos
+      - ~/data/taos/dnode3/log:/var/log/taos
 ```
 
 :::note
 
 * `VERSION` 环境变量被用来设置 tdengine image tag
 * 在新创建的实例上必须设置 `TAOS_FIRST_EP` 以使其能够加入 TDengine 集群；如果有高可用需求，则需要同时使用 `TAOS_SECOND_EP`
+
 :::
 
 2. 启动集群
@@ -397,24 +406,22 @@ networks:
 services:
   td-1:
     image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
     environment:
       TAOS_FQDN: "td-1"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td1:/var/lib/taos/
-      - taoslog-td1:/var/log/taos/
+      # /var/lib/taos: TDengine 默认数据文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/data为你自己的数据目录
+      - ~/data/taos/dnode1/data:/var/lib/taos
+      # /var/log/taos: TDengine 默认日志文件目录。可通过[配置文件]修改位置。你可以修改~/data/taos/dnode1/log为你自己的日志目录
+      - ~/data/taos/dnode1/log:/var/log/taos
   td-2:
     image: tdengine/tdengine:$VERSION
-    networks:
-      - inter
     environment:
       TAOS_FQDN: "td-2"
       TAOS_FIRST_EP: "td-1"
     volumes:
-      - taosdata-td2:/var/lib/taos/
-      - taoslog-td2:/var/log/taos/
+      - ~/data/taos/dnode2/data:/var/lib/taos
+      - ~/data/taos/dnode2/log:/var/log/taos
   adapter:
     image: tdengine/tdengine:$VERSION
     entrypoint: "taosadapter"
@@ -446,11 +453,6 @@ services:
         >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
         nginx -g 'daemon off;'",
       ]
-volumes:
-  taosdata-td1:
-  taoslog-td1:
-  taosdata-td2:
-  taoslog-td2:
 ```
 
 ## 使用 docker swarm 部署

docs/zh/10-deployment/index.md

@@ -6,7 +6,7 @@ description: 部署 TDengine 集群的多种方式
 
 TDengine 支持集群，提供水平扩展的能力。如果需要获得更高的处理能力，只需要多增加节点即可。TDengine 采用虚拟节点技术，将一个节点虚拟化为多个虚拟节点，以实现负载均衡。同时，TDengine可以将多个节点上的虚拟节点组成虚拟节点组，通过多副本机制，以保证供系统的高可用。TDengine的集群功能完全开源。
 
-本章节主要介绍如何在主机上人工部署集群，以及如何使用 Kubernetes 和 Helm部署集群。
+本章节主要介绍如何在主机上人工部署集群，docker部署，以及如何使用 Kubernetes 和 Helm部署集群。
 
 ```mdx-code-block
 import DocCardList from '@theme/DocCardList';

docs/zh/12-taos-sql/06-select.md

@@ -315,7 +315,7 @@ WHERE (column|tbname) match/MATCH/nmatch/NMATCH _regex_
 
 ### 使用限制
 
-只能针对表名（即 tbname 筛选）、binary/nchar 类型标签值进行正则表达式过滤，不支持普通列的过滤。
+只能针对表名（即 tbname 筛选）、binary/nchar 类型值进行正则表达式过滤。
 
 正则匹配字符串长度不能超过 128 字节。可以通过参数 _maxRegexStringLen_ 设置和调整最大允许的正则匹配字符串，该参数是客户端配置参数，需要重启才能生效。
 

docs/zh/12-taos-sql/10-function.md

@@ -1265,3 +1265,140 @@ SELECT SERVER_STATUS();
 ```
 
 **说明**：检测服务端是否所有 dnode 都在线，如果是则返回成功，否则返回无法建立连接的错误。
+
+
+## Geometry 函数
+
+### Geometry 输入函数：
+
+#### ST_GeomFromText
+
+```sql
+ST_GeomFromText(VARCHAR WKT expr)
+```
+
+**功能说明**：根据 Well-Known Text (WKT) 表示从指定的几何值创建几何数据。
+
+**返回值类型**：GEOMETRY
+
+**适用数据类型**：VARCHAR
+
+**适用表类型**：标准表和超表
+
+**使用说明**：输入可以是 WKT 字符串之一，例如点（POINT）、线串（LINESTRING）、多边形（POLYGON）、多点集（MULTIPOINT）、多线串（MULTILINESTRING）、多多边形（MULTIPOLYGON）、几何集合（GEOMETRYCOLLECTION）。输出是以二进制字符串形式定义的 GEOMETRY 数据类型。
+
+### Geometry 输出函数：
+
+#### ST_AsText
+
+```sql
+ST_AsText(GEOMETRY geom)
+```
+
+**功能说明**：从几何数据中返回指定的 Well-Known Text (WKT) 表示。
+
+**返回值类型**：VARCHAR
+
+**适用数据类型**：GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：输出可以是 WKT 字符串之一，例如点（POINT）、线串（LINESTRING）、多边形（POLYGON）、多点集（MULTIPOINT）、多线串（MULTILINESTRING）、多多边形（MULTIPOLYGON）、几何集合（GEOMETRYCOLLECTION）。
+
+### Geometry 关系函数：
+
+#### ST_Intersects
+
+```sql
+ST_Intersects(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+##功能说明**：比较两个几何对象，并在它们相交时返回 true。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：如果两个几何对象有任何一个共享点，则它们相交。
+
+#### ST_Equals
+
+```sql
+ST_Equals(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**功能说明**：如果给定的几何对象是"空间相等"的，则返回 TRUE。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**："空间相等"意味着 ST_Contains(A,B) = true 和 ST_Contains(B,A) = true，并且点的顺序可能不同，但表示相同的几何结构。
+
+#### ST_Touches
+
+```sql
+ST_Touches(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**功能说明**：如果 A 和 B 相交，但它们的内部不相交，则返回 TRUE。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：A 和 B 至少有一个公共点，并且这些公共点位于至少一个边界中。对于点/点输入，关系始终为 FALSE，因为点没有边界。
+
+#### ST_Covers
+
+```sql
+ST_Covers(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**功能说明**：如果 B 中的每个点都位于几何形状 A 内部（与内部或边界相交），则返回 TRUE。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：A 包含 B 意味着 B 中的没有点位于 A 的外部（在外部）。
+
+#### ST_Contains
+
+```sql
+ST_Contains(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**功能说明**：如果 A 包含 B，描述：如果几何形状 A 包含几何形状 B，则返回 TRUE。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：A 包含 B 当且仅当 B 的所有点位于 A 的内部（即位于内部或边界上）（或等效地，B 的没有点位于 A 的外部），并且 A 和 B 的内部至少有一个公共点。
+
+#### ST_ContainsProperly
+
+```sql
+ST_ContainsProperly(GEOMETRY geomA, GEOMETRY geomB)
+```
+
+**功能说明**：如果 B 的每个点都位于 A 内部，则返回 TRUE。
+
+**返回值类型**：BOOL
+
+**适用数据类型**：GEOMETRY，GEOMETRY
+
+**适用表类型**：标准表和超表
+
+**使用说明**：B 的没有点位于 A 的边界或外部。

docs/zh/12-taos-sql/22-meta.md

@@ -98,7 +98,7 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数
 | 21  |      cachesize       | INT              | 表示每个 vnode 中用于缓存子表最近数据的内存大小。需要注意，`cachesize` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。  |
 | 22  |      wal_level       | INT              | WAL 级别。需要注意，`wal_level` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                         |
 | 23  |   wal_fsync_period   | INT              | 数据落盘周期。需要注意，`wal_fsync_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                     |
-| 24  | wal_retention_period | INT              | WAL 的保存时长。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                   |
+| 24  | wal_retention_period | INT              | WAL 的保存时长，单位为秒。需要注意，`wal_retention_period` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                   |
 | 25  |  wal_retention_size  | INT              | WAL 的保存上限。需要注意，`wal_retention_size` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                   |
 | 26  |   stt_trigger   | SMALLINT | 触发文件合并的落盘文件的个数。需要注意，`stt_trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 |
 | 27  |   table_prefix   | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。需要注意，`table_prefix` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 |
@@ -298,3 +298,13 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数
 | 7   | target_table | BINARY(192)  | 流计算写入的目标表                                                                                                   |
 | 8   |  watermark   | BIGINT       | watermark，详见 SQL 手册流式计算。需要注意，`watermark` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。      |
 | 9   |   trigger    | INT          | 计算结果推送模式，详见 SQL 手册流式计算。需要注意，`trigger` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。 |
+
+## INS_USER_PRIVILEGES
+
+| #   |   **列名**    | **数据类型** | **说明**                                                                                                             |
+| --- | :----------: | ------------ | -------------------------------------------------------------------------------------------------------------------- |
+| 1   | user_name    | VARCHAR(24)       | 用户名
+| 2   | privilege    | VARCHAR(10)       | 权限描述
+| 3   | db_name      | VARCHAR(65)       | 数据库名称
+| 4   | table_name   | VARCHAR(193)      | 表名称
+| 5   | condition    | VARCHAR(49152)    | 子表权限过滤条件

docs/zh/14-reference/09-support-platform/index.md

@@ -7,12 +7,13 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
 
 |              | **Windows server 2016/2019** | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18 以上** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **macOS** |
 | ------------ | ---------------------------- | ----------------- | ---------------- | ---------------- | ------------ | ----------------- | ---------------- | --------- |
-| X64          | ●                            | ●                 | ●                | ●                | ●            | ●                 | ●                | ●         |
-| 树莓派 ARM64 |                              |                   | ●                |                  |              |                   |                  |           |
-| 华为云 ARM64 |                              |                   |                  | ●                |              |                   |                  |           |
-| M1           |                              |                   |                  |                  |              |                   |                  | ●         |
+| X64          | ●/E                          | ●/E               | ●                | ●                | ●/E          | ●/E               | ●/E              | ●         |
+| 树莓派 ARM64  |                              |                   | ●                |                  |              |                   |                  |           |
+| 华为云 ARM64  |                              |                   |                  | ●                |              |                   |                  |           |
+| M1          |                               |                   |                  |                  |              |                   |                  | ●         |
 
-注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。
+注：1) ● 表示经过官方测试验证， ○ 表示非官方测试验证，E 表示仅企业版支持。
+   2) 社区版仅支持主流操作系统的较新版本，包括 Ubuntu 18+/CentOS 7+/RetHat/Debian/CoreOS/FreeBSD/OpenSUSE/SUSE Linux/Fedora/macOS 等。如果有其他操作系统及版本的需求，请联系企业版支持。
 
 ## TDengine 客户端和连接器支持的平台列表
 

docs/zh/14-reference/11-docker/_category_.yml

-label: TDengine Docker 镜像
\ No newline at end of file

docs/zh/14-reference/12-config/index.md

@@ -685,7 +685,16 @@ charset 的有效值是 UTF-8。
 | 适用范围 | 仅客户端适用                                             |
 | 含义     | schemaless 列数据是否顺序一致，从3.0.3.0开始，该配置废弃 |
 | 值域     | 0：不一致；1: 一致                                       |
-| 缺省值   | 0                                                        |
+| 缺省值   | 0 
+
+### smlTsDefaultName
+
+| 属性     | 说明                                                     |
+| -------- | -------------------------------------------------------- |
+| 适用范围 | 仅客户端适用                                             |
+| 含义     | schemaless自动建表的时间列名字通过该配置设置 |
+| 类型     | 字符串                                       |
+| 缺省值   | _ts |
 
 ## 其他
 

docs/zh/14-reference/13-schemaless/13-schemaless.md

@@ -35,12 +35,32 @@ tag_set 中的所有的数据自动转化为 nchar 数据类型，并不需要
 
 - 如果两边有英文双引号，表示 BINARY(32) 类型。例如 `"abc"`。
 - 如果两边有英文双引号而且带有 L 前缀，表示 NCHAR(32) 类型。例如 `L"报错信息"`。
-- 对空格、等号（=）、逗号（,）、双引号（"），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号）
+- 对空格、等号（=）、逗号（,）、双引号（"）、反斜杠（\），前面需要使用反斜杠（\）进行转义。（都指的是英文半角符号）。具体转义规则如下：
+
+| **序号**  | **域**      | **需转义字符**                  |
+| -------- | ----------- | ----------------------------- |
+| 1        | 超级表名      | 逗号，空格                      |
+| 2        | 标签名       | 逗号，等号，空格                 |
+| 3        | 标签值       | 逗号，等号，空格                 |
+| 4        | 列名         | 逗号，等号，空格                 |
+| 5        | 列值         | 双引号，反斜杠                  |
+
+两个连续的反斜杠，第一个作为转义符，只有一个反斜杠则无需转义. 反斜杠转义规则举例如下:
+
+| **序号**  | **反斜杠**     | **转义为**                     |
+| -------- | -----------   | ----------------------------- |
+| 1        | \             | \                             |
+| 2        | \\\\          | \                             |
+| 3        | \\\\\\        | \\\\                          |
+| 4        | \\\\\\\\      | \\\\                          |
+| 5        | \\\\\\\\\\    | \\\\\\                        |
+| 6        | \\\\\\\\\\\\  | \\\\\\                        |
+
 - 数值类型将通过后缀来区分数据类型：
 
-| **序号** | **后缀**    | **映射类型**                  | **大小(字节)** |
+| **序号**  | **后缀**    | **映射类型**                  | **大小(字节)** |
 | -------- | ----------- | ----------------------------- | -------------- |
-| 1        | 无或 f64    | double                        | 8              |
+| 1        | 无或 f64     | double                        | 8              |
 | 2        | f32         | float                         | 4              |
 | 3        | i8/u8       | TinyInt/UTinyInt              | 1              |
 | 4        | i16/u16     | SmallInt/USmallInt            | 2              |
@@ -84,7 +104,9 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
 6. 对 BINARY 或 NCHAR 列，如果数据行中所提供值的长度超出了列类型的限制，自动增加该列允许存储的字符长度上限（只增不减），以保证数据的完整保存。
 7. 整个处理过程中遇到的错误会中断写入过程，并返回错误代码。
 8. 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常，从3.0.3.0开始，自动检测顺序是否一致，该配置废弃。
-
+9. 由于sql建表表名不支持点号（.），所以schemaless也对点号（.）做了处理，如果schemaless自动建表的表名如果有点号（.），会自动替换为下划线（\_）。如果手动指定子表名的话，子表名里有点号（.），同样转化为下划线（\_）。
+10. taos.cfg 增加 smlTsDefaultName 配置（值为字符串），只在client端起作用，配置后，schemaless自动建表的时间列名字可以通过该配置设置。不配置的话，默认为 _ts
+   
 :::tip
 无模式所有的处理逻辑，仍会遵循 TDengine 对数据结构的底层限制，例如每行数据的总长度不能超过
 48KB（从 3.0.5.0 版本开始为 64KB），标签值的总长度不超过16KB。这方面的具体限制约束请参见 [TDengine SQL 边界限制](/taos-sql/limit)

docs/zh/20-third-party/50-qstudio.md

+---
+sidebar_label: qStudio
+title: qStudio
+description: 使用 qStudio 存取 TDengine 数据的详细指南
+---
+
+qStudio 是一款免费的多平台 SQL 数据分析工具，可以轻松浏览数据库中的表、变量、函数和配置设置。最新版本 qStudio 内嵌支持 TDengine。
+
+## 前置条件
+
+使用 qStudio 连接 TDengine 需要以下几方面的准备工作。
+
+- 安装 qStudio。qStudio 支持主流操作系统包括 Windows、macOS 和 Linux。请注意[下载](https://www.timestored.com/qstudio/download/)正确平台的安装包。
+- 安装 TDengine 实例，请确认 TDengine 正常运行，并且 taosAdapter 已经安装并正常运行，具体细节请参考 [taosAdapter 的使用手册](/reference/taosadapter)。
+
+## 使用 qStudio 连接 TDengine
+
+1. 启动 qStudio 应用，从菜单项选择“Server” 和 “Add Server...”，然后在 Server Type 下拉框中选择 TDengine。
+
+ ![qStudio 连接 TDengine](./qstudio/qstudio-connect-tdengine.webp)
+
+2. 配置 TDengine 连接，填入主机地址、端口号、用户名和密码。如果 TDengine 部署在本机，可以只填用户名和密码，默认用户名为 root，默认密码为 taosdata。点击“Test”可以对连接是否可用进行测试。如果本机没有安装 TDengine Java
+ 连接器，qStudio 会提示下载安装。
+
+ ![下载 Java 连接器](./qstudio/qstudio-jdbc-connector-download.webp)
+
+3. 连接成功将显示如下图所示。如果显示连接失败，请检查 TDengine 服务和 taosAdapter 是否正确运行，主机地址、端口号、用户名和密码是否正确。
+
+ ![连接成功](./qstudio/qstudio-connect-tdengine-test.webp)
+
+4. 使用 qStudio 选择数据库和表可以浏览 TDengine 服务的数据。
+
+ ![qStudio 浏览 TDengine 数据](./qstudio/qstudio-browse-data.webp)
+
+5. 也可以通过执行 SQL 命令的方式对 TDengine 数据进行操作。
+
+ ![qStudio SQL 命令](./qstudio/qstudio-sql-execution.webp)
+
+6. qStudio 支持根据数据绘制图表等功能，请参考 [qStudio 的帮助文档](https://www.timestored.com/qstudio/help)
+
+ ![qStudio Chart 功能](./qstudio/qstudio-chart.webp)

docs/zh/25-application/03-immigrate.md

@@ -371,7 +371,7 @@ Select min(val) from table_name
 等效函数：sum
 
 ```sql
-Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s)
+Select sum(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s)
 ```
 
 备注：该函数无插值需求，因此可用直接计算。

docs/zh/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
 
 import Release from "/components/ReleaseV3";
 
+## 3.1.0.0
+
+<Release type="tdengine" version="3.1.0.0" />
+
 ## 3.0.7.1
 
 <Release type="tdengine" version="3.0.7.1" />

examples/JDBC/JDBCDemo/readme.md

@@ -20,18 +20,12 @@ mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" -De
 ```
 
 ## Compile the Demo Code and Run It
-To compile taos-jdbcdriver, go to the source directory ``TDengine/src/connector/jdbc`` and execute
 ```
 mvn clean package -Dmaven.test.skip=true
 ```
 
-To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute
+To run JDBCDemo.jar, execute
 ```
-mvn clean package assembly:single
-```
-
-To run JDBCDemo.jar, go to ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute
-```
-java -Djava.ext.dirs=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host [HOSTNAME]
+java -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host [HOSTNAME]
 ```
 

examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java

@@ -16,8 +16,6 @@ public class JdbcRestfulDemo {
 
             Properties properties = new Properties();
             properties.setProperty("charset", "UTF-8");
-            properties.setProperty("locale", "en_US.UTF-8");
-            properties.setProperty("timezone", "UTC-8");
 
             Connection conn = DriverManager.getConnection(url, properties);
             Statement stmt = conn.createStatement();

include/common/tglobal.h

@@ -58,6 +58,7 @@ extern int32_t tsTagFilterResCacheSize;
 extern int32_t tsNumOfRpcThreads;
 extern int32_t tsNumOfRpcSessions;
 extern int32_t tsTimeToGetAvailableConn;
+extern int32_t tsKeepAliveIdle;
 extern int32_t tsNumOfCommitThreads;
 extern int32_t tsNumOfTaskQueueThreads;
 extern int32_t tsNumOfMnodeQueryThreads;
@@ -169,6 +170,8 @@ extern char tsUdfdLdLibPath[];
 // schemaless
 extern char tsSmlChildTableName[];
 extern char tsSmlTagName[];
+extern bool tsSmlDot2Underline;
+extern char tsSmlTsDefaultName[];
 // extern bool    tsSmlDataFormat;
 // extern int32_t tsSmlBatchSize;
 

include/libs/executor/storageapi.h

@@ -368,6 +368,8 @@ typedef struct SStateStore {
   bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
   bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
   void (*updateInfoDestroy)(SUpdateInfo* pInfo);
+  void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count);
+  void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count);
 
   SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark);
   void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);

include/libs/stream/tstream.h

@@ -30,6 +30,7 @@ extern "C" {
 
 typedef struct SStreamTask SStreamTask;
 
+#define SSTREAM_TASK_VER 1
 enum {
   STREAM_STATUS__NORMAL = 0,
   STREAM_STATUS__STOP,
@@ -266,13 +267,13 @@ typedef struct SCheckpointInfo {
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
-  int8_t        taskStatus;
-  int8_t        downstreamReady; // downstream tasks are all ready now, if this flag is set
-  int8_t        schedStatus;
-  int8_t        keepTaskStatus;
-  bool          transferState;
-  int8_t        timerActive;     // timer is active
-  int8_t        pauseAllowed;    // allowed task status to be set to be paused
+  int8_t taskStatus;
+  int8_t downstreamReady;  // downstream tasks are all ready now, if this flag is set
+  int8_t schedStatus;
+  int8_t keepTaskStatus;
+  bool   transferState;
+  int8_t timerActive;   // timer is active
+  int8_t pauseAllowed;  // allowed task status to be set to be paused
 } SStreamStatus;
 
 typedef struct SHistDataRange {
@@ -309,6 +310,7 @@ typedef struct {
 } STaskTimestamp;
 
 struct SStreamTask {
+  int64_t          ver;
   SStreamId        id;
   SSTaskBasicInfo  info;
   STaskOutputInfo  outputInfo;
@@ -589,10 +591,10 @@ bool    streamTaskShouldPause(const SStreamStatus* pStatus);
 bool    streamTaskIsIdle(const SStreamTask* pTask);
 int32_t streamTaskEndScanWAL(SStreamTask* pTask);
 
-SStreamChildEpInfo * streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
-int32_t streamScanExec(SStreamTask* pTask, int32_t batchSize);
+SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);
+int32_t             streamScanExec(SStreamTask* pTask, int32_t batchSize);
 
-char*   createStreamTaskIdStr(int64_t streamId, int32_t taskId);
+char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
 
 // recover and fill history
 void    streamTaskCheckDownstreamTasks(SStreamTask* pTask);
@@ -628,7 +630,8 @@ int32_t streamDispatchTransferStateMsg(SStreamTask* pTask);
 
 // agg level
 int32_t streamTaskScanHistoryPrepare(SStreamTask* pTask);
-int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq *pReq, SRpcHandleInfo* pRpcInfo);
+int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq,
+                                          SRpcHandleInfo* pRpcInfo);
 int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask);
 
 // stream task meta
@@ -642,7 +645,7 @@ int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 int32_t      streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
 int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int32_t taskId);
-int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);   // todo remove it
+int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);  // todo remove it
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 
@@ -659,7 +662,6 @@ int32_t streamTaskReleaseState(SStreamTask* pTask);
 int32_t streamTaskReloadState(SStreamTask* pTask);
 int32_t streamAlignTransferState(SStreamTask* pTask);
 
-
 #ifdef __cplusplus
 }
 #endif

include/libs/stream/tstreamUpdate.h

@@ -53,6 +53,8 @@ void         updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo);
 void         updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo);
 int32_t      updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo);
 int32_t      updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
+void         windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
+void         windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -780,6 +780,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TMQ_INVALID_TOPIC              TAOS_DEF_ERROR_CODE(0, 0x4009)
 #define TSDB_CODE_TMQ_NEED_INITIALIZED           TAOS_DEF_ERROR_CODE(0, 0x4010)
 #define TSDB_CODE_TMQ_NO_COMMITTED               TAOS_DEF_ERROR_CODE(0, 0x4011)
+#define TSDB_CODE_TMQ_SAME_COMMITTED_VALUE       TAOS_DEF_ERROR_CODE(0, 0x4012)
 
 // stream
 #define TSDB_CODE_STREAM_TASK_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x4100)

include/util/tdef.h

@@ -200,7 +200,7 @@ typedef enum ELogicConditionType {
 #define TSDB_STREAM_NAME_LEN          193                                // it is a null-terminated string
 #define TSDB_DB_NAME_LEN              65
 #define TSDB_DB_FNAME_LEN             (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
-#define TSDB_PRIVILEDGE_CONDITION_LEN 200
+#define TSDB_PRIVILEDGE_CONDITION_LEN 48*1024
 
 #define TSDB_FUNC_NAME_LEN       65
 #define TSDB_FUNC_COMMENT_LEN    1024 * 1024
@@ -369,8 +369,13 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SCHEMALESS_OFF          0
 #define TSDB_DEFAULT_DB_SCHEMALESS      TSDB_DB_SCHEMALESS_OFF
 #define TSDB_MIN_STT_TRIGGER            1
-#define TSDB_MAX_STT_TRIGGER            16
-#define TSDB_DEFAULT_SST_TRIGGER        2
+#ifdef TD_ENTERPRISE
+#define TSDB_MAX_STT_TRIGGER     16
+#define TSDB_DEFAULT_SST_TRIGGER 2
+#else
+#define TSDB_MAX_STT_TRIGGER     1
+#define TSDB_DEFAULT_SST_TRIGGER 1
+#endif
 #define TSDB_MIN_HASH_PREFIX            (2 - TSDB_TABLE_NAME_LEN)
 #define TSDB_MAX_HASH_PREFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_PREFIX        0

packaging/tools/install.sh

@@ -613,6 +613,11 @@ function install_examples() {
   fi
 }
 
+function install_web() {
+  if [ -d "${script_dir}/share" ]; then
+    ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||:
+  fi
+}
 
 function clean_service_on_sysvinit() {
   if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then
@@ -888,6 +893,7 @@ function updateProduct() {
   fi
 
   install_examples
+  install_web
   if [ -z $1 ]; then
     install_bin
     install_service
@@ -898,29 +904,29 @@ function updateProduct() {
     openresty_work=false
 
     echo
-    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
+    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}"
     [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
+      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml"
     if ((${service_mod} == 0)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}"
     elif ((${service_mod} == 1)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}service ${serverName2} start${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}service ${serverName2} start${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}"
     else
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ./${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ./${serverName2}${NC}"
       [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter &${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}"
     fi
     
-    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper &${NC}"
+    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}"
 
     if [ ${openresty_work} = 'true' ]; then
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
     else
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}"
     fi
 
     if ((${prompt_force} == 1)); then
@@ -968,7 +974,7 @@ function installProduct() {
       install_connector
   fi
   install_examples  
-
+  install_web
   if [ -z $1 ]; then # install service and client
     # For installing new
     install_bin
@@ -982,24 +988,24 @@ function installProduct() {
 
     # Ask if to start the service
     echo
-    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
+    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}"
     [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
+      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml"
     if ((${service_mod} == 0)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}"
     elif ((${service_mod} == 1)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}service ${serverName2} start${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}service ${serverName2} start${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}"
     else
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${serverName2}${NC}"
       [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter &${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}"
     fi
 
-    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper &${NC}"
+    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}"
 
     if [ ! -z "$firstEp" ]; then
       tmpFqdn=${firstEp%%:*}
@@ -1010,14 +1016,14 @@ function installProduct() {
         tmpPort=""
       fi
       if [[ "$tmpPort" != "" ]]; then
-        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
       else
-        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
       fi
       echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
       echo
     elif [ ! -z "$serverFqdn" ]; then
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}"
       echo
     fi
 

packaging/tools/makepkg.sh

@@ -319,6 +319,11 @@ if [[ $dbName == "taos" ]]; then
     mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
   fi
 
+  if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then    
+    mkdir -p ${install_dir}/share/        
+    cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:    
+  fi
+
 fi
 
 # Copy driver

source/client/inc/clientSml.h

@@ -64,8 +64,8 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-#define TS        "_ts"
-#define TS_LEN    3
+//#define TS        "_ts"
+//#define TS_LEN    3
 #define VALUE     "_value"
 #define VALUE_LEN 6
 
@@ -258,6 +258,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
+void    smlStrReplace(char* src, int32_t len);
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientSml.c

@@ -104,7 +104,7 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   SUserAuthRes authRes = {0};
 
   code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
-
+  nodesDestroyNode(authRes.pCond);
 
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
@@ -114,6 +114,15 @@ inline bool smlDoubleToInt64OverFlow(double num) {
   return false;
 }
 
+void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
     memset(pBuf->buf, 0, pBuf->len);
@@ -193,6 +202,9 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) {
     if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
       memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
       strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN));
+      if(tsSmlDot2Underline){
+        smlStrReplace(childTableName, strlen(childTableName));
+      }
       taosArrayRemove(tags, i);
       break;
     }
@@ -838,6 +850,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
     char  *measure = taosMemoryMalloc(superTableLen);
     memcpy(measure, superTable, superTableLen);
     PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen);
+    smlStrReplace(measure, superTableLen);
     memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
     memcpy(pName.tname, measure, superTableLen);
     taosMemoryFree(measure);
@@ -1051,7 +1064,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
     taosMemoryFreeClear(sTableData->tableMeta);
     sTableData->tableMeta = pTableMeta;
     uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
-           pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
+           pTableMeta->sversion, pTableMeta->tversion);
+    tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
   }
   uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat,
          info->needModifySchema);
@@ -1394,7 +1408,14 @@ static int32_t smlInsertData(SSmlHandle *info) {
   SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
   while (oneTable) {
     SSmlTableInfo *tableData = *oneTable;
-    tstrncpy(pName.tname, tableData->sTableName, tableData->sTableNameLen + 1);
+
+    int   measureLen = tableData->sTableNameLen;
+    char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen);
+    memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+    memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
+    memcpy(pName.tname, measure, measureLen);
 
     if (info->pRequest->tableList == NULL) {
       info->pRequest->tableList = taosArrayInit(1, sizeof(SName));
@@ -1411,6 +1432,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
 
     code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
     if(code != TSDB_CODE_SUCCESS){
+      taosMemoryFree(measure);
       return code;
     }
 
@@ -1418,6 +1440,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
     code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName);
+      taosMemoryFree(measure);
       return code;
     }
     taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg));
@@ -1426,6 +1449,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
         (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen);
     if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) {
       uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName);
+      taosMemoryFree(measure);
       return TSDB_CODE_SML_INTERNAL_ERROR;
     }
 
@@ -1435,11 +1459,6 @@ static int32_t smlInsertData(SSmlHandle *info) {
     uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
            tableData->uid, info->dataFormat);
 
-    int   measureLen = tableData->sTableNameLen;
-    char *measure = (char *)taosMemoryMalloc(tableData->sTableNameLen);
-    memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
-    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-
     code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
                        (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen, info->ttl, info->msgBuf.buf,
                        info->msgBuf.len);

source/client/src/clientSmlJson.c

@@ -996,8 +996,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = TS,
-                 .keyLen = TS_LEN,
+  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
+                 .keyLen = strlen(tsSmlTsDefaultName),
                  .type = TSDB_DATA_TYPE_TIMESTAMP,
                  .i = ts,
                  .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
@@ -1200,8 +1200,8 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = TS,
-                 .keyLen = TS_LEN,
+  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
+                 .keyLen = strlen(tsSmlTsDefaultName),
                  .type = TSDB_DATA_TYPE_TIMESTAMP,
                  .i = ts,
                  .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};

source/client/src/clientSmlLine.c

@@ -157,6 +157,7 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
           measure = (char *)taosMemoryMalloc(currElement->measureLen);
           memcpy(measure, currElement->measure, currElement->measureLen);
           PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+          smlStrReplace(measure, measureLen);
         }
         STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
         if (currElement->measureEscaped) {
@@ -365,6 +366,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
           measure = (char *)taosMemoryMalloc(currElement->measureLen);
           memcpy(measure, currElement->measure, currElement->measureLen);
           PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+          smlStrReplace(measure, measureLen);
         }
         STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
         if (currElement->measureEscaped) {
@@ -651,8 +653,8 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
   // add ts to
-  SSmlKv kv = {.key = TS,
-               .keyLen = TS_LEN,
+  SSmlKv kv = {.key = tsSmlTsDefaultName,
+               .keyLen = strlen(tsSmlTsDefaultName),
                .type = TSDB_DATA_TYPE_TIMESTAMP,
                .i = ts,
                .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,

source/client/src/clientSmlTelnet.c

@@ -260,8 +260,8 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = TS,
-                 .keyLen = TS_LEN,
+  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
+                 .keyLen = strlen(tsSmlTsDefaultName),
                  .type = TSDB_DATA_TYPE_TIMESTAMP,
                  .i = ts,
                  .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};

source/client/src/clientTmq.c

@@ -586,31 +586,37 @@ static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STq
   if(code != 0){
     goto end;
   }
-  if (offsetVal->type > 0 && !tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) {
-    char offsetBuf[TSDB_OFFSET_LEN] = {0};
-    tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal);
+  if (offsetVal->type <= 0) {
+    code = TSDB_CODE_TMQ_INVALID_MSG;
+    goto end;
+  }
+  if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)){
+    code = TSDB_CODE_TMQ_SAME_COMMITTED_VALUE;
+    goto end;
+  }
+  char offsetBuf[TSDB_OFFSET_LEN] = {0};
+  tFormatOffset(offsetBuf, tListLen(offsetBuf), offsetVal);
 
-    char commitBuf[TSDB_OFFSET_LEN] = {0};
-    tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset);
+  char commitBuf[TSDB_OFFSET_LEN] = {0};
+  tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset);
 
-    SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0);
-    if (pParamSet == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto end;
-    }
-    code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet);
-    if (code != TSDB_CODE_SUCCESS) {
-      tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s",
-               tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno));
-      taosMemoryFree(pParamSet);
-      goto end;
-    }
-
-    tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s",
-            tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf);
-    pVg->offsetInfo.committedOffset = *offsetVal;
+  SMqCommitCbParamSet* pParamSet = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0);
+  if (pParamSet == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, offsetVal, pTopicName, pParamSet);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s",
+             tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno));
+    taosMemoryFree(pParamSet);
+    goto end;
   }
 
+  tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s",
+          tmq->consumerId, pTopicName, pVg->vgId, offsetBuf, commitBuf);
+  pVg->offsetInfo.committedOffset = *offsetVal;
+
 end:
   taosRUnLockLatch(&tmq->lock);
   return code;
@@ -650,7 +656,8 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c
   code = asyncCommitOffset(tmq, pTopicName, vgId, &offsetVal, pCommitFp, userParam);
 
 end:
-  if(code != TSDB_CODE_SUCCESS){
+  if(code != TSDB_CODE_SUCCESS && pCommitFp != NULL){
+    if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS;
     pCommitFp(tmq, code, userParam);
   }
 }
@@ -1859,8 +1866,8 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p
 static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId){
   if (!pVg->seekUpdated) {
     tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId);
-    if(reqOffset->type != 0) pVg->offsetInfo.beginOffset = *reqOffset;
-    if(rspOffset->type != 0) pVg->offsetInfo.endOffset = *rspOffset;
+    pVg->offsetInfo.beginOffset = *reqOffset;
+    pVg->offsetInfo.endOffset = *rspOffset;
   } else {
     tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId);
   }
@@ -1948,7 +1955,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
         }
         taosWUnLockLatch(&tmq->lock);
       } else {
-        tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
+        tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
                  tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch);
         pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pollRspWrapper);
@@ -1979,7 +1986,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
         taosWUnLockLatch(&tmq->lock);
         return pRsp;
       } else {
-        tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
+        tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
                  tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch);
         pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pollRspWrapper);
@@ -2036,7 +2043,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
         taosWUnLockLatch(&tmq->lock);
         return pRsp;
       } else {
-        tscDebug("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
+        tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d",
                  tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch);
         pRspWrapper = tmqFreeRspWrapper(pRspWrapper);
         taosFreeQitem(pollRspWrapper);
@@ -2350,7 +2357,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) {
   tsem_destroy(&pInfo->sem);
   taosMemoryFree(pInfo);
 
-  tscDebug("consumer:0x%" PRIx64 " sync commit done, code:%s", tmq->consumerId, tstrerror(code));
+  tscInfo("consumer:0x%" PRIx64 " sync res commit done, code:%s", tmq->consumerId, tstrerror(code));
   return code;
 }
 
@@ -2406,15 +2413,17 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId,
   tsem_init(&pInfo->sem, 0, 0);
   pInfo->code = 0;
 
-  asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo);
-
-  tsem_wait(&pInfo->sem);
-  code = pInfo->code;
+  code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo);
+  if(code == 0){
+    tsem_wait(&pInfo->sem);
+    code = pInfo->code;
+  }
 
+  if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS;
   tsem_destroy(&pInfo->sem);
   taosMemoryFree(pInfo);
 
-  tscInfo("consumer:0x%" PRIx64 " sync send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code));
+  tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code));
 
   return code;
 }
@@ -2451,10 +2460,11 @@ void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, i
 
   code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param);
 
-  tscInfo("consumer:0x%" PRIx64 " async send seek to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code));
+  tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code));
 
 end:
   if(code != 0 && cb != NULL){
+    if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS;
     cb(tmq, code, param);
   }
 }

source/client/test/clientTests.cpp

@@ -1251,7 +1251,8 @@ TEST(clientCase, td_25129) {
   }
 
   for(int i = 0; i < numOfAssign; i++){
-    printf("assign i:%d, vgId:%d, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
+    int64_t committed = tmq_committed(tmq, topicName, pAssign[i].vgId);
+    printf("assign i:%d, vgId:%d, committed:%lld, offset:%lld, start:%lld, end:%lld\n", i, pAssign[i].vgId, committed, pAssign[i].currentOffset, pAssign[i].begin, pAssign[i].end);
   }
 
   while (1) {

source/common/src/tglobal.c

@@ -14,8 +14,8 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "os.h"
 #include "tglobal.h"
+#include "os.h"
 #include "tconfig.h"
 #include "tgrant.h"
 #include "tlog.h"
@@ -49,6 +49,8 @@ bool    tsPrintAuth = false;
 int32_t tsNumOfRpcThreads = 1;
 int32_t tsNumOfRpcSessions = 10000;
 int32_t tsTimeToGetAvailableConn = 500000;
+int32_t tsKeepAliveIdle = 60;
+
 int32_t tsNumOfCommitThreads = 2;
 int32_t tsNumOfTaskQueueThreads = 4;
 int32_t tsNumOfMnodeQueryThreads = 4;
@@ -63,7 +65,7 @@ int32_t tsNumOfQnodeFetchThreads = 1;
 int32_t tsNumOfSnodeStreamThreads = 4;
 int32_t tsNumOfSnodeWriteThreads = 1;
 int32_t tsMaxStreamBackendCache = 128;  // M
-int32_t tsPQSortMemThreshold = 16; // M
+int32_t tsPQSortMemThreshold = 16;      // M
 
 // sync raft
 int32_t tsElectInterval = 25 * 1000;
@@ -105,6 +107,8 @@ char *tsClientCrashReportUri = "/ccrashreport";
 char *tsSvrCrashReportUri = "/dcrashreport";
 
 // schemaless
+bool tsSmlDot2Underline = true;
+char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts";
 char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
 char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = "";  // user defined child table name can be specified in tag value.
                                                      // If set to empty system will generate table name using MD5 hash.
@@ -119,8 +123,8 @@ int32_t tsQueryPolicy = 1;
 int32_t tsQueryRspPolicy = 0;
 int64_t tsQueryMaxConcurrentTables = 200;  // unit is TSDB_TABLE_NUM_UNIT
 bool    tsEnableQueryHb = true;
-bool    tsEnableScience = false;  // on taos-cli show float and doulbe with scientific notation if true
-bool    tsTtlChangeOnWrite = false; // ttl delete time changes on last write if true
+bool    tsEnableScience = false;     // on taos-cli show float and doulbe with scientific notation if true
+bool    tsTtlChangeOnWrite = false;  // ttl delete time changes on last write if true
 int32_t tsQuerySmaOptimize = 0;
 int32_t tsQueryRsmaTolerance = 1000;  // the tolerance time (ms) to judge from which level to query rsma data.
 bool    tsQueryPlannerTrace = false;
@@ -366,13 +370,17 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1;
   //  if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT) != 0) return -1;
   //  if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX,
+                  CFG_SCOPE_CLIENT) != 0)
+    return -1;
   if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1;
@@ -385,7 +393,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;
 
   tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000);
-  if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1;
+  if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0)
+    return -1;
+
+  tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000);
+  if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 2;
   tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
@@ -445,7 +457,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000,
+                  CFG_SCOPE_SERVER) != 0)
+    return -1;
   if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, CFG_SCOPE_SERVER) != 0) return -1;
@@ -461,6 +475,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
   if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1;
 
+  tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000);
+  if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1;
+
   tsNumOfCommitThreads = tsNumOfCores / 2;
   tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
   if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@@ -473,7 +490,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1;
 
-  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
   tsNumOfVnodeFetchThreads = tsNumOfCores / 4;
   tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
@@ -493,7 +511,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsNumOfSnodeStreamThreads = tsNumOfCores / 4;
   tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4);
-  if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
   tsNumOfSnodeWriteThreads = tsNumOfCores / 4;
   tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4);
@@ -501,14 +520,18 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1;
   tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL);
-  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH) != 0)
+  if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX,
+                  CFG_SCOPE_BOTH) != 0)
     return -1;
 
   if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0)
+    return -1;
+  if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
-  if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
   if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER) != 0) return -1;
@@ -538,7 +561,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1;
 
-  if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER) != 0)
+  if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX,
+                  CFG_SCOPE_SERVER) != 0)
     return -1;
 
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER) != 0) return -1;
@@ -549,7 +573,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "checkpointInterval", tsCheckpointInterval, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
 
-  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0)
+    return -1;
 
   if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "keepTimeOffset", tsKeepTimeOffset, 0, 23, CFG_SCOPE_SERVER) != 0) return -1;
@@ -604,6 +629,13 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->stype = stype;
   }
 
+  pItem = cfgGetItem(tsCfg, "keepAliveIdle");
+  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
+    tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 720000);
+    pItem->i32 = tsKeepAliveIdle;
+    pItem->stype = stype;
+  }
+
   pItem = cfgGetItem(tsCfg, "numOfCommitThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
     tsNumOfCommitThreads = numOfCores / 2;
@@ -801,6 +833,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
 
   tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
   tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
+  tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN);
+  tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval;
   //  tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
 
   //  tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
@@ -833,6 +867,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
 
   tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
+
+  tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
   return 0;
 }
 
@@ -872,6 +908,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
   tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
 
+  tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
+
   tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
   tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
   tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
@@ -902,7 +940,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
   tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
 
-  tmqMaxTopicNum= cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
+  tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
 
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
@@ -1014,7 +1052,7 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
         taosSetCoreDump(enableCore);
       } else if (strcasecmp("enableQueryHb", name) == 0) {
         tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
-      }  else if (strcasecmp("ttlChangeOnWrite", name) == 0) {
+      } else if (strcasecmp("ttlChangeOnWrite", name) == 0) {
         tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
       }
       break;
@@ -1243,6 +1281,10 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
         //        tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
         //      } else if (strcasecmp("smlBatchSize", name) == 0) {
         //        tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32;
+      } else if(strcasecmp("smlTsDefaultName", name) == 0) {
+        tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN);
+      } else if(strcasecmp("smlDot2Underline", name) == 0) {
+        tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval;
       } else if (strcasecmp("shellActivityTimer", name) == 0) {
         tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
       } else if (strcasecmp("supportVnodes", name) == 0) {

source/common/src/tmsg.c

@@ -7227,6 +7227,9 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
       return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts;
     } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) {
       return pLeft->uid == pRight->uid;
+    } else {
+      uError("offset type:%d", pLeft->type);
+      ASSERT(0);
     }
   }
   return false;

source/dnode/mnode/impl/src/mndConsumer.c

@@ -94,7 +94,7 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId){
 
 bool mndRebTryStart() {
   int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1);
-  mInfo("tq timer, rebalance counter old val:%d", old);
+  mDebug("tq timer, rebalance counter old val:%d", old);
   return old == 0;
 }
 
@@ -116,7 +116,7 @@ void mndRebCntDec() {
     int32_t newVal = val - 1;
     int32_t oldVal = atomic_val_compare_exchange_32(&mqRebInExecCnt, val, newVal);
     if (oldVal == val) {
-      mInfo("rebalance trans end, rebalance counter:%d", newVal);
+      mDebug("rebalance trans end, rebalance counter:%d", newVal);
       break;
     }
   }
@@ -420,6 +420,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
 
     SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName);
     if(pSub == NULL){
+      ASSERT(0);
       continue;
     }
     taosWLockLatch(&pSub->lock);
@@ -515,7 +516,10 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
       char            *topic = taosArrayGetP(pConsumer->currentTopics, i);
       SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
       // txn guarantees pSub is created
-      if(pSub == NULL) continue;
+      if(pSub == NULL) {
+        ASSERT(0);
+        continue;
+      }
       taosRLockLatch(&pSub->lock);
 
       SMqSubTopicEp topicEp = {0};
@@ -524,6 +528,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
       // 2.1 fetch topic schema
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
       if(pTopic == NULL) {
+        ASSERT(0);
         taosRUnLockLatch(&pSub->lock);
         mndReleaseSubscribe(pMnode, pSub);
         continue;
@@ -898,7 +903,7 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d",
          pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status),
          pConsumer->epoch);
-  pConsumer->subscribeTime = taosGetTimestampMs();
+  pConsumer->subscribeTime = pConsumer->createTime;
   return 0;
 }
 

source/dnode/mnode/impl/src/mndDef.c

@@ -70,6 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
     if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
     for (int32_t j = 0; j < innerSz; j++) {
       SStreamTask *pTask = taosArrayGetP(pArray, j);
+      pTask->ver = SSTREAM_TASK_VER;
       if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1;
     }
   }
@@ -154,7 +155,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
   return 0;
 }
 
-static void* freeStreamTasks(SArray* pTaskLevel) {
+static void *freeStreamTasks(SArray *pTaskLevel) {
   int32_t numOfLevel = taosArrayGetSize(pTaskLevel);
   for (int32_t i = 0; i < numOfLevel; i++) {
     SArray *pLevel = taosArrayGetP(pTaskLevel, i);
@@ -192,14 +193,14 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
   SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
   if (pVgEpNew == NULL) return NULL;
   pVgEpNew->vgId = pVgEp->vgId;
-//  pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
+  //  pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
   pVgEpNew->epSet = pVgEp->epSet;
   return pVgEpNew;
 }
 
 void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
   if (pVgEp) {
-//    taosMemoryFreeClear(pVgEp->qmsg);
+    //    taosMemoryFreeClear(pVgEp->qmsg);
     taosMemoryFree(pVgEp);
   }
 }
@@ -207,14 +208,14 @@ void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
 int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
-//  tlen += taosEncodeString(buf, pVgEp->qmsg);
+  //  tlen += taosEncodeString(buf, pVgEp->qmsg);
   tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
   return tlen;
 }
 
 void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
   buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
-  if(sver == 1){
+  if (sver == 1) {
     uint64_t size = 0;
     buf = taosDecodeVariantU64(buf, &size);
     buf = POINTER_SHIFT(buf, size);
@@ -223,7 +224,7 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
   return (void *)buf;
 }
 
-SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char* cgroup) {
+SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char *cgroup) {
   SMqConsumerObj *pConsumer = taosMemoryCalloc(1, sizeof(SMqConsumerObj));
   if (pConsumer == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -260,12 +261,12 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char* cgroup) {
 }
 
 void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer, bool delete) {
-  if(pConsumer == NULL) return;
+  if (pConsumer == NULL) return;
   taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree);
   taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree);
   taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
   taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
-  if(delete){
+  if (delete) {
     taosMemoryFree(pConsumer);
   }
 }
@@ -392,7 +393,7 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s
     taosArrayPush(pConsumer->assignedTopics, &topic);
   }
 
-  if(sver > 1){
+  if (sver > 1) {
     buf = taosDecodeFixedI8(buf, &pConsumer->withTbName);
     buf = taosDecodeFixedI8(buf, &pConsumer->autoCommit);
     buf = taosDecodeFixedI32(buf, &pConsumer->autoCommitInterval);
@@ -401,18 +402,18 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s
   return (void *)buf;
 }
 
-//SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
-//  SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
-//  if (pConsumerEpNew == NULL) return NULL;
-//  pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
-//  pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL);
-//  return pConsumerEpNew;
-//}
+// SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
+//   SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
+//   if (pConsumerEpNew == NULL) return NULL;
+//   pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
+//   pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL);
+//   return pConsumerEpNew;
+// }
 //
-//void tDeleteSMqConsumerEp(void *data) {
-//  SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data;
-//  taosArrayDestroy(pConsumerEp->vgs);
-//}
+// void tDeleteSMqConsumerEp(void *data) {
+//   SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data;
+//   taosArrayDestroy(pConsumerEp->vgs);
+// }
 
 int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
   int32_t tlen = 0;
@@ -420,7 +421,7 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
   tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
   int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows);
   tlen += taosEncodeFixedI32(buf, szVgs);
-  for (int32_t j= 0; j < szVgs; ++j) {
+  for (int32_t j = 0; j < szVgs; ++j) {
     OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j);
     tlen += taosEncodeFixedI32(buf, offRows->vgId);
     tlen += taosEncodeFixedI64(buf, offRows->rows);
@@ -434,28 +435,28 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
       // do nothing
     }
   }
-//#if 0
-//  int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
-//  tlen += taosEncodeFixedI32(buf, sz);
-//  for (int32_t i = 0; i < sz; i++) {
-//    SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
-//    tlen += tEncodeSMqVgEp(buf, pVgEp);
-//  }
-//#endif
+  // #if 0
+  //   int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
+  //   tlen += taosEncodeFixedI32(buf, sz);
+  //   for (int32_t i = 0; i < sz; i++) {
+  //     SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
+  //     tlen += tEncodeSMqVgEp(buf, pVgEp);
+  //   }
+  // #endif
   return tlen;
 }
 
 void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) {
   buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
   buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver);
-  if (sver > 1){
+  if (sver > 1) {
     int32_t szVgs = 0;
     buf = taosDecodeFixedI32(buf, &szVgs);
-    if(szVgs > 0){
+    if (szVgs > 0) {
       pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
       if (NULL == pConsumerEp->offsetRows) return NULL;
-      for (int32_t j= 0; j < szVgs; ++j) {
-        OffsetRows* offRows = taosArrayReserve(pConsumerEp->offsetRows, 1);
+      for (int32_t j = 0; j < szVgs; ++j) {
+        OffsetRows *offRows = taosArrayReserve(pConsumerEp->offsetRows, 1);
         buf = taosDecodeFixedI32(buf, &offRows->vgId);
         buf = taosDecodeFixedI64(buf, &offRows->rows);
         buf = taosDecodeFixedI8(buf, &offRows->offset.type);
@@ -470,21 +471,21 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t s
       }
     }
   }
-//#if 0
-//  int32_t sz;
-//  buf = taosDecodeFixedI32(buf, &sz);
-//  pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
-//  for (int32_t i = 0; i < sz; i++) {
-//    SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
-//    buf = tDecodeSMqVgEp(buf, pVgEp);
-//    taosArrayPush(pConsumerEp->vgs, &pVgEp);
-//  }
-//#endif
+  // #if 0
+  //   int32_t sz;
+  //   buf = taosDecodeFixedI32(buf, &sz);
+  //   pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
+  //   for (int32_t i = 0; i < sz; i++) {
+  //     SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
+  //     buf = tDecodeSMqVgEp(buf, pVgEp);
+  //     taosArrayPush(pConsumerEp->vgs, &pVgEp);
+  //   }
+  // #endif
 
   return (void *)buf;
 }
 
-SMqSubscribeObj *tNewSubscribeObj(const char* key) {
+SMqSubscribeObj *tNewSubscribeObj(const char *key) {
   SMqSubscribeObj *pSubObj = taosMemoryCalloc(1, sizeof(SMqSubscribeObj));
   if (pSubObj == NULL) {
     return NULL;
@@ -577,7 +578,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
 
   int32_t szVgs = taosArrayGetSize(pSub->offsetRows);
   tlen += taosEncodeFixedI32(buf, szVgs);
-  for (int32_t j= 0; j < szVgs; ++j) {
+  for (int32_t j = 0; j < szVgs; ++j) {
     OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j);
     tlen += taosEncodeFixedI32(buf, offRows->vgId);
     tlen += taosEncodeFixedI64(buf, offRows->rows);
@@ -617,14 +618,14 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) {
   buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver);
   buf = taosDecodeStringTo(buf, pSub->dbName);
 
-  if (sver > 1){
+  if (sver > 1) {
     int32_t szVgs = 0;
     buf = taosDecodeFixedI32(buf, &szVgs);
-    if(szVgs > 0){
+    if (szVgs > 0) {
       pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
       if (NULL == pSub->offsetRows) return NULL;
-      for (int32_t j= 0; j < szVgs; ++j) {
-        OffsetRows* offRows = taosArrayReserve(pSub->offsetRows, 1);
+      for (int32_t j = 0; j < szVgs; ++j) {
+        OffsetRows *offRows = taosArrayReserve(pSub->offsetRows, 1);
         buf = taosDecodeFixedI32(buf, &offRows->vgId);
         buf = taosDecodeFixedI64(buf, &offRows->rows);
         buf = taosDecodeFixedI8(buf, &offRows->offset.type);
@@ -639,71 +640,71 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) {
       }
     }
     buf = taosDecodeString(buf, &pSub->qmsg);
-  }else{
+  } else {
     pSub->qmsg = taosStrdup("");
   }
   return (void *)buf;
 }
 
-//SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
-//  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
-//  if (pEntryNew == NULL) return NULL;
-//  pEntryNew->epoch = pEntry->epoch;
-//  pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
-//  return pEntryNew;
-//}
+// SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+//   SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
+//   if (pEntryNew == NULL) return NULL;
+//   pEntryNew->epoch = pEntry->epoch;
+//   pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
+//   return pEntryNew;
+// }
 //
-//void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
-//  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
-//}
-
-//int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
-//  int32_t tlen = 0;
-//  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
-//  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
-//  return tlen;
-//}
+// void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+//   taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
+// }
+
+// int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
+//   int32_t tlen = 0;
+//   tlen += taosEncodeFixedI32(buf, pEntry->epoch);
+//   tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
+//   return tlen;
+// }
 //
-//void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
-//  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
-//  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
-//  return (void *)buf;
-//}
-
-//SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
-//  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
-//  if (pLogNew == NULL) return pLogNew;
-//  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
-//  pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
-//  return pLogNew;
-//}
+// void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
+//   buf = taosDecodeFixedI32(buf, &pEntry->epoch);
+//   buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+//   return (void *)buf;
+// }
+
+// SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+//   SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
+//   if (pLogNew == NULL) return pLogNew;
+//   memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
+//   pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
+//   return pLogNew;
+// }
 //
-//void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
-//  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
-//}
-
-//int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
-//  int32_t tlen = 0;
-//  tlen += taosEncodeString(buf, pLog->key);
-//  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
-//  return tlen;
-//}
+// void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+//   taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
+// }
+
+// int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
+//   int32_t tlen = 0;
+//   tlen += taosEncodeString(buf, pLog->key);
+//   tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
+//   return tlen;
+// }
 //
-//void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
-//  buf = taosDecodeStringTo(buf, pLog->key);
-//  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
-//  return (void *)buf;
-//}
+// void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
+//   buf = taosDecodeStringTo(buf, pLog->key);
+//   buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+//   return (void *)buf;
+// }
 //
-//int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
-//  int32_t tlen = 0;
-//  tlen += taosEncodeString(buf, pOffset->key);
-//  tlen += taosEncodeFixedI64(buf, pOffset->offset);
-//  return tlen;
-//}
+// int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
+//   int32_t tlen = 0;
+//   tlen += taosEncodeString(buf, pOffset->key);
+//   tlen += taosEncodeFixedI64(buf, pOffset->offset);
+//   return tlen;
+// }
 //
-//void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) {
-//  buf = taosDecodeStringTo(buf, pOffset->key);
-//  buf = taosDecodeFixedI64(buf, &pOffset->offset);
-//  return buf;
-//}
+// void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) {
+//   buf = taosDecodeStringTo(buf, pOffset->key);
+//   buf = taosDecodeFixedI64(buf, &pOffset->offset);
+//   return buf;
+// }

source/dnode/mnode/impl/src/mndIndex.c

@@ -515,7 +515,6 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i
     if (pDb == NULL) return 0;
   }
   SSmaAndTagIter *pIter = pShow->pIter;
-  int             invalid = -1;
   while (numOfRows < rows) {
     pIter->pIdxIter = sdbFetch(pSdb, SDB_IDX, pIter->pIdxIter, (void **)&pIdx);
     if (pIter->pIdxIter == NULL) break;
@@ -552,7 +551,7 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
 
-    colDataSetVal(pColInfo, numOfRows, (const char *)&invalid, false);
+    colDataSetVal(pColInfo, numOfRows, NULL, true);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false);

source/dnode/mnode/impl/src/mndShow.c

@@ -20,6 +20,7 @@
 
 #define SHOW_STEP_SIZE 100
 #define SHOW_COLS_STEP_SIZE 4096
+#define SHOW_PRIVILEGES_STEP_SIZE 2048
 
 static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq);
 static void      mndFreeShowObj(SShowObj *pShow);
@@ -234,6 +235,8 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
 
   if(pShow->type == TSDB_MGMT_TABLE_COL){   // expend capacity for ins_columns
     rowsToRead = SHOW_COLS_STEP_SIZE;
+  } else if (pShow->type == TSDB_MGMT_TABLE_PRIVILEGES) {
+    rowsToRead = SHOW_PRIVILEGES_STEP_SIZE;
   }
   ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
   if (retrieveFp == NULL) {

source/dnode/mnode/impl/src/mndStream.c

@@ -28,7 +28,7 @@
 #include "parser.h"
 #include "tname.h"
 
-#define MND_STREAM_VER_NUMBER   2
+#define MND_STREAM_VER_NUMBER   3
 #define MND_STREAM_RESERVE_SIZE 64
 
 #define MND_STREAM_MAX_NUM 60
@@ -140,10 +140,12 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
   void       *buf = NULL;
 
   int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto STREAM_DECODE_OVER;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
+    goto STREAM_DECODE_OVER;
+  }
 
-  if (sver != 1 && sver != 2) {
-    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+  if (sver != MND_STREAM_VER_NUMBER) {
+    terrno = 0;
     goto STREAM_DECODE_OVER;
   }
 
@@ -198,12 +200,13 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) {
 
 static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) {
   mTrace("stream:%s, perform update action", pOldStream->name);
-  atomic_exchange_64(&pOldStream->updateTime, pNewStream->updateTime);
+
   atomic_exchange_32(&pOldStream->version, pNewStream->version);
 
   taosWLockLatch(&pOldStream->lock);
 
   pOldStream->status = pNewStream->status;
+  pOldStream->updateTime = pNewStream->updateTime;
 
   taosWUnLockLatch(&pOldStream->lock);
   return 0;
@@ -429,9 +432,11 @@ FAIL:
   return 0;
 }
 
-int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
+int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
   SEncoder encoder;
   tEncoderInit(&encoder, NULL, 0);
+
+  pTask->ver = SSTREAM_TASK_VER;
   tEncodeStreamTask(&encoder, pTask);
 
   int32_t size = encoder.pos;
@@ -520,7 +525,6 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
   SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
     return -1;
   }
 
@@ -537,7 +541,6 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream
   if (pCommitRaw == NULL) return -1;
   if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
     return -1;
   }
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
@@ -1264,7 +1267,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         // task id
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
 
-        char idstr[128] = {0};
+        char    idstr[128] = {0};
         int32_t len = tintToHex(pTask->id.taskId, &idstr[4]);
         idstr[2] = '0';
         idstr[3] = 'x';
@@ -1304,7 +1307,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         colDataSetVal(pColInfo, numOfRows, (const char *)&level, false);
 
         // status
-        char status[20 + VARSTR_HEADER_SIZE] = {0};
+        char   status[20 + VARSTR_HEADER_SIZE] = {0};
         int8_t taskStatus = atomic_load_8(&pTask->status.taskStatus);
         if (taskStatus == TASK_STATUS__NORMAL) {
           memcpy(varDataVal(status), "normal", 6);
@@ -1370,7 +1373,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
   return 0;
 }
 
-int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray* tasks) {
+int32_t mndPauseAllStreamTaskImpl(STrans *pTrans, SArray *tasks) {
   int32_t size = taosArrayGetSize(tasks);
   for (int32_t i = 0; i < size; i++) {
     SArray *pTasks = taosArrayGetP(tasks, i);
@@ -1409,7 +1412,6 @@ static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, in
   if (pCommitRaw == NULL) return -1;
   if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
     return -1;
   }
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
@@ -1431,7 +1433,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
   if (pStream == NULL) {
     if (pauseReq.igNotExists) {
       mInfo("stream:%s, not exist, if exist is set", pauseReq.name);
-      sdbRelease(pMnode->pSdb, pStream);
       return 0;
     } else {
       terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
@@ -1440,6 +1441,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
   }
 
   if (pStream->status == STREAM_STATUS__PAUSE) {
+    sdbRelease(pMnode->pSdb, pStream);
     return 0;
   }
 
@@ -1491,7 +1493,6 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
   return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
-
 static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t igUntreated) {
   SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq));
   if (pReq == NULL) {

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -692,6 +692,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
       taosArrayDestroy(rebOutput.modifyConsumers);
       taosArrayDestroy(rebOutput.rebVgs);
 
+      taosHashCancelIterate(pReq->rebSubHash, pIter);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       mInfo("mq re-balance failed, due to out of memory");
       taosHashCleanup(pReq->rebSubHash);
@@ -1168,7 +1169,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1);
     
-    mDebug("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
+    mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic),
            consumerId, varDataVal(cgroup), pVgEp->vgId);
 
     // offset

source/dnode/mnode/impl/src/mndUser.c

@@ -1174,26 +1174,30 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3
     if (strcmp("t", value) != 0) {
       SNode  *pAst = NULL;
       int32_t sqlLen = 0;
-      char    sql[TSDB_EXPLAIN_RESULT_ROW_SIZE] = {0};
+      size_t bufSz = strlen(value) + 1;
+      char* sql = taosMemoryMalloc(bufSz + 1);
+      char* obj = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
 
-      if (nodesStringToNode(value, &pAst) == 0) {
-        nodesNodeToSQL(pAst, sql, TSDB_EXPLAIN_RESULT_ROW_SIZE, &sqlLen);
+      if (sql != NULL && obj != NULL && nodesStringToNode(value, &pAst) == 0) {
+        nodesNodeToSQL(pAst, sql, bufSz, &sqlLen);
         nodesDestroyNode(pAst);
       } else {
         sqlLen = 5;
         sprintf(sql, "error");
       }
 
-      char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false);
+      taosMemoryFree(obj);
+      taosMemoryFree(sql);
     } else {
-      char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
+      char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false);
+      taosMemoryFree(condition);
     }
 
     (*numOfRows)++;
@@ -1209,16 +1213,34 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
   int32_t   cols = 0;
   char     *pWrite;
 
+  bool     fetchNextUser = pShow->restore ? false : true;
+  pShow->restore = false;
+
   while (numOfRows < rows) {
-    pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
-    if (pShow->pIter == NULL) break;
+    if (fetchNextUser) {
+      pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser);
+      if (pShow->pIter == NULL) break;
+    } else {
+      fetchNextUser = true;
+      void *pKey = taosHashGetKey(pShow->pIter, NULL);
+      pUser = sdbAcquire(pSdb, SDB_USER, pKey);
+      if (!pUser) {
+        continue;
+      }
+    }
 
     int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
     int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
     int32_t numOfTopics = taosHashGetSize(pUser->topics);
     int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
     int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
-    if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) break;
+    if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) {
+      mInfo("will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables %d", 
+        numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs);
+      pShow->restore = true;
+      sdbRelease(pSdb, pUser);
+      break;
+    }
 
     if (pUser->superUser) {
       cols = 0;
@@ -1242,10 +1264,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
 
-      char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
+      char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
+      taosMemoryFree(condition);
 
       numOfRows++;
     }
@@ -1276,10 +1299,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
 
-      char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
+      char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
+      taosMemoryFree(condition);
 
       numOfRows++;
       db = taosHashIterate(pUser->readDbs, db);
@@ -1311,10 +1335,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
 
-      char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
+      char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
+      taosMemoryFree(condition);
 
       numOfRows++;
       db = taosHashIterate(pUser->writeDbs, db);
@@ -1348,10 +1373,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false);
 
-      char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0};
+      char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
       STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
+      taosMemoryFree(condition);
 
       numOfRows++;
       topic = taosHashIterate(pUser->topics, topic);

source/dnode/snode/src/snode.c

@@ -346,9 +346,9 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
     rsp.status = streamTaskCheckStatus(pTask);
     streamMetaReleaseTask(pSnode->pMeta, pTask);
 
-    qDebug("s-task:%s recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), status:%s, rsp status %d",
-            pTask->id.idStr, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId,
-            streamGetTaskStatusStr(pTask->status.taskStatus), rsp.status);
+    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
+            pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
   } else {
     rsp.status = 0;
     qDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64

source/dnode/snode/src/snodeInitApi.c

@@ -78,6 +78,8 @@ void initStateStoreAPI(SStateStore* pStore) {
   pStore->updateInfoIsUpdated = updateInfoIsUpdated;
   pStore->updateInfoIsTableInserted = updateInfoIsTableInserted;
   pStore->updateInfoDestroy = updateInfoDestroy;
+  pStore->windowSBfDelete = windowSBfDelete;
+  pStore->windowSBfAdd = windowSBfAdd;
 
   pStore->updateInfoInitP = updateInfoInitP;
   pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;

source/dnode/vnode/src/inc/tsdb.h

@@ -841,48 +841,6 @@ typedef enum {
   READ_MODE_ALL,
 } EReadMode;
 
-typedef struct STsdbReaderInfo {
-  uint64_t      suid;
-  STSchema     *pSchema;
-  EReadMode     readMode;
-  uint64_t      rowsNum;
-  STimeWindow   window;
-  SVersionRange verRange;
-  int16_t       order;
-} STsdbReaderInfo;
-
-typedef struct {
-  SArray *pTombData;
-} STableLoadInfo;
-
-struct SDataFileReader;
-
-typedef struct SCacheRowsReader {
-  STsdb                  *pTsdb;
-  STsdbReaderInfo         info;
-  TdThreadMutex           readerMutex;
-  SVnode                 *pVnode;
-  STSchema               *pSchema;
-  STSchema               *pCurrSchema;
-  uint64_t                uid;
-  char                  **transferBuf;  // todo remove it soon
-  int32_t                 numOfCols;
-  SArray                 *pCidList;
-  int32_t                *pSlotIds;
-  int32_t                 type;
-  int32_t                 tableIndex;  // currently returned result tables
-  STableKeyInfo          *pTableList;  // table id list
-  int32_t                 numOfTables;
-  uint64_t               *uidList;
-  SSHashObj              *pTableMap;
-  SArray                 *pLDataIterArray;
-  struct SDataFileReader *pFileReader;
-  STFileSet              *pCurFileSet;
-  STsdbReadSnap          *pReadSnap;
-  char                   *idstr;
-  int64_t                 lastTs;
-} SCacheRowsReader;
-
 typedef struct {
   TSKEY   ts;
   int8_t  dirty;
@@ -892,14 +850,10 @@ typedef struct {
 int32_t tsdbOpenCache(STsdb *pTsdb);
 void    tsdbCloseCache(STsdb *pTsdb);
 int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *row);
-int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype);
-int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype);
 int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 
 int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
 int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup);
-int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
-int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h);
 int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h);
 
 int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHandle **handle);
@@ -909,8 +863,6 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey);
 
-// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
-
 // ========== inline functions ==========
 static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
   TSDBKEY *pKey1 = (TSDBKEY *)p1;

source/dnode/vnode/src/tq/tq.c

@@ -146,6 +146,20 @@ void tqClose(STQ* pTq) {
     return;
   }
 
+  void* pIter = taosHashIterate(pTq->pPushMgr, NULL);
+  while (pIter) {
+    STqHandle* pHandle = *(STqHandle**)pIter;
+    int32_t    vgId = TD_VID(pTq->pVnode);
+
+    if(pHandle->msg != NULL) {
+      tqPushEmptyDataRsp(pHandle, vgId);
+      rpcFreeCont(pHandle->msg->pCont);
+      taosMemoryFree(pHandle->msg);
+      pHandle->msg = NULL;
+    }
+    pIter = taosHashIterate(pTq->pPushMgr, pIter);
+  }
+
   tqOffsetClose(pTq->pOffsetStore);
   taosHashCleanup(pTq->pHandle);
   taosHashCleanup(pTq->pPushMgr);
@@ -278,6 +292,10 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
   tqInitDataRsp(&dataRsp, &req);
   dataRsp.blockNum = 0;
   dataRsp.rspOffset = dataRsp.reqOffset;
+  char buf[TSDB_OFFSET_LEN] = {0};
+  tFormatOffset(buf, TSDB_OFFSET_LEN, &dataRsp.reqOffset);
+  tqInfo("tqPushEmptyDataRsp to consumer:0x%"PRIx64 " vgId:%d, offset:%s, reqId:0x%" PRIx64, req.consumerId, vgId, buf, req.reqId);
+
   tqSendDataRsp(pHandle, pHandle->msg, &req, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
   tDeleteMqDataRsp(&dataRsp);
   return 0;
@@ -336,10 +354,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t
   STqOffset* pOffset = &vgOffset.offset;
 
   if (pOffset->val.type == TMQ_OFFSET__SNAPSHOT_DATA || pOffset->val.type == TMQ_OFFSET__SNAPSHOT_META) {
-    tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
             pOffset->subKey, vgId, pOffset->val.uid, pOffset->val.ts);
   } else if (pOffset->val.type == TMQ_OFFSET__LOG) {
-    tqInfo("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId,
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId,
             pOffset->val.version);
   } else {
     tqError("invalid commit offset type:%d", pOffset->val.type);
@@ -367,12 +385,13 @@ int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg) {
   SRpcMsg     rsp = {.info = pMsg->info};
   int         code = 0;
 
-  tqDebug("tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s", req.consumerId, vgId, req.subKey);
   if (tDeserializeSMqSeekReq(pMsg->pCont, pMsg->contLen, &req) < 0) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
 
+  tqDebug("tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s", req.consumerId, vgId, req.subKey);
+
   STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
   if (pHandle == NULL) {
     tqWarn("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", req.consumerId, vgId, req.subKey);
@@ -515,10 +534,11 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) {
 
     while (pIter) {
       STqHandle* pHandle = *(STqHandle**)pIter;
-      tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId);
+      tqInfo("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId);
 
       if (ASSERT(pHandle->msg != NULL)) {
         tqError("pHandle->msg should not be null");
+        taosHashCancelIterate(pTq->pPushMgr, pIter);
         break;
       }else{
         SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info};
@@ -556,10 +576,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
     // 1. find handle
     pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
     if (pHandle == NULL) {
-      tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
-      terrno = TSDB_CODE_INVALID_MSG;
-      taosWUnLockLatch(&pTq->lock);
-      return -1;
+      do{
+        if (tqMetaGetHandle(pTq, req.subKey) == 0){
+          pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
+          if(pHandle != NULL){
+            break;
+          }
+        }
+        tqError("tmq poll: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", consumerId, vgId, req.subKey);
+        terrno = TSDB_CODE_INVALID_MSG;
+        taosWUnLockLatch(&pTq->lock);
+        return -1;
+      }while(0);
     }
 
     // 2. check re-balance status
@@ -849,30 +877,28 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
     taosWLockLatch(&pTq->lock);
 
     if (pHandle->consumerId == req.newConsumerId) {  // do nothing
-      tqInfo("vgId:%d consumer:0x%" PRIx64 " remains, no switch occurs, should not reach here", req.vgId,
-             req.newConsumerId);
+      tqInfo("vgId:%d no switch consumer:0x%" PRIx64 " remains, because redo wal log", req.vgId, req.newConsumerId);
     } else {
-      tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId,
-             req.newConsumerId);
+      tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId);
       atomic_store_64(&pHandle->consumerId, req.newConsumerId);
+      //    atomic_add_fetch_32(&pHandle->epoch, 1);
+
+      // kill executing task
+      //    if(tqIsHandleExec(pHandle)) {
+      //      qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
+      //      if (pTaskInfo != NULL) {
+      //        qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
+      //      }
+
+      //      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+      //        qStreamCloseTsdbReader(pTaskInfo);
+      //      }
+      //    }
+      // remove if it has been register in the push manager, and return one empty block to consumer
+      tqUnregisterPushHandle(pTq, pHandle);
+      ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
     }
-    //    atomic_add_fetch_32(&pHandle->epoch, 1);
-
-    // kill executing task
-    //    if(tqIsHandleExec(pHandle)) {
-    //      qTaskInfo_t pTaskInfo = pHandle->execHandle.task;
-    //      if (pTaskInfo != NULL) {
-    //        qKillTask(pTaskInfo, TSDB_CODE_SUCCESS);
-    //      }
-
-    //      if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    //        qStreamCloseTsdbReader(pTaskInfo);
-    //      }
-    //    }
-    // remove if it has been register in the push manager, and return one empty block to consumer
-    tqUnregisterPushHandle(pTq, pHandle);
     taosWUnLockLatch(&pTq->lock);
-    ret = tqMetaSaveHandle(pTq, req.subKey, pHandle);
   }
 
 end:
@@ -1041,9 +1067,9 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
     rsp.status = streamTaskCheckStatus(pTask);
     streamMetaReleaseTask(pTq->pStreamMeta, pTask);
 
-    tqDebug("s-task:%s recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), status:%s, rsp status %d",
-            pTask->id.idStr, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId,
-            streamGetTaskStatusStr(pTask->status.taskStatus), rsp.status);
+    const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus);
+    tqDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
+            pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
   } else {
     rsp.status = 0;
     tqDebug("tq recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d",

source/dnode/vnode/src/tq/tqMeta.c

@@ -338,7 +338,7 @@ static int buildHandle(STQ* pTq, STqHandle* handle){
       taosArrayDestroy(tbUidList);
       return -1;
     }
-    tqDebug("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
+    tqInfo("vgId:%d, tq try to get ctb for stb subscribe, suid:%" PRId64, pVnode->config.vgId, handle->execHandle.execTb.suid);
     handle->execHandle.pTqReader = tqReaderOpen(pVnode);
     tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL);
     taosArrayDestroy(tbUidList);
@@ -356,7 +356,7 @@ static int restoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle){
   if(buildHandle(pTq, handle) < 0){
     return -1;
   }
-  tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
+  tqInfo("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
   return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
 }
 
@@ -384,7 +384,7 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
   if(buildHandle(pTq, handle) < 0){
     return -1;
   }
-  tqDebug("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
+  tqInfo("tq restore %s consumer %" PRId64 " vgId:%d", handle->subKey, handle->consumerId, vgId);
   return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
 }
 

source/dnode/vnode/src/tq/tqPush.c

@@ -78,12 +78,12 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) {
   memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen);
   pHandle->msg->contLen = pMsg->contLen;
   int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES);
-  tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret,
+  tqInfo("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret,
           pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen);
   return 0;
 }
 
-int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
+int tqUnregisterPushHandle(STQ* pTq, void *handle) {
   STqHandle *pHandle = (STqHandle*)handle;
   int32_t    vgId = TD_VID(pTq->pVnode);
 
@@ -91,7 +91,7 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
     return 0;
   }
   int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey));
-  tqDebug("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
+  tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
 
   if(pHandle->msg != NULL) {
 //    tqPushDataRsp(pHandle, vgId);

source/dnode/vnode/src/tq/tqRestore.c

@@ -251,19 +251,18 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
     int32_t status = pTask->status.taskStatus;
 
     // non-source or fill-history tasks don't need to response the WAL scan action.
-    if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
+    if ((pTask->info.taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.downstreamReady == 0)) {
       streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    if (status != TASK_STATUS__NORMAL/* && status != TASK_STATUS__SCAN_HISTORY_WAL*/) {
+    if (status != TASK_STATUS__NORMAL) {
       tqDebug("s-task:%s not ready for new submit block from wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status));
       streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
     if ((pTask->info.fillHistory == 1) && pTask->status.transferState) {
-//      ASSERT(status == TASK_STATUS__SCAN_HISTORY_WAL);
       ASSERT(status == TASK_STATUS__NORMAL);
       // the maximum version of data in the WAL has reached already, the step2 is done
       tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,

source/dnode/vnode/src/tq/tqUtil.c

@@ -317,6 +317,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
     // the offset value can not be monotonious increase??
     offset = reqOffset;
   } else {
+    uError("req offset type is 0");
     return TSDB_CODE_TMQ_INVALID_MSG;
   }
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1020,10 +1020,10 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
     code = tsdbCacheLoadFromRocks(pTsdb, uid, pLastArray, remainCols, pr, ltype);
 
     taosThreadMutexUnlock(&pTsdb->lruMutex);
-  }
 
-  if (remainCols) {
-    taosArrayDestroy(remainCols);
+    if (remainCols) {
+      taosArrayDestroy(remainCols);
+    }
   }
 
   return code;
@@ -1592,10 +1592,51 @@ _err:
   return code;
 }
 
+static void freeTableInfoFunc(void *param) {
+  void **p = (void **)param;
+  taosMemoryFreeClear(*p);
+}
+
+static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) {
+  if (!pReader->pTableMap) {
+    pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
+
+    tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc);
+  }
+
+  STableLoadInfo  *pInfo = NULL;
+  STableLoadInfo **ppInfo = tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid));
+  if (!ppInfo) {
+    pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo));
+    tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
+
+    return pInfo;
+  }
+
+  return *ppInfo;
+}
+
+static uint64_t *getUidList(SCacheRowsReader *pReader) {
+  if (!pReader->uidList) {
+    int32_t numOfTables = pReader->numOfTables;
+
+    pReader->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
+
+    for (int32_t i = 0; i < numOfTables; ++i) {
+      uint64_t uid = pReader->pTableList[i].uid;
+      pReader->uidList[i] = uid;
+    }
+
+    taosSort(pReader->uidList, numOfTables, sizeof(uint64_t), uidComparFunc);
+  }
+
+  return pReader->uidList;
+}
+
 static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader,
                                bool isFile) {
   int32_t   code = 0;
-  uint64_t *uidList = pReader->uidList;
+  uint64_t *uidList = getUidList(pReader);
   int32_t   numOfTables = pReader->numOfTables;
   int64_t   suid = pReader->info.suid;
 
@@ -1618,7 +1659,7 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
     }
 
     uint64_t        uid = uidList[j];
-    STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid));
+    STableLoadInfo *pInfo = getTableLoadInfo(pReader, uid);
     if (pInfo->pTombData == NULL) {
       pInfo->pTombData = taosArrayInit(4, sizeof(SDelData));
     }
@@ -1660,17 +1701,16 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
       }
 
       if (newTable) {
-        pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid));
+        pInfo = getTableLoadInfo(pReader, uid);
         if (pInfo->pTombData == NULL) {
           pInfo->pTombData = taosArrayInit(4, sizeof(SDelData));
         }
       }
 
       if (record.version <= pReader->info.verRange.maxVer) {
-        /*
         tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
                   TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
-        */
+
         SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
         taosArrayPush(pInfo->pTombData, &delData);
       }
@@ -1796,12 +1836,10 @@ struct CacheNextRowIter;
 
 typedef struct SFSNextRowIter {
   SFSNEXTROWSTATES         state;         // [input]
-  STsdb                   *pTsdb;         // [input]
   SBlockIdx               *pBlockIdxExp;  // [input]
   STSchema                *pTSchema;      // [input]
   tb_uid_t                 suid;
   tb_uid_t                 uid;
-  int32_t                  nFileSet;
   int32_t                  iFileSet;
   STFileSet               *pFileSet;
   TFileSetArray           *aDFileSet;
@@ -1832,10 +1870,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
                                 int nCols) {
   SFSNextRowIter *state = (SFSNextRowIter *)iter;
   int32_t         code = 0;
+  STsdb          *pTsdb = state->pr->pTsdb;
 
   if (SFSNEXTROW_FS == state->state) {
-    state->nFileSet = TARRAY2_SIZE(state->aDFileSet);
-    state->iFileSet = state->nFileSet;
+    state->iFileSet = TARRAY2_SIZE(state->aDFileSet);
 
     state->state = SFSNEXTROW_FILESET;
   }
@@ -1854,7 +1892,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
     STFileObj **pFileObj = state->pFileSet->farr;
     if (pFileObj[0] != NULL || pFileObj[3] != NULL) {
       if (state->pFileSet != state->pr->pCurFileSet) {
-        SDataFileReaderConfig conf = {.tsdb = state->pTsdb, .szPage = state->pTsdb->pVnode->config.tsdbPageSize};
+        SDataFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize};
         const char           *filesName[4] = {0};
         if (pFileObj[0] != NULL) {
           conf.files[0].file = *pFileObj[0]->f;
@@ -1884,6 +1922,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
         state->pr->pCurFileSet = state->pFileSet;
 
         loadDataTomb(state->pr, state->pr->pFileReader);
+
+        int32_t code = tsdbDataFileReadBrinBlk(state->pr->pFileReader, &state->pr->pBlkArray);
+        if (code != TSDB_CODE_SUCCESS) {
+          goto _err;
+        }
       }
 
       if (!state->pIndexList) {
@@ -1891,12 +1934,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
       } else {
         taosArrayClear(state->pIndexList);
       }
-      const TBrinBlkArray *pBlkArray = NULL;
 
-      int32_t code = tsdbDataFileReadBrinBlk(state->pr->pFileReader, &pBlkArray);
-      if (code != TSDB_CODE_SUCCESS) {
-        goto _err;
-      }
+      const TBrinBlkArray *pBlkArray = state->pr->pBlkArray;
 
       for (int i = TARRAY2_SIZE(pBlkArray) - 1; i >= 0; --i) {
         SBrinBlk *pBrinBlk = &pBlkArray->data[i];
@@ -1912,21 +1951,20 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
 
       int indexSize = TARRAY_SIZE(state->pIndexList);
       if (indexSize <= 0) {
-        clearLastFileSet(state);
-        state->state = SFSNEXTROW_FILESET;
-        goto _next_fileset;
+        goto _check_stt_data;
       }
 
       state->state = SFSNEXTROW_INDEXLIST;
       state->iBrinIndex = indexSize;
     }
 
+  _check_stt_data:
     if (state->pFileSet != state->pr->pCurFileSet) {
       state->pr->pCurFileSet = state->pFileSet;
     }
 
-    code = lastIterOpen(&state->lastIter, state->pFileSet, state->pTsdb, state->pTSchema, state->suid, state->uid,
-                        state->pr, state->lastTs, aCols, nCols);
+    code = lastIterOpen(&state->lastIter, state->pFileSet, pTsdb, state->pTSchema, state->suid, state->uid, state->pr,
+                        state->lastTs, aCols, nCols);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -2346,7 +2384,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
 
   pIter->fsState.pRowIter = pIter;
   pIter->fsState.state = SFSNEXTROW_FS;
-  pIter->fsState.pTsdb = pTsdb;
   pIter->fsState.aDFileSet = pReadSnap->pfSetArray;
   pIter->fsState.pBlockIdxExp = &pIter->idx;
   pIter->fsState.pTSchema = pTSchema;
@@ -2463,14 +2500,17 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI
         pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
 
         uint64_t        uid = pIter->idx.uid;
-        STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pIter->pr->pTableMap, &uid, sizeof(uid));
-        SArray         *pTombData = pInfo->pTombData;
-        if (pTombData) {
-          taosArrayAddAll(pTombData, pIter->pMemDelData);
-
-          code = tsdbBuildDeleteSkyline(pTombData, 0, (int32_t)(TARRAY_SIZE(pTombData) - 1), pIter->pSkyline);
+        STableLoadInfo *pInfo = getTableLoadInfo(pIter->pr, uid);
+        if (pInfo->pTombData == NULL) {
+          pInfo->pTombData = taosArrayInit(4, sizeof(SDelData));
         }
 
+        taosArrayAddAll(pInfo->pTombData, pIter->pMemDelData);
+
+        size_t delSize = TARRAY_SIZE(pInfo->pTombData);
+        if (delSize > 0) {
+          code = tsdbBuildDeleteSkyline(pInfo->pTombData, 0, (int32_t)(delSize - 1), pIter->pSkyline);
+        }
         pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1;
       }
 

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -18,6 +18,7 @@
 #include "tcommon.h"
 #include "tsdb.h"
 #include "tsdbDataFileRW.h"
+#include "tsdbReadUtil.h"
 
 #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
 
@@ -133,21 +134,6 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t uidComparFunc(const void* p1, const void* p2) {
-  uint64_t pu1 = *(uint64_t*)p1;
-  uint64_t pu2 = *(uint64_t*)p2;
-  if (pu1 == pu2) {
-    return 0;
-  } else {
-    return (pu1 < pu2) ? -1 : 1;
-  }
-}
-
-static void freeTableInfoFunc(void* param) {
-  void** p = (void**)param;
-  taosMemoryFreeClear(*p);
-}
-
 int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
                                 SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr) {
   *pReader = NULL;
@@ -173,27 +159,6 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
   p->pTableList = pTableIdList;
   p->numOfTables = numOfTables;
 
-  p->pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
-  if (p->pTableMap == NULL) {
-    tsdbCacherowsReaderClose(p);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  p->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
-  if (p->uidList == NULL) {
-    tsdbCacherowsReaderClose(p);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  for (int32_t i = 0; i < numOfTables; ++i) {
-    uint64_t uid = p->pTableList[i].uid;
-    p->uidList[i] = uid;
-    STableLoadInfo* pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo));
-    tSimpleHashPut(p->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
-  }
-
-  tSimpleHashSetFreeFp(p->pTableMap, freeTableInfoFunc);
-
-  taosSort(p->uidList, numOfTables, sizeof(uint64_t), uidComparFunc);
-
   int32_t code = setTableSchema(p, suid, idstr);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbCacherowsReaderClose(p);
@@ -216,14 +181,6 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
     }
   }
 
-  SVnodeCfg* pCfg = &((SVnode*)pVnode)->config;
-  int32_t    numOfStt = pCfg->sttTrigger;
-  p->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
-  if (p->pLDataIterArray == NULL) {
-    tsdbCacherowsReaderClose(p);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
   p->idstr = taosStrdup(idstr);
   taosThreadMutexInit(&p->readerMutex, NULL);
 
@@ -250,9 +207,11 @@ void* tsdbCacherowsReaderClose(void* pReader) {
 
   taosMemoryFree(p->pCurrSchema);
 
-  int64_t loadBlocks = 0;
-  double  elapse = 0;
-  destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse);
+  if (p->pLDataIterArray) {
+    int64_t loadBlocks = 0;
+    double  elapse = 0;
+    destroySttBlockReader(p->pLDataIterArray, &loadBlocks, &elapse);
+  }
 
   if (p->pFileReader) {
     tsdbDataFileReaderClose(&p->pFileReader);
@@ -318,7 +277,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   int32_t           code = TSDB_CODE_SUCCESS;
   SArray*           pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol));
   bool              hasRes = false;
-  SArray*           pLastCols = NULL;
 
   void** pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES);
   if (pRes == NULL) {
@@ -327,57 +285,47 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   }
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
-    pRes[j] =
-        taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[/*-1 == slotIds[j] ? 0 : */ slotIds[j]].bytes +
-                                VARSTR_HEADER_SIZE);
+    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE);
     SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
     p->ts = INT64_MIN;
   }
 
-  pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol));
-  if (pLastCols == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _end;
-  }
-
-  for (int32_t i = 0; i < pr->numOfCols; ++i) {
-    int32_t          slotId = slotIds[i];
-    struct STColumn* pCol = &pr->pSchema->columns[slotId];
-    SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
-
-    if (IS_VAR_DATA_TYPE(pCol->type)) {
-      p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
-    }
-    taosArrayPush(pLastCols, &p);
-  }
-
   taosThreadMutexLock(&pr->readerMutex);
   code = tsdbTakeReadSnap2((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap);
   if (code != TSDB_CODE_SUCCESS) {
     goto _end;
   }
 
-  int8_t ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3;
+  int8_t         ltype = (pr->type & CACHESCAN_RETRIEVE_LAST) >> 3;
+  STableKeyInfo* pTableList = pr->pTableList;
 
   // retrieve the only one last row of all tables in the uid list.
   if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_SINGLE)) {
+    SArray* pLastCols = taosArrayInit(pr->numOfCols, sizeof(SLastCol));
+    if (pLastCols == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _end;
+    }
+
+    for (int32_t i = 0; i < pr->numOfCols; ++i) {
+      int32_t          slotId = slotIds[i];
+      struct STColumn* pCol = &pr->pSchema->columns[slotId];
+      SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
+
+      if (IS_VAR_DATA_TYPE(pCol->type)) {
+        p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char));
+      }
+      taosArrayPush(pLastCols, &p);
+    }
+
     int64_t st = taosGetTimestampUs();
     int64_t totalLastTs = INT64_MAX;
-
     for (int32_t i = 0; i < pr->numOfTables; ++i) {
-      STableKeyInfo* pKeyInfo = &pr->pTableList[i];
+      tb_uid_t uid = pTableList[i].uid;
 
-      tsdbCacheGetBatch(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
-      // tsdbCacheGet(pr->pTsdb, pKeyInfo->uid, pRow, pr, ltype);
-      if (TARRAY_SIZE(pRow) <= 0) {
-        taosArrayClearEx(pRow, freeItem);
-        // taosArrayClear(pRow);
-        continue;
-      }
-      SLastCol* pColVal = taosArrayGet(pRow, 0);
-      if (COL_VAL_IS_NONE(&pColVal->colVal)) {
+      tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
+      if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
         taosArrayClearEx(pRow, freeItem);
-        // taosArrayClear(pRow);
         continue;
       }
 
@@ -400,9 +348,9 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
             p->ts = pColVal->ts;
             if (k == 0) {
               if (TARRAY_SIZE(pTableUidList) == 0) {
-                taosArrayPush(pTableUidList, &pKeyInfo->uid);
+                taosArrayPush(pTableUidList, &uid);
               } else {
-                taosArraySet(pTableUidList, 0, &pKeyInfo->uid);
+                taosArraySet(pTableUidList, 0, &uid);
               }
             }
 
@@ -437,32 +385,25 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
       }
 
       taosArrayClearEx(pRow, freeItem);
-      // taosArrayClear(pRow);
     }
 
     if (hasRes) {
       saveOneRow(pLastCols, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
     }
+
+    taosArrayDestroyEx(pLastCols, freeItem);
   } else if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_TYPE_ALL)) {
     for (int32_t i = pr->tableIndex; i < pr->numOfTables; ++i) {
-      tb_uid_t uid = pr->pTableList[i].uid;
+      tb_uid_t uid = pTableList[i].uid;
 
       tsdbCacheGetBatch(pr->pTsdb, uid, pRow, pr, ltype);
-      if (TARRAY_SIZE(pRow) <= 0) {
+      if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) {
         taosArrayClearEx(pRow, freeItem);
-        // taosArrayClear(pRow);
-        continue;
-      }
-      SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, 0);
-      if (COL_VAL_IS_NONE(&pColVal->colVal)) {
-        taosArrayClearEx(pRow, freeItem);
-        // taosArrayClear(pRow);
         continue;
       }
 
       saveOneRow(pRow, pResBlock, pr, slotIds, dstSlotIds, pRes, pr->idstr);
       taosArrayClearEx(pRow, freeItem);
-      // taosArrayClear(pRow);
 
       taosArrayPush(pTableUidList, &uid);
 
@@ -478,11 +419,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 _end:
   tsdbUntakeReadSnap2((STsdbReader*)pr, pr->pReadSnap, true);
 
-  int64_t loadBlocks = 0;
-  double  elapse = 0;
-  pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, &loadBlocks, &elapse);
-  pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
-
   taosThreadMutexUnlock(&pr->readerMutex);
 
   if (pRes != NULL) {
@@ -492,9 +428,7 @@ _end:
   }
 
   taosMemoryFree(pRes);
-  // taosArrayDestroyEx(pRow, freeItem);
   taosArrayDestroy(pRow);
-  taosArrayDestroyEx(pLastCols, freeItem);
 
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbDataFileRW.c

@@ -1229,11 +1229,16 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom
 
       int32_t c = tTombRecordCompare(record, record1);
       if (c < 0) {
-        break;
+        goto _write;
       } else if (c > 0) {
         code = tTombBlockPut(writer->tombBlock, record1);
         TSDB_CHECK_CODE(code, lino, _exit);
 
+        tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
+                  ", version:%" PRId64,
+                  TD_VID(writer->config->tsdb->pVnode), writer->fd[TSDB_FTYPE_TOMB]->path, writer->config->cid,
+                  record1->suid, record1->uid, record1->version);
+
         if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
           code = tsdbDataFileDoWriteTombBlock(writer);
           TSDB_CHECK_CODE(code, lino, _exit);
@@ -1266,6 +1271,11 @@ _write:
   code = tTombBlockPut(writer->tombBlock, record);
   TSDB_CHECK_CODE(code, lino, _exit);
 
+  tsdbTrace("vgId:%d write tomb record to tomb file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
+            ", version:%" PRId64,
+            TD_VID(writer->config->tsdb->pVnode), writer->fd[TSDB_FTYPE_TOMB]->path, writer->config->cid, record->suid,
+            record->uid, record->version);
+
   if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
     code = tsdbDataFileDoWriteTombBlock(writer);
     TSDB_CHECK_CODE(code, lino, _exit);

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -537,6 +537,9 @@ static void tsdbDoWaitBgTask(STFileSystem *fs, STFSBgTask *task) {
 
   if (task->numWait == 0) {
     taosThreadCondDestroy(task->done);
+    if (task->free) {
+      task->free(task->arg);
+    }
     taosMemoryFree(task);
   }
 }
@@ -546,6 +549,9 @@ static void tsdbDoDoneBgTask(STFileSystem *fs, STFSBgTask *task) {
     taosThreadCondBroadcast(task->done);
   } else {
     taosThreadCondDestroy(task->done);
+    if (task->free) {
+      task->free(task->arg);
+    }
     taosMemoryFree(task);
   }
 }
@@ -627,7 +633,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
       SSttLvl *lvl = TARRAY2_FIRST(fset->lvlArr);
       if (lvl->level != 0 || TARRAY2_SIZE(lvl->fobjArr) < fs->tsdb->pVnode->config.sttTrigger) continue;
 
-      code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, fs->tsdb, NULL);
+      code = tsdbFSScheduleBgTask(fs, TSDB_BG_TASK_MERGER, tsdbMerge, NULL, fs->tsdb, NULL);
       TSDB_CHECK_CODE(code, lino, _exit);
 
       break;
@@ -774,19 +780,20 @@ static int32_t tsdbFSRunBgTask(void *arg) {
   return 0;
 }
 
-static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg,
-                                        int64_t *taskid) {
+static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int32_t (*run)(void *),
+                                        void (*destroy)(void *), void *arg, int64_t *taskid) {
   if (fs->stop) {
+    if (destroy) {
+      destroy(arg);
+    }
     return 0;  // TODO: use a better error code
   }
 
-  // check if same task is on
-  // if (fs->bgTaskRunning && fs->bgTaskRunning->type == type) {
-  //   return 0;
-  // }
-
   for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) {
     if (task->type == type) {
+      if (destroy) {
+        destroy(arg);
+      }
       return 0;
     }
   }
@@ -798,6 +805,7 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32
 
   task->type = type;
   task->run = run;
+  task->free = destroy;
   task->arg = arg;
   task->scheduleTime = taosGetTimestampMs();
   task->taskid = ++fs->taskid;
@@ -819,9 +827,10 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32
   return 0;
 }
 
-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg, int64_t *taskid) {
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
+                             int64_t *taskid) {
   taosThreadMutexLock(fs->mutex);
-  int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, arg, taskid);
+  int32_t code = tsdbFSScheduleBgTaskImpl(fs, type, run, free, arg, taskid);
   taosThreadMutexUnlock(fs->mutex);
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -59,7 +59,8 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
 int32_t tsdbFSEditCommit(STFileSystem *fs);
 int32_t tsdbFSEditAbort(STFileSystem *fs);
 // background task
-int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void *arg, int64_t *taskid);
+int32_t tsdbFSScheduleBgTask(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *), void *arg,
+                             int64_t *taskid);
 int32_t tsdbFSWaitBgTask(STFileSystem *fs, int64_t taskid);
 int32_t tsdbFSWaitAllBgTask(STFileSystem *fs);
 int32_t tsdbFSDisableBgTask(STFileSystem *fs);
@@ -70,6 +71,7 @@ int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
 struct STFSBgTask {
   EFSBgTaskT type;
   int32_t (*run)(void *arg);
+  void (*free)(void *arg);
   void *arg;
 
   TdThreadCond done[1];

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -15,6 +15,7 @@
 
 #include "tsdb.h"
 #include "tsdbFSet2.h"
+#include "tsdbMerge.h"
 #include "tsdbReadUtil.h"
 #include "tsdbSttFileRW.h"
 
@@ -352,10 +353,14 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t uidComparFn(const void *p1, const void *p2) {
-  const uint64_t *uid1 = p1;
+static int32_t suidComparFn(const void *target, const void *p2) {
+  const uint64_t *targetUid = target;
   const uint64_t *uid2 = p2;
-  return (*uid1) - (*uid2);
+  if (*uid2 == (*targetUid)) {
+    return 0;
+  } else {
+    return (*targetUid) < (*uid2) ? -1 : 1;
+  }
 }
 
 static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint64_t suid, uint64_t uid,
@@ -372,29 +377,55 @@ static bool existsFromSttBlkStatis(const TStatisBlkArray *pStatisBlkArray, uint6
     }
   }
 
-  //  for (; i < TARRAY2_SIZE(pStatisBlkArray); ++i) {
-  //    SStatisBlk *p = &pStatisBlkArray->data[i];
-  //    if (p->minTbid.uid <= uid && p->maxTbid.uid >= uid) {
-  //      break;
-  //    }
-  //
-  //    if (p->maxTbid.uid < uid) {
-  //      break;
-  //    }
-  //  }
-
   if (i >= TARRAY2_SIZE(pStatisBlkArray)) {
     return false;
   }
 
-  SStatisBlk    *p = &pStatisBlkArray->data[i];
-  STbStatisBlock block = {0};
-  tsdbSttFileReadStatisBlock(pReader, p, &block);
+  while (i < TARRAY2_SIZE(pStatisBlkArray)) {
+    SStatisBlk *p = &pStatisBlkArray->data[i];
+    if (p->minTbid.suid > suid) {
+      return false;
+    }
+
+    STbStatisBlock block = {0};
+    tsdbSttFileReadStatisBlock(pReader, p, &block);
+
+    int32_t index = tarray2SearchIdx(block.suid, &suid, sizeof(int64_t), suidComparFn, TD_EQ);
+    if (index == -1) {
+      tStatisBlockDestroy(&block);
+      return false;
+    }
+
+    int32_t j = index;
+    if (block.uid->data[j] == uid) {
+      tStatisBlockDestroy(&block);
+      return true;
+    } else if (block.uid->data[j] > uid) {
+      while (j >= 0 && block.suid->data[j] == suid) {
+        if (block.uid->data[j] == uid) {
+          tStatisBlockDestroy(&block);
+          return true;
+        } else {
+          j -= 1;
+        }
+      }
+    } else {
+      j = index + 1;
+      while (j < block.suid->size && block.suid->data[j] == suid) {
+        if (block.uid->data[j] == uid) {
+          tStatisBlockDestroy(&block);
+          return true;
+        } else {
+          j += 1;
+        }
+      }
+    }
 
-  int32_t index = tarray2SearchIdx(block.uid, &uid, sizeof(int64_t), uidComparFn, TD_EQ);
-  tStatisBlockDestroy(&block);
+    tStatisBlockDestroy(&block);
+    i += 1;
+  }
 
-  return (index != -1);
+  return false;
 }
 
 int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t iStt, int8_t backward,
@@ -413,6 +444,13 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader
   pIter->pReader = pSttFileReader;
   pIter->pBlockLoadInfo = pBlockLoadInfo;
 
+  if (pIter->pReader == NULL) {
+    tsdbError("stt file reader is null, %s", idStr);
+    pIter->pSttBlk = NULL;
+    pIter->iSttBlk = -1;
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (!pBlockLoadInfo->sttBlockLoaded) {
     int64_t st = taosGetTimestampUs();
 
@@ -445,12 +483,12 @@ int32_t tLDataIterOpen2(struct SLDataIter *pIter, SSttFileReader *pSttFileReader
     tsdbDebug("load the stt file info completed, elapsed time:%.2fms, %s", el, idStr);
   }
 
-  //  bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader);
-  //  if (!exists) {
-  //    pIter->iSttBlk = -1;
-  //    pIter->pSttBlk = NULL;
-  //    return TSDB_CODE_SUCCESS;
-  //  }
+  // bool exists = existsFromSttBlkStatis(pBlockLoadInfo->pSttStatisBlkArray, suid, uid, pIter->pReader);
+  // if (!exists) {
+  //   pIter->iSttBlk = -1;
+  //   pIter->pSttBlk = NULL;
+  //   return TSDB_CODE_SUCCESS;
+  // }
 
   // find the start block, actually we could load the position to avoid repeatly searching for the start position when
   // the skey is updated.
@@ -759,7 +797,6 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
 
   pMTree->ignoreEarlierTs = false;
 
-  // todo handle other level of stt files, here only deal with the first level stt
   int32_t size = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->size;
   if (size == 0) {
     goto _end;
@@ -784,6 +821,12 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
         SLDataIter *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
         taosArrayPush(pList, &pIter);
       }
+    } else if (numOfIter > TARRAY2_SIZE(pSttLevel->fobjArr)){
+        int32_t inc = numOfIter - TARRAY2_SIZE(pSttLevel->fobjArr);
+        for (int i = 0; i < inc; ++i) {
+            SLDataIter *pIter = taosArrayPop(pList);
+            destroyLDataIter(pIter);
+        }
     }
 
     for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) {  // open all last file
@@ -799,7 +842,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
 
         code = tsdbSttFileReaderOpen(pSttLevel->fobjArr->data[i]->fname, &conf, &pSttFileReader);
         if (code != TSDB_CODE_SUCCESS) {
-          return code;
+          tsdbError("open stt file reader error. file name %s, code %s, %s", pSttLevel->fobjArr->data[i]->fname,
+                    tstrerror(code), pMTree->idStr);
         }
       }
 
@@ -814,7 +858,7 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
       if (code != TSDB_CODE_SUCCESS) {
         goto _end;
       }
-
+      
       bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
       if (hasVal) {
         tMergeTreeAddIter(pMTree, pIter);

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -1121,6 +1121,8 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl
   SBrinRecord* p = taosArrayGet(pTableBlockScanInfo->pBlockList, pBlockInfo->tbBlockIdx + step);
   memcpy(pRecord, p, sizeof(SBrinRecord));
 
+  *nextIndex = pBlockInfo->tbBlockIdx + step;
+
   //  tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk);
   return true;
 }
@@ -1729,45 +1731,41 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
 
     // row in last file block
     TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
-    int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
+    int64_t ts = getCurrentKeyInLastBlock(pLastBlockReader);
+
     if (ASCENDING_TRAVERSE(pReader->info.order)) {
-      if (key < tsLast) {
-        return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
-      } else if (key > tsLast) {
-        return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false);
-      }
-    } else {
-      if (key > tsLast) {
+      if (key < ts) {  // imem, mem are all empty, file blocks (data blocks and last block) exist
         return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
-      } else if (key < tsLast) {
-        return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false);
-      }
-    }
-    // the following for key == tsLast
-    SRow*   pTSRow = NULL;
-    int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-
-    doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader);
+      } else if (key == ts) {
+        SRow*   pTSRow = NULL;
+        int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->info.pSchema);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
 
-    TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
-    tsdbRowMergerAdd(pMerger, pRow1, NULL);
+        doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader);
 
-    doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr);
+        TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree);
+        tsdbRowMergerAdd(pMerger, pRow1, NULL);
 
-    code = tsdbRowMergerGetRow(pMerger, &pTSRow);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
+        doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, pMerger, &pReader->info.verRange, pReader->idStr);
 
-    code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
+        code = tsdbRowMergerGetRow(pMerger, &pTSRow);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
 
-    taosMemoryFree(pTSRow);
-    tsdbRowMergerClear(pMerger);
-    return code;
+        code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
 
+        taosMemoryFree(pTSRow);
+        tsdbRowMergerClear(pMerger);
+        return code;
+      } else {  // key > ts
+        return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false);
+      }
+    } else {  // desc order
+      return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, pBlockData, true);
+    }
   } else {  // only last block exists
     return doMergeFileBlockAndLastBlock(pLastBlockReader, pReader, pBlockScanInfo, NULL, false);
   }
@@ -2194,8 +2192,7 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
   TSDBROW *pRow = NULL, *piRow = NULL;
-  int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] :
-                                                                   (ASCENDING_TRAVERSE(pReader->info.order) ? INT64_MAX : INT64_MIN);
+  int64_t key = (pBlockData->nRow > 0 && (!pDumpInfo->allDumped)) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN;
   if (pBlockScanInfo->iter.hasVal) {
     pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader);
   }
@@ -2569,18 +2566,9 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
 
     // load the last data block of current table
     STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;
-    if (pScanInfo == NULL) {
-      tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr);
-      bool hasNexTable = moveToNextTable(pUidList, pStatus);
-      if (!hasNexTable) {
-        return TSDB_CODE_SUCCESS;
-      }
-
-      continue;
-    }
-
     if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) {
       // reset the index in last block when handing a new file
+      //      doCleanupTableScanInfo(pScanInfo);
       bool hasNexTable = moveToNextTable(pUidList, pStatus);
       if (!hasNexTable) {
         return TSDB_CODE_SUCCESS;
@@ -2589,6 +2577,9 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
       continue;
     }
 
+    // reset the index in last block when handing a new file
+    //    doCleanupTableScanInfo(pScanInfo);
+
     bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader);
     if (!hasDataInLastFile) {
       bool hasNexTable = moveToNextTable(pUidList, pStatus);
@@ -2678,32 +2669,16 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
         (ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey;
     code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
   } else {
-    bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
-    int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
-    if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) ||
-                                 (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) {
-      // whole block is required, return it directly
-      SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
-      pInfo->rows = pBlockInfo->record.numRow;
-      pInfo->id.uid = pScanInfo->uid;
-      pInfo->dataLoad = 0;
-      pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey};
-      setComposedBlockFlag(pReader, false);
-      setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
+    if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->info.order)) {
+      // only return the rows in last block
+      int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
+      ASSERT(tsLast >= pBlockInfo->record.lastKey);
 
-      // update the last key for the corresponding table
-      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey;
-      tsdbDebug("%p uid:%" PRIu64
-                " clean file block retrieved from file, global index:%d, "
-                "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
-                pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow,
-                pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
-    } else {
       SBlockData* pBData = &pReader->status.fileBlockData;
       tBlockDataReset(pBData);
 
       SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
-      tsdbDebug("load data in last block firstly %s", pReader->idStr);
+      tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr);
 
       int64_t st = taosGetTimestampUs();
 
@@ -2734,8 +2709,23 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
                   pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
                   pResBlock->info.rows, el, pReader->idStr);
       }
-    }
+    } else {  // whole block is required, return it directly
+      SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
+      pInfo->rows = pBlockInfo->record.numRow;
+      pInfo->id.uid = pScanInfo->uid;
+      pInfo->dataLoad = 0;
+      pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey};
+      setComposedBlockFlag(pReader, false);
+      setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
 
+      // update the last key for the corresponding table
+      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey;
+      tsdbDebug("%p uid:%" PRIu64
+                " clean file block retrieved from file, global index:%d, "
+                "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
+                pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow,
+                pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
+    }
   }
 
   return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;
@@ -4109,6 +4099,11 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
 
     tsdbDataFileReaderClose(&pReader->pFileReader);
 
+    int64_t loadBlocks = 0;
+    double  elapse = 0;
+    pReader->status.pLDataIterArray = destroySttBlockReader(pReader->status.pLDataIterArray, &loadBlocks, &elapse);
+    pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
+
     // resetDataBlockScanInfo excluding lastKey
     STableBlockScanInfo** p = NULL;
     int32_t               iter = 0;
@@ -4179,7 +4174,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
     }
   }
 
-  tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false);
+  tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false);
   pReader->pReadSnap = NULL;
   pReader->flag = READER_STATUS_SUSPEND;
 

source/dnode/vnode/src/tsdb/tsdbReadUtil.c

@@ -22,7 +22,7 @@
 #include "tsdbUtil2.h"
 #include "tsimplehash.h"
 
-static int32_t uidComparFunc(const void* p1, const void* p2) {
+int32_t uidComparFunc(const void* p1, const void* p2) {
   uint64_t pu1 = *(uint64_t*)p1;
   uint64_t pu2 = *(uint64_t*)p2;
   if (pu1 == pu2) {

source/dnode/vnode/src/tsdb/tsdbReadUtil.h

@@ -36,6 +36,16 @@ typedef enum {
   EXTERNAL_ROWS_NEXT = 0x3,
 } EContentData;
 
+typedef struct STsdbReaderInfo {
+  uint64_t      suid;
+  STSchema*     pSchema;
+  EReadMode     readMode;
+  uint64_t      rowsNum;
+  STimeWindow   window;
+  SVersionRange verRange;
+  int16_t       order;
+} STsdbReaderInfo;
+
 typedef struct SBlockInfoBuf {
   int32_t currentIndex;
   SArray* pData;
@@ -215,6 +225,8 @@ typedef struct SBrinRecordIter {
   SBrinRecord      record;
 } SBrinRecordIter;
 
+int32_t uidComparFunc(const void* p1, const void* p2);
+
 STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id);
 
 SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
@@ -241,6 +253,41 @@ void    loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piM
 int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
 int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
 
+typedef struct {
+  SArray* pTombData;
+} STableLoadInfo;
+
+struct SDataFileReader;
+
+typedef struct SCacheRowsReader {
+  STsdb*                  pTsdb;
+  STsdbReaderInfo         info;
+  TdThreadMutex           readerMutex;
+  SVnode*                 pVnode;
+  STSchema*               pSchema;
+  STSchema*               pCurrSchema;
+  uint64_t                uid;
+  char**                  transferBuf;  // todo remove it soon
+  int32_t                 numOfCols;
+  SArray*                 pCidList;
+  int32_t*                pSlotIds;
+  int32_t                 type;
+  int32_t                 tableIndex;  // currently returned result tables
+  STableKeyInfo*          pTableList;  // table id list
+  int32_t                 numOfTables;
+  uint64_t*               uidList;
+  SSHashObj*              pTableMap;
+  SArray*                 pLDataIterArray;
+  struct SDataFileReader* pFileReader;
+  STFileSet*              pCurFileSet;
+  const TBrinBlkArray*    pBlkArray;
+  STsdbReadSnap*          pReadSnap;
+  char*                   idstr;
+  int64_t                 lastTs;
+} SCacheRowsReader;
+
+int32_t tsdbCacheGetBatch(STsdb* pTsdb, tb_uid_t uid, SArray* pLastArray, SCacheRowsReader* pr, int8_t ltype);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -100,7 +100,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
               .type = fobj->f->type,
               .did = did[0],
               .fid = fobj->f->fid,
-              .cid = rtner->cid,
+              .cid = fobj->f->cid,
               .size = fobj->f->size,
               .stt[0] =
                   {
@@ -125,6 +125,7 @@ _exit:
 
 typedef struct {
   STsdb  *tsdb;
+  int32_t sync;
   int64_t now;
 } SRtnArg;
 
@@ -251,28 +252,33 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
   }
-  taosMemoryFree(arg);
   return code;
 }
 
-int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now, int64_t *taskid) {
+static void tsdbFreeRtnArg(void *arg) {
+  SRtnArg *rArg = (SRtnArg *)arg;
+  if (rArg->sync) {
+    tsem_post(&rArg->tsdb->pVnode->canCommit);
+  }
+  taosMemoryFree(arg);
+}
+
+int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
   SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
   if (arg == NULL) return TSDB_CODE_OUT_OF_MEMORY;
-
   arg->tsdb = tsdb;
+  arg->sync = sync;
   arg->now = now;
 
-  int32_t code = tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, arg, taskid);
-  if (code) taosMemoryFree(arg);
-
-  return code;
-}
+  if (sync) {
+    tsem_wait(&tsdb->pVnode->canCommit);
+  }
 
-int32_t tsdbSyncRetention(STsdb *tsdb, int64_t now) {
   int64_t taskid;
-
-  int32_t code = tsdbAsyncRetention(tsdb, now, &taskid);
-  if (code) return code;
-
-  return tsdbFSWaitBgTask(tsdb->pFS, taskid);
+  int32_t code =
+      tsdbFSScheduleBgTask(tsdb->pFS, TSDB_BG_TASK_RETENTION, tsdbDoRetention2, tsdbFreeRtnArg, arg, &taskid);
+  if (code) {
+    tsdbFreeRtnArg(arg);
+  }
+  return code;
 }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -342,18 +342,18 @@ static int32_t tsdbSnapCmprTombData(STsdbSnapReader* reader, uint8_t** data) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  int64_t size = sizeof(SSnapDataHdr);
+  int64_t size = 0;
   for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) {
     size += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i);
   }
 
-  data[0] = taosMemoryMalloc(size);
+  data[0] = taosMemoryMalloc(size + sizeof(SSnapDataHdr));
   if (data[0] == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  SSnapDataHdr* hdr = (SSnapDataHdr*)data[0];
+  SSnapDataHdr* hdr = (SSnapDataHdr*)(data[0]);
   hdr->type = SNAP_DATA_DEL;
   hdr->size = size;
 
@@ -938,7 +938,7 @@ static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombB
   int32_t code = 0;
   int32_t lino = 0;
 
-  int64_t size = hdr->size - sizeof(*hdr);
+  int64_t size = hdr->size;
   ASSERT(size % TOMB_RECORD_ELEM_NUM == 0);
   size = size / TOMB_RECORD_ELEM_NUM;
   ASSERT(size % sizeof(int64_t) == 0);

source/dnode/vnode/src/tsdb/tsdbSttFileRW.c

@@ -975,6 +975,11 @@ int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *re
 _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  } else {
+    tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
+              ", version:%" PRId64,
+              TD_VID(writer->config->tsdb->pVnode), writer->fd->path, writer->config->cid, record->suid, record->uid,
+              record->version);
   }
   return code;
 }