Merge branch '2.6' into docs/sangshuduo/taosbenchmark-docs-for2.6

.gitignore

 build/
 .ycm_extra_conf.py
 .vscode/
+.cache/
+compile_commands.json
 .idea/
 cmake-build-debug/
 cmake-build-release/
+cmake-build-relwithdebinfo/
 cscope.out
 cscope.files
 tags

Jenkinsfile2

@@ -23,31 +23,31 @@ def sync_source() {
             sh '''
                 cd ${WKC}
                 git clean -fxd
-                git checkout master
+                git checkout -f  master
             '''
         } else if (env.CHANGE_TARGET == '2.0') {
             sh '''
                 cd ${WKC}
                 git clean -fxd
-                git checkout 2.0
+                git checkout -f   2.0
             '''
         } else if (env.CHANGE_TARGET == '2.4') {
             sh '''
                 cd ${WKC}
                 git clean -fxd
-                git checkout 2.4
+                git checkout -f   2.4
             '''
         } else if (env.CHANGE_TARGET == '2.6') {
             sh '''
                 cd ${WKC}
                 git clean -fxd
-                git checkout 2.6
+                git checkout -f   2.6
             '''
         } else {
             sh '''
                 cd ${WKC}
                 git clean -fxd
-                git checkout develop
+                git checkout  -f   develop
             '''
         }
     }
@@ -79,7 +79,7 @@ def sync_source() {
         } else if (env.CHANGE_TARGET == '2.6') {
             sh '''
                 cd ${WK}
-                git checkout 2.6
+                git checkout -f 2.6
             '''
         } else {
             sh '''
@@ -194,47 +194,47 @@ def pre_test_win(){
         if (env.CHANGE_TARGET == 'master') {
             bat '''
                 cd %WIN_INTERNAL_ROOT%
-                git checkout master
+                git checkout -f  master
             '''
             bat '''
                 cd %WIN_COMMUNITY_ROOT%
-                git checkout master
+                git checkout -f master
             '''
         } else if (env.CHANGE_TARGET == '2.0') {
             bat '''
                 cd %WIN_INTERNAL_ROOT%
-                git checkout 2.0
+                git checkout -f 2.0
             '''
             bat '''
                 cd %WIN_COMMUNITY_ROOT%
-                git checkout 2.0
+                git checkout -f 2.0
             '''
         } else if (env.CHANGE_TARGET == '2.4') {
             bat '''
                 cd %WIN_INTERNAL_ROOT%
-                git checkout 2.4
+                git checkout -f 2.4
             '''
             bat '''
                 cd %WIN_COMMUNITY_ROOT%
-                git checkout 2.4
+                git checkout -f  2.4
             '''
         } else if (env.CHANGE_TARGET == '2.6') {
             bat '''
                 cd %WIN_INTERNAL_ROOT%
-                git checkout 2.6
+                git checkout -f 2.6
             '''
             bat '''
                 cd %WIN_COMMUNITY_ROOT%
-                git checkout 2.6
+                git checkout -f  2.6
             '''
         } else {
             bat '''
                 cd %WIN_INTERNAL_ROOT%
-                git checkout develop
+                git checkout -f  develop
             '''
             bat '''
                 cd %WIN_COMMUNITY_ROOT%
-                git checkout develop
+                git checkout -f develop
             '''
         }
     }

docs/en/12-taos-sql/02-database.md

@@ -20,20 +20,20 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
 3. The maximum length of database name is 33 bytes.
 4. The maximum length of a SQL statement is 65,480 bytes.
 5. Below are the parameters that can be used when creating a database
-   - cache: [Description](/reference/config/#cache)
-   - blocks: [Description](/reference/config/#blocks)
-   - days: [Description](/reference/config/#days)
-   - keep: [Description](/reference/config/#keep)
-   - minRows: [Description](/reference/config/#minrows)
-   - maxRows: [Description](/reference/config/#maxrows)
-   - wal: [Description](/reference/config/#wallevel)
-   - fsync: [Description](/reference/config/#fsync)
-   - update: [Description](/reference/config/#update)
-   - cacheLast: [Description](/reference/config/#cachelast)
-   - replica: [Description](/reference/config/#replica)
-   - quorum: [Description](/reference/config/#quorum)
-   - comp: [Description](/reference/config/#comp)
-   - precision: [Description](/reference/config/#precision)
+   - cache: [Description](../../reference/config/#cache)
+   - blocks: [Description](../../reference/config/#blocks)
+   - days: [Description](../../reference/config/#days)
+   - keep: [Description](../../reference/config/#keep)
+   - minRows: [Description](../../reference/config/#minrows)
+   - maxRows: [Description](../../reference/config/#maxrows)
+   - wal: [Description](../../reference/config/#wallevel)
+   - fsync: [Description](../../reference/config/#fsync)
+   - update: [Description](../../reference/config/#update)
+   - cacheLast: [Description](../../reference/config/#cachelast)
+   - replica: [Description](../../reference/config/#replica)
+   - quorum: [Description](../../reference/config/#quorum)
+   - comp: [Description](../../reference/config/#comp)
+   - precision: [Description](../../reference/config/#precision)
 6. Please note that all of the parameters mentioned in this section are configured in configuration file `taos.cfg` on the TDengine server. If not specified in the `create database` statement, the values from taos.cfg are used by default. To override default parameters, they must be specified in the `create database` statement.
    
 :::

docs/en/12-taos-sql/10-function.md

@@ -191,7 +191,7 @@ Query OK, 1 row(s) in set (0.000921s)
 SELECT MODE(field_name) FROM tb_name [WHERE clause];
 ```
 
-**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. It can't be used on timestamp column or tags.
+**Description**:The value which has the highest frequency of occurrence. One random value is returned if there are multiple values which have highest frequency of occurrence. It can't be used on timestamp column or tags.
 
 **Return value type**:Same as the data type of the column being operated upon
 

docs/zh/12-taos-sql/02-database.md

@@ -20,20 +20,20 @@ CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [DAYS days] [UPDATE 1];
 3. 数据库名最大长度为 33；
 4. 一条 SQL 语句的最大长度为 65480 个字符；
 5. 创建数据库时可用的参数有：
-   - cache: [详细说明](/reference/config/#cache)
-   - blocks: [详细说明](/reference/config/#blocks)
-   - days: [详细说明](/reference/config/#days)
-   - keep: [详细说明](/reference/config/#keep)
-   - minRows: [详细说明](/reference/config/#minrows)
-   - maxRows: [详细说明](/reference/config/#maxrows)
-   - wal: [详细说明](/reference/config/#wallevel)
-   - fsync: [详细说明](/reference/config/#fsync)
-   - update: [详细说明](/reference/config/#update)
-   - cacheLast: [详细说明](/reference/config/#cachelast)
-   - replica: [详细说明](/reference/config/#replica)
-   - quorum: [详细说明](/reference/config/#quorum)
-   - comp: [详细说明](/reference/config/#comp)
-   - precision: [详细说明](/reference/config/#precision)
+   - cache: [详细说明](../../reference/config/#cache)
+   - blocks: [详细说明](../../reference/config/#blocks)
+   - days: [详细说明](../../reference/config/#days)
+   - keep: [详细说明](../../reference/config/#keep)
+   - minRows: [详细说明](../../reference/config/#minrows)
+   - maxRows: [详细说明](../../reference/config/#maxrows)
+   - wal: [详细说明](../../reference/config/#wallevel)
+   - fsync: [详细说明](../../reference/config/#fsync)
+   - update: [详细说明](../../reference/config/#update)
+   - cacheLast: [详细说明](../../reference/config/#cachelast)
+   - replica: [详细说明](../../reference/config/#replica)
+   - quorum: [详细说明](../../reference/config/#quorum)
+   - comp: [详细说明](../../reference/config/#comp)
+   - precision: [详细说明](../../reference/config/#precision)
 6. 请注意上面列出的所有参数都可以配置在配置文件 `taosd.cfg` 中作为创建数据库时使用的默认配置， `create database` 的参数中明确指定的会覆盖配置文件中的设置。
 
 :::
@@ -86,7 +86,7 @@ REPLICA 参数是指修改数据库副本数，取值范围 [1, 3]。在集群
 ALTER DATABASE db_name KEEP 365;
 ```
 
-KEEP 参数是指修改数据文件保存的天数，缺省值为 3650，取值范围 [days, 365000]，必须大于或等于 days 参数值。
+KEEP 参数是指修改数据文件保存的天数，缺省值为 3650，取值范围 [days, 36500]，必须大于或等于 days 参数值。
 
 ```
 ALTER DATABASE db_name QUORUM 2;

docs/zh/12-taos-sql/10-function.md

@@ -193,7 +193,7 @@ Query OK, 1 row(s) in set (0.000921s)
 SELECT MODE(field_name) FROM tb_name [WHERE clause];
 ```
 
-**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出空。不能匹配标签、时间戳输出。
+**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，则随机输出其中某个值。不能匹配标签、时间戳输出。
 
 **返回数据类型**：同应用的字段。
 

examples/c/apitest.c

@@ -279,7 +279,8 @@ void verify_schema_less(TAOS* taos) {
   int code = 0, affected_rows = 0;
 
   char* lines[] = {
-      "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000",
+      // test support \n \t \r 
+      "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"aaa\tbbb\nccc\\\\tddd\\\\neee\\nfff\\rggg\\thhh\",c2=false,c4=4f64 1626006833639000000",
       "st,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000",
       "ste,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1626056811823316532",
       "st,t1=4i64,t2=5f64,t3=\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000",

packaging/cfg/taos.cfg

@@ -191,6 +191,9 @@ keepColumnName            1
 # enable/disable system monitor
 # monitor               1
 
+# enable/disable system audit
+# audit                 0
+
 # enable/disable recording the SQL statements via restful interface
 # httpEnableRecordSql   0
 
@@ -310,3 +313,12 @@ keepColumnName            1
 
 # unit Hour. Latency of data migration
 # keepTimeOffset     0
+
+# taosc write batch size, maximum 4096, suggested value 64 ~ 512, default 0, 0 means disable write batching.
+# writeBatchSize        0
+
+# taosc write batch timeout in milliseconds, maximum 2048, suggested value 2 ~ 100, default 10.
+# writeBatchTimeout     10
+
+# using thread local write batching. this option is not available when writeBatchSize = 0.
+# writeBatchThreadLocal 0

packaging/tools/install.sh

@@ -105,6 +105,10 @@ elif echo $osinfo | grep -qwi "debian"; then
 elif echo $osinfo | grep -qwi "Kylin"; then
   #  echo "This is Kylin system"
   os_type=1
+elif echo $osinfo | grep -qwi "KylinSec"; then 
+  # echo "This is KylinSec system" 
+  os_type=2 
+  csudo="" 
 elif echo $osinfo | grep -qwi "Red"; then
   #  echo "This is Red Hat system"
   os_type=1

src/client/inc/tscBatchMerge.h

+/*
+* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+*
+* This program is free software: you can use, redistribute, and/or modify
+* it under the terms of the GNU Affero General Public License, version 3
+* or later ("AGPL"), as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TDENGINE_TSCBATCHMERGE_H
+#define TDENGINE_TSCBATCHMERGE_H
+
+#include "hash.h"
+#include "taosmsg.h"
+#include "tarray.h"
+#include "tscUtil.h"
+#include "tsclient.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * A builder of SSubmitBlk.
+ */
+typedef struct SSubmitBlkBuilder {
+  // the metadata of the SSubmitBlk.
+  SSubmitBlk* metadata;
+  
+  // the array stores all the rows in a table, aka SArray<SMemRow>.
+  SArray* rows;
+  
+} SSubmitBlkBuilder;
+
+/**
+ * The builder to build SSubmitMsg::blocks.
+ */
+typedef struct SSubmitMsgBlocksBuilder {
+  // SHashObj<table_uid, SSubmitBlkBuilder*>.
+  SHashObj* blockBuilders;
+  int64_t vgId;
+} SSubmitMsgBlocksBuilder;
+
+/**
+ * STableDataBlocksBuilder is a tool to build data blocks by append the existing data blocks in a vnode.
+ */
+typedef struct STableDataBlocksBuilder {
+  SSubmitMsgBlocksBuilder* blocksBuilder;
+  STableDataBlocks* firstBlock;
+  int64_t vgId;
+} STableDataBlocksBuilder;
+
+/**
+ * STableDataBlocksListBuilder is a tool to build vnode data blocks list by appending exist data blocks.
+ */
+typedef struct STableDataBlocksListBuilder {
+  SHashObj* dataBlocksBuilders;
+} STableDataBlocksListBuilder;
+
+/**
+ * A Builder to build SInsertStatementParam::pTableNameList.
+ */
+typedef struct STableNameListBuilder {
+  // store the unsorted table names, SArray<SName*>.
+  SArray* pTableNameList;
+} STableNameListBuilder;
+
+/**
+ * Create a SSubmitBlkBuilder using exist metadata.
+ * 
+ * @param metadata  the metadata.
+ * @return          the SSubmitBlkBuilder.
+ */
+SSubmitBlkBuilder* createSSubmitBlkBuilder(SSubmitBlk* metadata);
+
+/**
+ * Destroy the SSubmitBlkBuilder.
+ * 
+ * @param builder the SSubmitBlkBuilder.
+ */
+void destroySSubmitBlkBuilder(SSubmitBlkBuilder* builder);
+
+/**
+ * Append a SSubmitBlk* to the builder. The table uid in pBlock must be the same with the builder's.
+ * 
+ * @param builder  the SSubmitBlkBuilder.
+ * @param pBlock   the pBlock to append.
+ * @return         whether the append is success.
+ */
+bool appendSSubmitBlkBuilder(SSubmitBlkBuilder* builder, SSubmitBlk *pBlock);
+
+
+/**
+ * Build and write SSubmitBlk to `target`
+ * 
+ * @param builder   the SSubmitBlkBuilder.
+ * @param target    the target to write.
+ * @param nRows     the number of rows in SSubmitBlk*.
+ * @return          the writen bytes.
+ */
+size_t writeSSubmitBlkBuilder(SSubmitBlkBuilder* builder, SSubmitBlk* target, size_t* nRows);
+
+/**
+ * Get the expected writen bytes of `writeSSubmitBlkBuilder`.
+ * 
+ * @param builder  the SSubmitBlkBuilder.
+ * @return         the expected writen bytes of `writeSSubmitBlkBuilder`.
+ */
+size_t nWriteSSubmitBlkBuilder(SSubmitBlkBuilder* builder);
+
+/**
+ * Create a SSubmitMsgBuilder.
+ * 
+ * @param vgId  the vgId of SSubmitMsg.
+ * @return      the SSubmitMsgBuilder.
+ */
+SSubmitMsgBlocksBuilder* createSSubmitMsgBuilder(int64_t vgId);
+
+/**
+ * Get the expected writen bytes of `writeSSubmitMsgBlocksBuilder`.
+ * 
+ * @param builder  the SSubmitMsgBlocksBuilder.
+ * @return         the expected writen bytes of `writeSSubmitMsgBlocksBuilder`.
+ */
+size_t nWriteSSubmitMsgBuilder(SSubmitMsgBlocksBuilder* builder);
+
+/**
+ * Build and write SSubmitMsg::blocks to `pBlocks`
+ * 
+ * @param builder   the SSubmitBlkBuilder.
+ * @param pBlocks   the target to write.
+ * @param nRows     the number of row in SSubmitMsg::blocks.
+ * @return          the writen bytes.
+ */
+size_t writeSSubmitMsgBlocksBuilder(SSubmitMsgBlocksBuilder* builder, SSubmitBlk* pBlocks, size_t* nRows);
+
+/**
+ * Get the number of block in SSubmitMsgBlocksBuilder.
+ * @param builder the SSubmitMsgBlocksBuilder.
+ * @return        the number of SSubmitBlk block.
+ */
+size_t nBlockSSubmitMsgBlocksBuilder(SSubmitMsgBlocksBuilder* builder);
+
+/**
+ * Destroy the SSubmitMsgBlocksBuilder.
+ * 
+ * @param builder the SSubmitMsgBlocksBuilder to destroy.
+ */
+void destroySSubmitMsgBuilder(SSubmitMsgBlocksBuilder* builder);
+
+/**
+ * Append SSubmitMsg* to the SSubmitMsgBlocksBuilder.
+ * 
+ * @param builder       the SSubmitMsgBlocksBuilder.
+ * @param pBlocks       the SSubmitBlk in SSubmitMsg::blocks. 
+ * @param nBlocks       the number of blocks in SSubmitMsg.
+ * @return whether the append is success.
+ */
+bool appendSSubmitMsgBlocks(SSubmitMsgBlocksBuilder* builder, SSubmitBlk* pBlocks, size_t nBlocks);
+
+/**
+ * Create the STableDataBlocksBuilder.
+ * 
+ * @param vgId the vgId of STableDataBlocksBuilder.
+ * @return the STableDataBlocksBuilder.
+ */
+STableDataBlocksBuilder* createSTableDataBlocksBuilder(int64_t vgId);
+
+/**
+ * Destroy the STableDataBlocksBuilder.
+ * @param builder the STableDataBlocksBuilder.
+ */
+void destroySTableDataBlocksBuilder(STableDataBlocksBuilder *builder);
+
+/**
+ * Append a data blocks to STableDataBlocksBuilder.
+ * @param builder    the STableDataBlocksBuilder.
+ * @param dataBlocks the dataBlocks to append. the vgId of dataBlocks must be same with the STableDataBlocksBuilder.
+ * @return           whether the append is success.
+ */
+bool appendSTableDataBlocksBuilder(STableDataBlocksBuilder* builder, STableDataBlocks* dataBlocks);
+
+/**
+ * Build the data blocks for single vnode.
+ * @param builder   the STableDataBlocksBuilder.
+ * @param nRows     the number of row in STableDataBlocks.
+ * @return          the data blocks for single vnode.
+ */
+STableDataBlocks* buildSTableDataBlocksBuilder(STableDataBlocksBuilder* builder, size_t* nRows);
+
+/**
+ * Create the STableDataBlocksListBuilder.
+ * 
+ * @return the STableDataBlocksListBuilder.
+ */
+STableDataBlocksListBuilder* createSTableDataBlocksListBuilder();
+
+/**
+ * Destroy the STableDataBlocksListBuilder.
+ * 
+ * @param builder the STableDataBlocksListBuilder.
+ */
+void destroySTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder);
+
+/**
+ * Append a data blocks to STableDataBlocksListBuilder.
+ * 
+ * @param builder       the STableDataBlocksListBuilder.
+ * @param dataBlocks    the data blocks.
+ * @return              whether the append is success.
+ */
+bool appendSTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder, STableDataBlocks* dataBlocks);
+
+/**
+ * Build the vnode data blocks list.
+ * 
+ * @param builder   the STableDataBlocksListBuilder.
+ * @param nTables   the number of table in vnode data blocks list.
+ * @param nRows     the number of row in vnode data blocks list. 
+ * @return          the vnode data blocks list.
+ */
+SArray* buildSTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder, size_t* nTables, size_t* nRows);
+
+/**
+ * Create STableNameListBuilder.
+ */
+STableNameListBuilder* createSTableNameListBuilder();
+
+/**
+ * Destroy the STableNameListBuilder.
+ * @param builder the STableNameListBuilder.
+ */
+void destroySTableNameListBuilder(STableNameListBuilder* builder);
+
+/**
+ * Insert a SName to builder.
+ * 
+ * @param builder  the STableNameListBuilder.
+ * @param name     the table name.
+ * @return         whether it is success.
+ */
+bool insertSTableNameListBuilder(STableNameListBuilder* builder, SName* name);
+
+/**
+ * Build the STable name list.
+ * 
+ * @param builder       the STableNameListBuilder.
+ * @param numOfTables   the number of table.
+ * @return              the STable name list.
+ */
+SName** buildSTableNameListBuilder(STableNameListBuilder* builder, size_t* numOfTables);
+
+/**
+ * Merge the KV-PayLoad SQL objects into single one. 
+ * The statements here must be an insertion statement and no schema attached.
+ * 
+ * @param polls  the array of SSqlObj*.
+ * @param nPolls the number of SSqlObj* in the array.
+ * @param result the returned result. result is not null!
+ * @return       the status code.
+ */
+int32_t tscMergeSSqlObjs(SSqlObj** polls, size_t nPolls, SSqlObj *result);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TSCBATCHMERGE_H

src/client/inc/tscBatchWrite.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TSCBATCHWRITE_H
+#define TDENGINE_TSCBATCHWRITE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tthread.h"
+
+// forward declaration.
+typedef struct STscObj                   STscObj;
+typedef struct SSqlObj                   SSqlObj;
+typedef struct SDispatcherTimeoutManager SDispatcherTimeoutManager;
+
+/**
+ * SAsyncBatchWriteDispatcher is an async batching write dispatcher (ABWD). ABWD accepts the recent SQL requests and put
+ * them in a queue waiting to be scheduled. When the number of requests in the queue reaches batch_size, it merges the
+ * requests in the queue and sends them to the server, thus reducing the network overhead caused by multiple
+ * communications to the server and directly improving the throughput of small object asynchronous writes.
+ */
+typedef struct SAsyncBatchWriteDispatcher {
+  // the client object.
+  STscObj* pClient;
+
+  // the timeout manager.
+  SDispatcherTimeoutManager* timeoutManager;
+
+  // the mutex to protect the dispatcher.
+  pthread_mutex_t bufferMutex;
+
+  // the cond to signal when buffer not full.
+  pthread_cond_t notFull;
+
+  // the maximum number of insertion rows in a batch.
+  int32_t batchSize;
+
+  // the number of insertion rows in the buffer.
+  int32_t currentSize;
+
+  // the number of items in the buffer.
+  int32_t bufferSize;
+
+  // whether the dispatcher is shutdown.
+  bool shutdown;
+
+  SSqlObj* buffer[];
+} SAsyncBatchWriteDispatcher;
+
+/**
+ * The manager of SAsyncBatchWriteDispatcher. Call dispatcherAcquire(...) to get the SAsyncBatchWriteDispatcher
+ * instance. SDispatcherManager will manage the life cycle of SAsyncBatchWriteDispatcher.
+ */
+typedef struct SDispatcherManager {
+  pthread_key_t key;
+
+  // the maximum number of insertion rows in a batch.
+  int32_t batchSize;
+
+  // the batching timeout in milliseconds.
+  int32_t timeoutMs;
+
+  // specifies whether the dispatcher is thread local, if the dispatcher is not
+  // thread local, we will use the global dispatcher below.
+  bool isThreadLocal;
+
+  // the global dispatcher, if thread local enabled, global will be set to NULL.
+  SAsyncBatchWriteDispatcher* pGlobal;
+
+  // the client object.
+  STscObj* pClient;
+
+} SDispatcherManager;
+
+/**
+ * Control the timeout of the dispatcher queue.
+ */
+typedef struct SDispatcherTimeoutManager {
+  // the dispatcher that timeout manager belongs to.
+  SAsyncBatchWriteDispatcher* dispatcher;
+
+  // the background thread.
+  pthread_t background;
+
+  // the mutex to sleep the background thread.
+  pthread_mutex_t sleepMutex;
+
+  // the cond to signal to background thread.
+  pthread_cond_t timeout;
+
+  // the batching timeout in milliseconds.
+  int32_t timeoutMs;
+
+  // whether the timeout manager is shutdown.
+  bool shutdown;
+} SDispatcherTimeoutManager;
+
+/**
+ * A batch that polls from SAsyncBatchWriteDispatcher::buffer.
+ */
+typedef struct SBatchRequest {
+  size_t   nRequests;
+  SSqlObj* pRequests[];
+} SBatchRequest;
+
+/**
+ * Create the dispatcher timeout manager.
+ */
+SDispatcherTimeoutManager* createSDispatcherTimeoutManager(SAsyncBatchWriteDispatcher* dispatcher, int32_t timeoutMs);
+
+/**
+ * Destroy the dispatcher timeout manager.
+ */
+void destroySDispatcherTimeoutManager(SDispatcherTimeoutManager* manager);
+
+/**
+ * Check if the timeout manager is shutdown.
+ * @param manager   the timeout manager.
+ * @return          whether the timeout manager is shutdown.
+ */
+bool isShutdownSDispatcherTimeoutManager(SDispatcherTimeoutManager* manager);
+
+/**
+ * Shutdown the SDispatcherTimeoutManager.
+ * @param manager the SDispatcherTimeoutManager.
+ */
+void shutdownSDispatcherTimeoutManager(SDispatcherTimeoutManager* manager);
+
+/**
+ * Merge SSqlObjs into single SSqlObj.
+ *
+ * @param pRequest  the batch request.
+ * @param batch     the batch SSqlObj*.
+ * @return          the status code.
+ */
+int32_t dispatcherBatchBuilder(SBatchRequest* pRequest, SSqlObj** batch);
+
+/**
+ * Merge the sql statements and execute the merged sql statement asynchronously.
+ *
+ * @param pRequest the batch request. the request will be promised to free after calling this function.
+ */
+void dispatcherAsyncExecute(SBatchRequest* pRequest);
+
+/**
+ * Merge the sql statements and execute the merged sql statement.
+ *
+ * @param pRequest the batch request. you must call free(pRequest) after calling this function.
+ */
+void dispatcherExecute(SBatchRequest* pRequest);
+
+/**
+ * Create the async batch write dispatcher.
+ *
+ * @param pClient   the client object.
+ * @param batchSize When user submit an insert sql to `taos_query_a`, the SSqlObj* will be buffered instead of executing
+ * it. If the number of the buffered rows reach `batchSize`, all the SSqlObj* will be merged and sent to vnodes.
+ * @param timeout   The SSqlObj* will be sent to vnodes no more than `timeout` milliseconds. But the actual time
+ *                  vnodes received the SSqlObj* depends on the network quality.
+ */
+SAsyncBatchWriteDispatcher* createSAsyncBatchWriteDispatcher(STscObj* pClient, int32_t batchSize, int32_t timeoutMs);
+
+/**
+ * Destroy the async auto batch dispatcher.
+ */
+void destroySAsyncBatchWriteDispatcher(SAsyncBatchWriteDispatcher* dispatcher);
+
+/**
+ * Check if the current sql object can be dispatch by ABWD.
+ * 1. auto batch feature on the sql object must be enabled.
+ * 2. must be an `insert into ... value ...` statement.
+ * 3. the payload type must be kv payload.
+ * 4. no schema attached.
+ *
+ * @param dispatcher    the dispatcher.
+ * @param pSql          the sql object to check.
+ * @return              returns true if the sql object can be dispatch by ABWD.
+ */
+bool dispatcherCanDispatch(SAsyncBatchWriteDispatcher* dispatcher, SSqlObj* pSql);
+
+/**
+ * Try to offer the SSqlObj* to the dispatcher. If the number of row reach `batchSize`, the function
+ * will merge the SSqlObj* in the buffer and send them to the vnodes.
+ *
+ * @param pSql the insert statement to offer.
+ * @return     if offer success, returns true.
+ */
+bool dispatcherTryDispatch(SAsyncBatchWriteDispatcher* dispatcher, SSqlObj* pSql);
+
+/**
+ * Create the manager of SAsyncBatchWriteDispatcher.
+ *
+ * @param pClient       the client object.
+ * @param batchSize     the batchSize of SAsyncBatchWriteDispatcher.
+ * @param timeoutMs     the timeoutMs of SAsyncBatchWriteDispatcher.
+ * @param isThreadLocal specifies whether the dispatcher is thread local.
+ * @return the SAsyncBatchWriteDispatcher manager.
+ */
+SDispatcherManager* createDispatcherManager(STscObj* pClient, int32_t batchSize, int32_t timeoutMs, bool isThreadLocal);
+
+/**
+ * Destroy the SDispatcherManager.
+ * (will destroy all the instances of SAsyncBatchWriteDispatcher in the thread local variable)
+ *
+ * @param manager  the SDispatcherManager.
+ */
+void destroyDispatcherManager(SDispatcherManager* manager);
+
+/**
+ * Get an instance of SAsyncBatchWriteDispatcher.
+ *
+ * @param manager   the SDispatcherManager.
+ * @return          the SAsyncBatchWriteDispatcher instance.
+ */
+SAsyncBatchWriteDispatcher* dispatcherAcquire(SDispatcherManager* manager);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TSCBATCHWRITE_H

src/client/inc/tscUtil.h

@@ -144,6 +144,8 @@ void doRetrieveSubqueryData(SSchedMsg *pMsg);
 SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
                                    uint32_t offset);
 
+void destroySTableDataBlocksList(SArray* pDataBlocks);
+void destroySTableDataBlocks(STableDataBlocks* pDataBlocks);
 void*   tscDestroyBlockArrayList(SSqlObj* pSql, SArray* pDataBlockList);
 void*  tscDestroyUdfArrayList(SArray* pUdfList);
 void*   tscDestroyBlockHashTable(SSqlObj* pSql, SHashObj* pBlockHashTable, bool removeMeta);

src/client/inc/tsclient.h

@@ -45,6 +45,7 @@ typedef enum {
 
 // forward declaration
 struct SSqlInfo;
+typedef struct SDispatcherManager SDispatcherManager;
 
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
 typedef void (*_freeSqlSupporter)(void **);
@@ -256,7 +257,7 @@ typedef struct SInsertStatementParam {
 
   int32_t      batchSize;               // for parameter ('?') binding and batch processing
   int32_t      numOfParams;
-
+  int32_t      numOfRows;
   int32_t      numOfFiles;
 
   char         msg[512];                // error message
@@ -353,6 +354,7 @@ typedef struct STscObj {
   pthread_mutex_t    mutex;
   int32_t            numOfObj; // number of sqlObj from this tscObj
   
+  SDispatcherManager*dispatcherManager;
   SReqOrigin         from;
 } STscObj;
 
@@ -400,9 +402,10 @@ typedef struct SSqlObj {
   struct SSqlObj  *prev, *next;
   int64_t          self;
   
-  // connect alive
+
   int64_t          lastAlive;
   void *           pPrevContext;
+  bool             enableBatch;
 } SSqlObj;
 
 typedef struct SSqlStream {
@@ -503,7 +506,7 @@ void tscCloseTscObj(void *pObj);
 TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, TAOS **taos);
 TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res);
-TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param);
+TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param, bool enableBatch);
 // get taos connection unused session number
 int32_t taos_unused_session(TAOS* taos);
 

src/client/src/tscAsync.c

@@ -13,16 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+
 #include "os.h"
+#include "osAtomic.h"
+#include "tarray.h"
 #include "tutil.h"
 
+#include "qTableMeta.h"
 #include "tnote.h"
 #include "trpc.h"
+#include "tscBatchWrite.h"
 #include "tscLog.h"
 #include "tscSubquery.h"
 #include "tscUtil.h"
-#include "tsched.h"
-#include "qTableMeta.h"
 #include "tsclient.h"
 
 static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
@@ -334,6 +337,26 @@ bool appendTagsFilter(SSqlObj* pSql) {
     return false;
   }
 
+  // check tags is blank or ''
+  char* p1 = pTscObj->tags;
+  if (strcmp(p1, "\'\'") == 0) {
+    tscDebug("TAGS 0x%" PRIx64 " tags is empty. user=%s", pSql->self, pTscObj->user);
+    return false;
+  }
+  bool blank = true;
+  while(*p1 != 0) {
+    if(*p1 != ' ') {
+      blank = false;
+      break;
+    }
+    ++p1;
+  }
+  // result
+  if(blank) {
+    tscDebug("TAGS 0x%" PRIx64 " tags is all blank. user=%s", pSql->self, pTscObj->user);
+    return false;
+  }
+
   char * p = insertTags(pSql->sqlstr, pTscObj->tags);
   if(p == NULL) {
     return false;
@@ -378,7 +401,6 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
   pCmd->resColumnId = TSDB_RES_COL_ID;
 
   taosAcquireRef(tscObjRef, pSql->self);
-
   int32_t code = tsParseSql(pSql, true);
 
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
@@ -393,17 +415,26 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
     return;
   }
   
-  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
+  if (pObj->dispatcherManager != NULL) {
+    SAsyncBatchWriteDispatcher * dispatcher = dispatcherAcquire(pObj->dispatcherManager);
+    if (dispatcherTryDispatch(dispatcher, pSql)) {
+      taosReleaseRef(tscObjRef, pSql->self);
+      tscDebug("sql obj %p has been buffer in insert buffer", pSql);
+      return;
+    }
+  }
+  
+  SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
   executeQuery(pSql, pQueryInfo);
   taosReleaseRef(tscObjRef, pSql->self);
 }
 
 // TODO return the correct error code to client in tscQueueAsyncError
 void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
-  taos_query_ra(taos, sqlstr, fp, param);
+  taos_query_ra(taos, sqlstr, fp, param, tsWriteBatchSize > 0);
 }
 
-TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) {
+TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param, bool enableBatch) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
     tscError("pObj:%p is NULL or freed", pObj);
@@ -429,6 +460,8 @@ TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, v
     return NULL;
   }
   
+  pSql->enableBatch = enableBatch;
+  
   doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen);
 
   return pSql;

src/client/src/tscBatchMerge.c

+/*
+* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+*
+* This program is free software: you can use, redistribute, and/or modify
+* it under the terms of the GNU Affero General Public License, version 3
+* or later ("AGPL"), as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "tscBatchMerge.h"
+
+/**
+ * A util function to compare two SName.
+ */
+static int32_t compareSName(const void *left, const void *right) {
+  if (left == right) {
+    return 0;
+  }
+  SName* x = *(SName** ) left;
+  SName* y = *(SName** ) right;
+  return memcmp(x, y, sizeof(SName));
+}
+
+/**
+ * A util function to sort SArray<SMemRow> by key.
+ */
+static int32_t compareSMemRow(const void *x, const void *y) {
+  TSKEY left = memRowKey(*(void **) x);
+  TSKEY right = memRowKey(*(void **) y);
+  if (left == right) {
+    return 0;
+  } else {
+    return left > right ? 1 : -1;
+  }
+}
+
+/**
+ * If the SSubmitBlkBuilder of pBlock->uid is present, returns it. Otherwise, build a new SSubmitBlkBuilder.
+ * 
+ * @param builder the SSubmitMsgBlocksBuilder.
+ * @param pBlock  the SSubmitBlk.
+ * @return        the SSubmitBlkBuilder (NULL means failure).
+ */
+inline static SSubmitBlkBuilder* computeIfAbsentSSubmitBlkBuilder(SSubmitMsgBlocksBuilder* builder, SSubmitBlk* pBlock) {
+  SSubmitBlkBuilder** iter = taosHashGet(builder->blockBuilders, &pBlock->uid, sizeof(pBlock->uid));
+  SSubmitBlkBuilder* blocksBuilder = NULL;
+  if (iter) {
+    return *iter;
+  }
+  
+  blocksBuilder = createSSubmitBlkBuilder(pBlock);
+  if (!blocksBuilder) {
+    return NULL;
+  }
+    
+  if (taosHashPut(builder->blockBuilders, &pBlock->uid, sizeof(pBlock->uid), &blocksBuilder, sizeof(SArray*))) {
+    destroySSubmitBlkBuilder(blocksBuilder);
+    return NULL;
+  }
+  
+  return blocksBuilder;
+}
+
+SName** buildSTableNameListBuilder(STableNameListBuilder* builder, size_t* nTables) {
+  if (!taosArrayGetSize(builder->pTableNameList)) {
+    *nTables = 0;
+    return NULL;
+  }
+
+  // sort and unique.
+  taosArraySort(builder->pTableNameList, compareSName);
+  size_t tail = 0;
+  size_t nNames = taosArrayGetSize(builder->pTableNameList);
+  for (size_t i = 1; i < nNames; ++i) {
+    SName* last = taosArrayGetP(builder->pTableNameList, tail);
+    SName* current = taosArrayGetP(builder->pTableNameList, i);
+    if (memcmp(last, current, sizeof(SName)) != 0) {
+      ++tail;
+      taosArraySet(builder->pTableNameList, tail, &current);
+    }
+  }
+
+  // build table names list.
+  SName** tableNames = calloc(tail + 1, sizeof(SName*));
+  if (!tableNames) {
+    return NULL;
+  }
+  
+  // clone data.
+  for (size_t i = 0; i <= tail; ++i) {
+    SName* clone = malloc(sizeof(SName));
+    if (!clone) {
+      goto error;
+    }
+    memcpy(clone, taosArrayGetP(builder->pTableNameList, i), sizeof(SName));
+    tableNames[i] = clone;
+  }
+
+  *nTables = tail + 1;
+  return tableNames;
+
+error:
+  for (size_t i = 0; i <= tail; ++i) {
+    if (tableNames[i]) {
+      free(tableNames[i]);
+    }
+  }
+  free(tableNames);
+  return NULL;
+}
+
+SSubmitBlkBuilder* createSSubmitBlkBuilder(SSubmitBlk* metadata) {
+  SSubmitBlkBuilder* builder = calloc(1, sizeof(SSubmitBlkBuilder));
+  if (!builder) {
+    return NULL;
+  }
+
+  builder->rows = taosArrayInit(1, sizeof(SMemRow));
+  if (!builder->rows) {
+    free(builder);
+    return NULL;
+  }
+
+  builder->metadata = calloc(1, sizeof(SSubmitBlk));
+  if (!builder->metadata) {
+    taosArrayDestroy(&builder->rows);
+    free(builder);
+    return NULL;
+  }
+  memcpy(builder->metadata, metadata, sizeof(SSubmitBlk));
+
+  return builder;
+}
+
+void destroySSubmitBlkBuilder(SSubmitBlkBuilder* builder) {
+  if (!builder) {
+    return;
+  }
+  taosArrayDestroy(&builder->rows);
+  free(builder->metadata);
+  free(builder);
+}
+
+bool appendSSubmitBlkBuilder(SSubmitBlkBuilder* builder, SSubmitBlk* pBlock) {
+  assert(pBlock->uid == builder->metadata->uid);
+  assert(pBlock->schemaLen == 0);
+  
+  // shadow copy all the SMemRow to SSubmitBlkBuilder::rows.
+  char* pRow = pBlock->data;
+  char* pEnd = pBlock->data + htonl(pBlock->dataLen);
+  while (pRow < pEnd) {
+    if (!taosArrayPush(builder->rows, &pRow)) {
+      return false;
+    }
+    pRow += memRowTLen(pRow);
+  }
+
+  return true;
+}
+
+size_t writeSSubmitBlkBuilder(SSubmitBlkBuilder* builder, SSubmitBlk* target, size_t* nRows) {
+  memcpy(target, builder->metadata, sizeof(SSubmitBlk));
+
+  // sort SSubmitBlkBuilder::rows by timestamp.
+  uint32_t dataLen = 0;
+  taosArraySort(builder->rows, compareSMemRow);
+  
+  // deep copy all the SMemRow to target.
+  size_t nMemRows = taosArrayGetSize(builder->rows);
+  for (int i = 0; i < nMemRows; ++i) {
+    char* pRow = taosArrayGetP(builder->rows, i);
+    memcpy(POINTER_SHIFT(target->data, dataLen), pRow, memRowTLen(pRow));
+    dataLen += memRowTLen(pRow);
+  }
+
+  *nRows = nMemRows;
+
+  target->schemaLen = 0;
+  target->dataLen = (int32_t) htonl(dataLen);
+  target->numOfRows = (int16_t) htons(*nRows);
+
+  return dataLen + sizeof(SSubmitBlk);
+}
+
+size_t nWriteSSubmitBlkBuilder(SSubmitBlkBuilder* builder) {
+  size_t dataLen = 0;
+  size_t nRows = taosArrayGetSize(builder->rows);
+  for (int i = 0; i < nRows; ++i) {
+    char* pRow = taosArrayGetP(builder->rows, i);
+    dataLen += memRowTLen(pRow);
+  }
+  return dataLen + sizeof(SSubmitBlk);
+}
+
+SSubmitMsgBlocksBuilder* createSSubmitMsgBuilder(int64_t vgId) {
+  SSubmitMsgBlocksBuilder* builder = calloc(1, sizeof(SSubmitMsgBlocksBuilder));
+  if (!builder) {
+    return NULL;
+  }
+  builder->vgId = vgId;
+
+  builder->blockBuilders = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  if (!builder->blockBuilders) {
+    free(builder);
+    return NULL;
+  }
+  return builder;
+}
+
+size_t nWriteSSubmitMsgBuilder(SSubmitMsgBlocksBuilder* builder) {
+  size_t nWrite = 0;
+  SSubmitBlkBuilder** iter = taosHashIterate(builder->blockBuilders, NULL);
+  while (iter) {
+    SSubmitBlkBuilder* blocksBuilder = *iter;
+    nWrite += nWriteSSubmitBlkBuilder(blocksBuilder);
+    iter = taosHashIterate(builder->blockBuilders, iter);
+  }
+  return nWrite;
+}
+
+size_t writeSSubmitMsgBlocksBuilder(SSubmitMsgBlocksBuilder* builder, SSubmitBlk* pBlocks, size_t* nRows) {
+  size_t nWrite = 0;
+  SSubmitBlkBuilder** iter = taosHashIterate(builder->blockBuilders, NULL);
+  
+  // copy all the SSubmitBlk to pBlocks.
+  while (iter) {
+    size_t nSubRows = 0;
+    SSubmitBlkBuilder* blocksBuilder = *iter;
+    SSubmitBlk* pBlock = POINTER_SHIFT(pBlocks, nWrite);
+    nWrite += writeSSubmitBlkBuilder(blocksBuilder, pBlock, &nSubRows);
+    *nRows += nSubRows;
+    iter = taosHashIterate(builder->blockBuilders, iter);
+  }
+  return nWrite;
+}
+
+size_t nBlockSSubmitMsgBlocksBuilder(SSubmitMsgBlocksBuilder* builder) {
+  return taosHashGetSize(builder->blockBuilders);
+}
+
+void destroySSubmitMsgBuilder(SSubmitMsgBlocksBuilder* builder) {
+  if (!builder) {
+    return;
+  }
+
+  SSubmitBlkBuilder** iter = taosHashIterate(builder->blockBuilders, NULL);
+  while (iter) {
+    destroySSubmitBlkBuilder(*iter);
+    iter = taosHashIterate(builder->blockBuilders, iter);
+  }
+  taosHashCleanup(builder->blockBuilders);
+  free(builder);
+}
+
+bool appendSSubmitMsgBlocks(SSubmitMsgBlocksBuilder* builder, SSubmitBlk* pBlocks, size_t nBlocks) {
+  SSubmitBlk* pBlock = pBlocks;
+  for (size_t i = 0; i < nBlocks; ++i) {
+    // not support SSubmitBlk with schema.
+    assert(pBlock->schemaLen == 0);
+    
+    // get the builder of specific table (by pBlock->uid).
+    SSubmitBlkBuilder* blocksBuilder = computeIfAbsentSSubmitBlkBuilder(builder, pBlock);
+    if (!blocksBuilder) {
+      return false;
+    }
+
+    if (!appendSSubmitBlkBuilder(blocksBuilder, pBlock)) {
+      return false;
+    }
+    
+    // go to next block.
+    size_t blockSize = sizeof (SSubmitBlk) + htonl(pBlock->dataLen);
+    pBlock = POINTER_SHIFT(pBlock, blockSize);
+  }
+  return true;
+}
+
+STableDataBlocksBuilder* createSTableDataBlocksBuilder(int64_t vgId) {
+  STableDataBlocksBuilder* builder = calloc(1, sizeof(STableDataBlocksBuilder));
+  if (!builder) {
+    return NULL;
+  }
+
+  builder->blocksBuilder = createSSubmitMsgBuilder(vgId);
+  if (!builder->blocksBuilder) {
+    free(builder);
+    return NULL;
+  }
+
+  builder->vgId = vgId;
+  builder->firstBlock = NULL;
+  return builder;
+}
+
+void destroySTableDataBlocksBuilder(STableDataBlocksBuilder* builder) {
+  if (!builder) {
+    return;
+  }
+
+  destroySSubmitMsgBuilder(builder->blocksBuilder);
+  free(builder);
+}
+
+bool appendSTableDataBlocksBuilder(STableDataBlocksBuilder* builder, STableDataBlocks* dataBlocks) {
+  // the data blocks vgId must be same with builder vgId.
+  if (!dataBlocks || dataBlocks->vgId != builder->vgId) {
+    return false;
+  }
+
+  if (!builder->firstBlock) {
+    builder->firstBlock = dataBlocks;
+  }
+
+  SSubmitBlk* pBlocks = (SSubmitBlk *)(dataBlocks->pData + dataBlocks->headerSize);
+  return appendSSubmitMsgBlocks(builder->blocksBuilder, pBlocks, dataBlocks->numOfTables);
+}
+
+STableDataBlocks* buildSTableDataBlocksBuilder(STableDataBlocksBuilder* builder, size_t* nRows) {
+  SSubmitMsgBlocksBuilder* blocksBuilder = builder->blocksBuilder;
+  STableDataBlocks *firstBlock = builder->firstBlock;
+  if (!firstBlock) {
+    return NULL;
+  }
+
+  size_t nWriteSize = nWriteSSubmitMsgBuilder(builder->blocksBuilder);
+  size_t nHeaderSize = firstBlock->headerSize;
+  size_t nAllocSize = nWriteSize + nHeaderSize;
+  
+  // allocate data blocks.
+  STableDataBlocks* dataBlocks = NULL;
+  int32_t code = tscCreateDataBlock(nAllocSize, 0, (int32_t) nHeaderSize, &firstBlock->tableName, firstBlock->pTableMeta, &dataBlocks);
+  if (code != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+  
+  // build the header (using first block).
+  dataBlocks->size = nHeaderSize;
+  memcpy(dataBlocks->pData, firstBlock->pData, nHeaderSize);
+  
+  // build the SSubmitMsg::blocks.
+  dataBlocks->size += writeSSubmitMsgBlocksBuilder(blocksBuilder, (SSubmitBlk *) (dataBlocks->pData + nHeaderSize), nRows);
+  dataBlocks->numOfTables = (int32_t) nBlockSSubmitMsgBlocksBuilder(blocksBuilder);
+  return dataBlocks;
+}
+
+STableDataBlocksListBuilder* createSTableDataBlocksListBuilder() {
+  STableDataBlocksListBuilder* builder = calloc(1, sizeof(STableDataBlocksListBuilder));
+  if (!builder) {
+    return NULL;
+  }
+
+  builder->dataBlocksBuilders = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  if (!builder->dataBlocksBuilders) {
+    free(builder);
+    return NULL;
+  }
+
+  return builder;
+}
+
+void destroySTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder) {
+  if (!builder) {
+    return;
+  }
+  
+  STableDataBlocksBuilder** iter = taosHashIterate(builder->dataBlocksBuilders, NULL);
+  while (iter) {
+    destroySTableDataBlocksBuilder(*iter);
+    iter = taosHashIterate(builder->dataBlocksBuilders, iter);
+  }
+  taosHashCleanup(builder->dataBlocksBuilders);
+  free(builder);
+}
+
+bool appendSTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder, STableDataBlocks* dataBlocks) {
+  // get the data blocks builder of specific vgId.
+  STableDataBlocksBuilder** item = taosHashGet(builder->dataBlocksBuilders, &dataBlocks->vgId, sizeof(dataBlocks->vgId));
+  STableDataBlocksBuilder* blocksBuilder = NULL;
+  if (item) {
+    blocksBuilder = *item;
+  } else {
+    blocksBuilder = createSTableDataBlocksBuilder(dataBlocks->vgId);
+    if (!blocksBuilder) {
+      return false;
+    }
+
+    if (taosHashPut(builder->dataBlocksBuilders, &dataBlocks->vgId, sizeof(dataBlocks->vgId), &blocksBuilder, sizeof(STableDataBlocksBuilder*))) {
+      destroySTableDataBlocksBuilder(blocksBuilder);
+      return false;
+    }
+  }
+  
+  // append to this builder.
+  return appendSTableDataBlocksBuilder(blocksBuilder, dataBlocks);
+}
+
+SArray* buildSTableDataBlocksListBuilder(STableDataBlocksListBuilder* builder, size_t* nTables, size_t* nRows) {
+  SArray* pVnodeDataBlockList = taosArrayInit(taosHashGetSize(builder->dataBlocksBuilders), sizeof(STableDataBlocks*));
+  if (!pVnodeDataBlockList) {
+    return NULL;
+  }
+  
+  // build data blocks of each vgId.
+  STableDataBlocksBuilder** iter = taosHashIterate(builder->dataBlocksBuilders, NULL);
+  while (iter) {
+    size_t nSubRows = 0;
+    STableDataBlocksBuilder* dataBlocksBuilder = *iter;
+    STableDataBlocks* dataBlocks = buildSTableDataBlocksBuilder(dataBlocksBuilder, &nSubRows);
+    if (!dataBlocks) {
+      goto error;
+    }
+    *nTables += dataBlocks->numOfTables;
+    *nRows += nSubRows;
+
+    taosArrayPush(pVnodeDataBlockList, &dataBlocks);
+    iter = taosHashIterate(builder->dataBlocksBuilders, iter);
+  }
+  return pVnodeDataBlockList;
+
+error:
+  for (int i = 0; i < taosArrayGetSize(pVnodeDataBlockList); ++i) {
+    STableDataBlocks* dataBlocks = taosArrayGetP(pVnodeDataBlockList, i);
+    tscDestroyDataBlock(NULL, dataBlocks, false);
+  }
+  taosArrayDestroy(&pVnodeDataBlockList);
+  return NULL;
+}
+
+STableNameListBuilder* createSTableNameListBuilder() {
+  STableNameListBuilder* builder = calloc(1, sizeof(STableNameListBuilder));
+  if (!builder) {
+    return NULL;
+  }
+
+  builder->pTableNameList = taosArrayInit(1, sizeof(SName*));
+  if (!builder->pTableNameList) {
+    free(builder);
+    return NULL;
+  }
+
+  return builder;
+}
+
+void destroySTableNameListBuilder(STableNameListBuilder* builder) {
+  if (!builder) {
+    return;
+  }
+  taosArrayDestroy(&builder->pTableNameList);
+  free(builder);
+}
+
+bool insertSTableNameListBuilder(STableNameListBuilder* builder, SName* name) {
+  return taosArrayPush(builder->pTableNameList, &name);
+}
+
+int32_t tscMergeSSqlObjs(SSqlObj** polls, size_t nPolls, SSqlObj* result) {
+  // statement array is empty.
+  if (!polls || !nPolls) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  STableDataBlocksListBuilder* builder = createSTableDataBlocksListBuilder();
+  if (!builder) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  STableNameListBuilder* nameListBuilder = createSTableNameListBuilder();
+  if (!nameListBuilder) {
+    destroySTableDataBlocksListBuilder(builder);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  // append the existing data blocks to builder.
+  for (size_t i = 0; i < nPolls; ++i) {
+    SSqlObj *pSql = polls[i];
+    SInsertStatementParam* pInsertParam = &pSql->cmd.insertParam;
+    if (!pInsertParam->pDataBlocks) {
+      continue;
+    }
+
+    assert(pInsertParam->payloadType == PAYLOAD_TYPE_KV);
+    assert(!pInsertParam->schemaAttached);
+
+    // append each vnode data block to the builder.
+    size_t nBlocks = taosArrayGetSize(pInsertParam->pDataBlocks);
+    for (size_t j = 0; j < nBlocks; ++j) {
+      STableDataBlocks* tableBlock = taosArrayGetP(pInsertParam->pDataBlocks, j);
+      if (!appendSTableDataBlocksListBuilder(builder, tableBlock)) {
+        destroySTableDataBlocksListBuilder(builder);
+        destroySTableNameListBuilder(nameListBuilder);
+        return TSDB_CODE_TSC_OUT_OF_MEMORY;
+      }
+
+      for (int k = 0; k < pInsertParam->numOfTables; ++k) {
+        if (!insertSTableNameListBuilder(nameListBuilder, pInsertParam->pTableNameList[k])) {
+          destroySTableDataBlocksListBuilder(builder);
+          destroySTableNameListBuilder(nameListBuilder);
+          return TSDB_CODE_TSC_OUT_OF_MEMORY;
+        }
+      }
+    }
+  }
+
+  // build the vnode data blocks.
+  size_t nBlocks = 0;
+  size_t nRows = 0;
+  SInsertStatementParam* pInsertParam = &result->cmd.insertParam;
+  SArray* pVnodeDataBlocksList = buildSTableDataBlocksListBuilder(builder, &nBlocks, &nRows);
+  if (!pVnodeDataBlocksList) {
+    destroySTableDataBlocksListBuilder(builder);
+    destroySTableNameListBuilder(nameListBuilder);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  // build the table name list.
+  size_t nTables = 0;
+  SName** pTableNameList = buildSTableNameListBuilder(nameListBuilder, &nTables);
+  if (!pTableNameList) {
+    destroySTableDataBlocksListBuilder(builder);
+    destroySTableNameListBuilder(nameListBuilder);
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  
+  if (nTables != nBlocks) {
+    destroySTableDataBlocksListBuilder(builder);
+    destroySTableNameListBuilder(nameListBuilder);
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  // replace table name list.
+  if (pInsertParam->pTableNameList) {
+    destroyTableNameList(pInsertParam);
+  }
+  pInsertParam->pTableNameList = pTableNameList;
+  pInsertParam->numOfTables = (int32_t) nTables;
+
+  // replace vnode data blocks.
+  if (pInsertParam->pDataBlocks) {
+    tscDestroyBlockArrayList(result, pInsertParam->pDataBlocks);
+  }
+  pInsertParam->pDataBlocks = pVnodeDataBlocksList;
+  pInsertParam->numOfRows = (int32_t) nRows;
+
+  // clean up.
+  destroySTableDataBlocksListBuilder(builder);
+  destroySTableNameListBuilder(nameListBuilder);
+  return TSDB_CODE_SUCCESS;
+}

src/client/src/tscBatchWrite.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "osAtomic.h"
+
+#include "tscBatchMerge.h"
+#include "tscBatchWrite.h"
+#include "tscLog.h"
+#include "tscSubquery.h"
+#include "tsclient.h"
+
+/**
+ * Represents the callback function and its context.
+ */
+typedef struct {
+  __async_cb_func_t fp;
+  void*             param;
+} SCallbackHandler;
+
+/**
+ * The context of `batchResultCallback`.
+ */
+typedef struct {
+  size_t           nHandlers;
+  SCallbackHandler handler[];
+} SBatchCallbackContext;
+
+/**
+ * Get the number of insertion row in the sql statement.
+ *
+ * @param pSql      the sql statement.
+ * @return int32_t  the number of insertion row.
+ */
+inline static int32_t statementGetInsertionRows(SSqlObj* pSql) { return pSql->cmd.insertParam.numOfRows; }
+
+/**
+ * Return the error result to the callback function, and release the sql object.
+ *
+ * @param pSql  the sql object.
+ * @param code  the error code of the error result.
+ */
+inline static void tscReturnsError(SSqlObj* pSql, int code) {
+  if (pSql == NULL) {
+    return;
+  }
+
+  pSql->res.code = code;
+  tscAsyncResultOnError(pSql);
+}
+
+/**
+ * Proxy function to perform sequentially insert operation.
+ *
+ * @param param     the context of `batchResultCallback`.
+ * @param tres      the result object.
+ * @param code      the error code.
+ */
+static void batchResultCallback(void* param, TAOS_RES* tres, int32_t code) {
+  SBatchCallbackContext* context = param;
+  SSqlObj*               res = tres;
+
+  // handle corner case [context == null].
+  if (context == NULL) {
+    tscError("context in `batchResultCallback` is null, which should not happen");
+    if (tres) {
+      taosReleaseRef(tscObjRef, res->self);
+    }
+    return;
+  }
+
+  // handle corner case [res == null].
+  if (res == NULL) {
+    tscError("tres in `batchResultCallback` is null, which should not happen");
+    free(context);
+    return;
+  }
+
+  // handle results.
+  tscDebug("async batch result callback, number of item: %zu", context->nHandlers);
+  for (int i = 0; i < context->nHandlers; ++i) {
+    // the result object is shared by many sql objects.
+    // therefore, we need to increase the ref count.
+    taosAcquireRef(tscObjRef, res->self);
+
+    SCallbackHandler* handler = &context->handler[i];
+    handler->fp(handler->param, res, code);
+  }
+
+  taosReleaseRef(tscObjRef, res->self);
+  free(context);
+}
+
+int32_t dispatcherBatchBuilder(SBatchRequest* pRequest, SSqlObj** batch) {
+  assert(pRequest);
+  assert(pRequest->pRequests);
+  assert(pRequest->nRequests);
+
+  // create the callback context.
+  SBatchCallbackContext* context =
+      calloc(1, sizeof(SBatchCallbackContext) + pRequest->nRequests * sizeof(SCallbackHandler));
+  if (context == NULL) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  tscDebug("create batch call back context: %p", context);
+
+  // initialize the callback context.
+  context->nHandlers = pRequest->nRequests;
+  for (size_t i = 0; i < pRequest->nRequests; ++i) {
+    SSqlObj* pSql = pRequest->pRequests[i];
+    context->handler[i].fp = pSql->fp;
+    context->handler[i].param = pSql->param;
+  }
+
+  // merge the statements into single one.
+  tscDebug("start to merge %zu sql objs", pRequest->nRequests);
+  SSqlObj* pFirst = pRequest->pRequests[0];
+  int32_t  code = tscMergeSSqlObjs(pRequest->pRequests, pRequest->nRequests, pFirst);
+  if (code != TSDB_CODE_SUCCESS) {
+    const char* msg = tstrerror(code);
+    tscDebug("failed to merge sql objects: %s", msg);
+    free(context);
+    return code;
+  }
+
+  pFirst->fp = batchResultCallback;
+  pFirst->param = context;
+  pFirst->fetchFp = pFirst->fp;
+  taosAcquireRef(tscObjRef, pFirst->self);
+  *batch = pFirst;
+
+  for (int i = 0; i < pRequest->nRequests; ++i) {
+    SSqlObj* pSql = pRequest->pRequests[i];
+    taosReleaseRef(tscObjRef, pSql->self);
+  }
+  return code;
+}
+
+/**
+ * Poll all the SSqlObj* in the dispatcher's buffer (No Lock). After call this function,
+ * you need to notify dispatcher->notFull by yourself.
+ *
+ * @param dispatcher    the dispatcher.
+ * @param nPolls        the number of polled SSqlObj*.
+ * @return              all the SSqlObj* in the buffer.
+ */
+inline static SBatchRequest* dispatcherPollAll(SAsyncBatchWriteDispatcher* dispatcher) {
+  if (!dispatcher->bufferSize) {
+    return NULL;
+  }
+
+  SBatchRequest* pRequest = malloc(sizeof(SBatchRequest) + sizeof(SSqlObj*) * dispatcher->bufferSize);
+  if (pRequest == NULL) {
+    tscError("failed to poll all items: out of memory");
+    return NULL;
+  }
+  
+  memcpy(pRequest->pRequests, dispatcher->buffer, sizeof(SSqlObj*) * dispatcher->bufferSize);
+  pRequest->nRequests = dispatcher->bufferSize;
+  dispatcher->currentSize = 0;
+  dispatcher->bufferSize = 0;
+  return pRequest;
+}
+
+/**
+ * Poll all the SSqlObj* in the dispatcher's buffer.
+ *
+ * @param dispatcher    the dispatcher.
+ * @return              all the SSqlObj* in the buffer.
+ */
+inline static SBatchRequest* dispatcherLockPollAll(SAsyncBatchWriteDispatcher* dispatcher) {
+  SBatchRequest* pRequest = NULL;
+  pthread_mutex_lock(&dispatcher->bufferMutex);
+  pRequest = dispatcherPollAll(dispatcher);
+  pthread_cond_broadcast(&dispatcher->notFull);
+  pthread_mutex_unlock(&dispatcher->bufferMutex);
+  return pRequest;
+}
+
+/**
+ * @brief Try to offer the SSqlObj* to the dispatcher.
+ *
+ * @param dispatcher  the async bulk write dispatcher.
+ * @param pSql        the sql object to offer.
+ * @return            return whether offer success.
+ */
+inline static bool dispatcherTryOffer(SAsyncBatchWriteDispatcher* dispatcher, SSqlObj* pSql) {
+  pthread_mutex_lock(&dispatcher->bufferMutex);
+
+  // if dispatcher is shutdown, must fail back to normal insertion.
+  // usually not happen, unless taos_query_a(...) after taos_close(...).
+  if (atomic_load_8(&dispatcher->shutdown)) {
+    pthread_mutex_unlock(&dispatcher->bufferMutex);
+    return false;
+  }
+
+  // the buffer is full.
+  while (dispatcher->currentSize >= dispatcher->batchSize) {
+    if (pthread_cond_wait(&dispatcher->notFull, &dispatcher->bufferMutex)) {
+      pthread_mutex_unlock(&dispatcher->bufferMutex);
+      return false;
+    }
+  }
+
+  dispatcher->buffer[dispatcher->bufferSize++] = pSql;
+  dispatcher->currentSize += statementGetInsertionRows(pSql);
+  tscDebug("sql obj %p has been write to insert buffer", pSql);
+
+  if (dispatcher->currentSize < dispatcher->batchSize) {
+    pthread_mutex_unlock(&dispatcher->bufferMutex);
+    return true;
+  }
+
+  // the dispatcher reaches batch size.
+  SBatchRequest* pRequest = dispatcherPollAll(dispatcher);
+  pthread_cond_broadcast(&dispatcher->notFull);
+  pthread_mutex_unlock(&dispatcher->bufferMutex);
+
+  if (pRequest) {
+    dispatcherAsyncExecute(pRequest);
+  }
+  return true;
+}
+
+void dispatcherExecute(SBatchRequest* pRequest) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  // no item in the buffer (items has been taken by other threads).
+  if (!pRequest) {
+    return;
+  }
+
+  assert(pRequest->pRequests);
+  assert(pRequest->nRequests);
+
+  // merge the statements into single one.
+  SSqlObj* pSql = NULL;
+  code = dispatcherBatchBuilder(pRequest, &pSql);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  tscDebug("merging %zu sql objs into %p", pRequest->nRequests, pSql);
+  tscHandleMultivnodeInsert(pSql);
+  return;
+_error:
+  tscError("send async batch sql obj failed, reason: %s", tstrerror(code));
+
+  // handling the failures.
+  for (size_t i = 0; i < pRequest->nRequests; ++i) {
+    SSqlObj* item = pRequest->pRequests[i];
+    tscReturnsError(item, code);
+  }
+}
+
+/**
+ * Get the timespec after `millis` ms
+ *
+ * @param t         the timespec.
+ * @param millis    the duration in milliseconds.
+ * @return          the timespec after `millis` ms.
+ */
+static inline void afterMillis(struct timespec* t, int32_t millis) {
+  t->tv_nsec += millis * 1000000L;
+  t->tv_sec += t->tv_nsec / 1000000000L;
+  t->tv_nsec %= 1000000000L;
+}
+
+/**
+ * Sleep until `timeout` timespec. When dispatcherShutdown(...) called, the function will return immediately.
+ *
+ * @param dispatcher the dispatcher thread to sleep.
+ * @param timeout    the timeout in CLOCK_REALTIME.
+ */
+inline static void timeoutManagerSleepUntil(SDispatcherTimeoutManager* manager, struct timespec* timeout) {
+  pthread_mutex_lock(&manager->sleepMutex);
+  while (true) {
+    // notified by dispatcherShutdown(...).
+    if (isShutdownSDispatcherTimeoutManager(manager)) {
+      break;
+    }
+    if (pthread_cond_timedwait(&manager->timeout, &manager->sleepMutex, timeout)) {
+      fflush(stdout);
+      break;
+    }
+  }
+  pthread_mutex_unlock(&manager->sleepMutex);
+}
+
+/**
+ * The thread to manage batching timeout.
+ */
+static void* timeoutManagerCallback(void* arg) {
+  SDispatcherTimeoutManager* manager = arg;
+  setThreadName("tscAsyncBackground");
+
+  while (!isShutdownSDispatcherTimeoutManager(manager)) {
+    struct timespec timeout;
+    clock_gettime(CLOCK_REALTIME, &timeout);
+    afterMillis(&timeout, manager->timeoutMs);
+
+    SBatchRequest* pRequest = dispatcherLockPollAll(manager->dispatcher);
+    if (pRequest) {
+      dispatcherAsyncExecute(pRequest);
+    }
+
+    // Similar to scheduleAtFixedRate in Java, if the execution time exceed
+    // `timeoutMs` milliseconds, then there will be no sleep.
+    timeoutManagerSleepUntil(manager, &timeout);
+  }
+  return NULL;
+}
+
+SAsyncBatchWriteDispatcher* createSAsyncBatchWriteDispatcher(STscObj* pClient, int32_t batchSize, int32_t timeoutMs) {
+  SAsyncBatchWriteDispatcher* dispatcher = calloc(1, sizeof(SAsyncBatchWriteDispatcher) + batchSize * sizeof(SSqlObj*));
+  if (!dispatcher) {
+    return NULL;
+  }
+
+  assert(pClient != NULL);
+
+  dispatcher->pClient = pClient;
+  dispatcher->currentSize = 0;
+  dispatcher->bufferSize = 0;
+  dispatcher->batchSize = batchSize;
+  atomic_store_8(&dispatcher->shutdown, false);
+
+  // init the mutex and the cond.
+  pthread_mutex_init(&dispatcher->bufferMutex, NULL);
+  pthread_cond_init(&dispatcher->notFull, NULL);
+
+  // init timeout manager.
+  dispatcher->timeoutManager = createSDispatcherTimeoutManager(dispatcher, timeoutMs);
+  if (!dispatcher->timeoutManager) {
+    pthread_mutex_destroy(&dispatcher->bufferMutex);
+    pthread_cond_destroy(&dispatcher->notFull);
+    free(dispatcher);
+    return NULL;
+  }
+
+  return dispatcher;
+}
+
+/**
+ * Shutdown the dispatcher and join the timeout thread.
+ *
+ * @param dispatcher the dispatcher.
+ */
+inline static void dispatcherShutdown(SAsyncBatchWriteDispatcher* dispatcher) {
+  atomic_store_8(&dispatcher->shutdown, true);
+  if (dispatcher->timeoutManager) {
+    shutdownSDispatcherTimeoutManager(dispatcher->timeoutManager);
+  }
+}
+
+void destroySAsyncBatchWriteDispatcher(SAsyncBatchWriteDispatcher* dispatcher) {
+  if (dispatcher == NULL) {
+    return;
+  }
+
+  dispatcherShutdown(dispatcher);
+
+  // poll and send all the statements in the buffer.
+  while (true) {
+    SBatchRequest* pRequest = dispatcherLockPollAll(dispatcher);
+    if (!pRequest) {
+      break;
+    }
+    dispatcherExecute(pRequest);
+    free(pRequest);
+  }
+  // destroy the timeout manager.
+  destroySDispatcherTimeoutManager(dispatcher->timeoutManager);
+
+  // destroy the mutex.
+  pthread_mutex_destroy(&dispatcher->bufferMutex);
+  pthread_cond_destroy(&dispatcher->notFull);
+
+  free(dispatcher);
+}
+
+bool dispatcherCanDispatch(SAsyncBatchWriteDispatcher* dispatcher, SSqlObj* pSql) {
+  if (pSql == NULL || !pSql->enableBatch) {
+    return false;
+  }
+
+  SSqlCmd*    pCmd = &pSql->cmd;
+  SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
+
+  // only support insert statement.
+  if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT)) {
+    return false;
+  }
+
+  SInsertStatementParam* pInsertParam = &pCmd->insertParam;
+
+  // file insert not support.
+  if (TSDB_QUERY_HAS_TYPE(pInsertParam->insertType, TSDB_QUERY_TYPE_FILE_INSERT)) {
+    return false;
+  }
+
+  // only support kv payload.
+  if (pInsertParam->payloadType != PAYLOAD_TYPE_KV) {
+    return false;
+  }
+
+  // no schema attached.
+  if (pInsertParam->schemaAttached) {
+    return false;
+  }
+
+  // too many insertion rows, fail back to normal insertion.
+  if (statementGetInsertionRows(pSql) >= dispatcher->batchSize) {
+    return false;
+  }
+
+  return true;
+}
+
+bool dispatcherTryDispatch(SAsyncBatchWriteDispatcher* dispatcher, SSqlObj* pSql) {
+  if (atomic_load_8(&dispatcher->shutdown)) {
+    return false;
+  }
+
+  // the sql object doesn't support bulk insertion.
+  if (!dispatcherCanDispatch(dispatcher, pSql)) {
+    return false;
+  }
+
+  // try to offer pSql to the buffer.
+  return dispatcherTryOffer(dispatcher, pSql);
+}
+
+/**
+ * Destroy the SAsyncBatchWriteDispatcher create by SDispatcherManager.
+ * @param arg the thread local SAsyncBatchWriteDispatcher.
+ */
+static void destroyDispatcher(void* arg) {
+  SAsyncBatchWriteDispatcher* dispatcher = arg;
+  if (!dispatcher) {
+    return;
+  }
+
+  destroySAsyncBatchWriteDispatcher(dispatcher);
+}
+
+SDispatcherManager* createDispatcherManager(STscObj* pClient, int32_t batchSize, int32_t timeoutMs,
+                                            bool isThreadLocal) {
+  SDispatcherManager* dispatcher = calloc(1, sizeof(SDispatcherManager));
+  if (!dispatcher) {
+    return NULL;
+  }
+
+  assert(pClient != NULL);
+
+  dispatcher->pClient = pClient;
+  dispatcher->batchSize = batchSize;
+  dispatcher->timeoutMs = timeoutMs;
+  dispatcher->isThreadLocal = isThreadLocal;
+
+  if (isThreadLocal) {
+    if (pthread_key_create(&dispatcher->key, destroyDispatcher)) {
+      free(dispatcher);
+      return NULL;
+    }
+  } else {
+    dispatcher->pGlobal = createSAsyncBatchWriteDispatcher(pClient, batchSize, timeoutMs);
+    if (!dispatcher->pGlobal) {
+      free(dispatcher);
+      return NULL;
+    }
+  }
+  return dispatcher;
+}
+
+SAsyncBatchWriteDispatcher* dispatcherAcquire(SDispatcherManager* manager) {
+  if (!manager->isThreadLocal) {
+    return manager->pGlobal;
+  }
+
+  SAsyncBatchWriteDispatcher* value = pthread_getspecific(manager->key);
+  if (value) {
+    return value;
+  }
+
+  value = createSAsyncBatchWriteDispatcher(manager->pClient, manager->batchSize, manager->timeoutMs);
+  if (value) {
+    pthread_setspecific(manager->key, value);
+    return value;
+  }
+
+  return NULL;
+}
+
+void destroyDispatcherManager(SDispatcherManager* manager) {
+  if (manager) {
+    if (manager->isThreadLocal) {
+      pthread_key_delete(manager->key);
+    }
+
+    if (manager->pGlobal) {
+      destroySAsyncBatchWriteDispatcher(manager->pGlobal);
+    }
+    free(manager);
+  }
+}
+
+SDispatcherTimeoutManager* createSDispatcherTimeoutManager(SAsyncBatchWriteDispatcher* dispatcher, int32_t timeoutMs) {
+  SDispatcherTimeoutManager* manager = calloc(1, sizeof(SDispatcherTimeoutManager));
+  if (!manager) {
+    return NULL;
+  }
+
+  manager->timeoutMs = timeoutMs;
+  manager->dispatcher = dispatcher;
+  atomic_store_8(&manager->shutdown, false);
+
+  pthread_mutex_init(&manager->sleepMutex, NULL);
+  pthread_cond_init(&manager->timeout, NULL);
+
+  // init background thread.
+  if (pthread_create(&manager->background, NULL, timeoutManagerCallback, manager)) {
+    pthread_mutex_destroy(&manager->sleepMutex);
+    pthread_cond_destroy(&manager->timeout);
+    free(manager);
+    return NULL;
+  }
+  return manager;
+}
+
+void destroySDispatcherTimeoutManager(SDispatcherTimeoutManager* manager) {
+  if (!manager) {
+    return;
+  }
+
+  shutdownSDispatcherTimeoutManager(manager);
+  manager->dispatcher->timeoutManager = NULL;
+
+  pthread_mutex_destroy(&manager->sleepMutex);
+  pthread_cond_destroy(&manager->timeout);
+  free(manager);
+}
+
+void shutdownSDispatcherTimeoutManager(SDispatcherTimeoutManager* manager) {
+  // mark shutdown, signal shutdown to timeout thread.
+  pthread_mutex_lock(&manager->sleepMutex);
+  atomic_store_8(&manager->shutdown, true);
+  pthread_cond_broadcast(&manager->timeout);
+  pthread_mutex_unlock(&manager->sleepMutex);
+
+  // make sure the timeout thread exit.
+  pthread_join(manager->background, NULL);
+}
+
+bool isShutdownSDispatcherTimeoutManager(SDispatcherTimeoutManager* manager) {
+  if (!manager) {
+    return true;
+  }
+  return atomic_load_8(&manager->shutdown);
+}
+
+/**
+ * The proxy function to call `dispatcherExecute`.
+ *
+ * @param pMsg the schedule message.
+ */
+static void dispatcherExecuteProxy(struct SSchedMsg* pMsg) {
+  SBatchRequest* pRequest = pMsg->ahandle;
+  if (!pRequest) {
+    return;
+  }
+  
+  pMsg->ahandle = NULL;
+  dispatcherExecute(pRequest);
+  free(pRequest);
+}
+
+void dispatcherAsyncExecute(SBatchRequest* pRequest) {
+  if (!pRequest) {
+    return;
+  }
+
+  assert(pRequest->pRequests);
+  assert(pRequest->nRequests);
+
+  SSchedMsg schedMsg = {0};
+  schedMsg.fp = dispatcherExecuteProxy;
+  schedMsg.ahandle = (void*) pRequest;
+  schedMsg.thandle = (void*) 1;
+  schedMsg.msg = 0;
+  taosScheduleTask(tscQhandle, &schedMsg);
+}

src/client/src/tscParseInsert.c

@@ -1577,6 +1577,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
     }
   }
 
+  pCmd->insertParam.numOfRows = totalNum;
   code = TSDB_CODE_SUCCESS;
   goto _clean;
 

src/client/src/tscParseLineProtocol.c

@@ -757,25 +757,25 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu
                                              SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) {
   for (int32_t i = 0; i < numPoints; ++i) {
     TAOS_SML_DATA_POINT * point = points + i;
-    SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx);
-
-    for (int j = 0; j < point->tagNum; ++j) {
-      TAOS_SML_KV* kv =  point->tags + j;
-      if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
-        int64_t ts = *(int64_t*)(kv->value);
-        ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision);
-        *(int64_t*)(kv->value) = ts;
-      }
-    }
-
-    for (int j = 0; j < point->fieldNum; ++j) {
-      TAOS_SML_KV* kv =  point->fields + j;
-      if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
-        int64_t ts = *(int64_t*)(kv->value);
-        ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision);
-        *(int64_t*)(kv->value) = ts;
-      }
-    }
+//    SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx);
+
+//    for (int j = 0; j < point->tagNum; ++j) {
+//      TAOS_SML_KV* kv =  point->tags + j;
+//      if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
+//        int64_t ts = *(int64_t*)(kv->value);
+//        ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision);
+//        *(int64_t*)(kv->value) = ts;
+//      }
+//    }
+//
+//    for (int j = 0; j < point->fieldNum; ++j) {
+//      TAOS_SML_KV* kv =  point->fields + j;
+//      if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
+//        int64_t ts = *(int64_t*)(kv->value);
+//        ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, stableSchema->precision);
+//        *(int64_t*)(kv->value) = ts;
+//      }
+//    }
 
     SArray* cTablePoints = NULL;
     SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName));
@@ -860,7 +860,14 @@ static int32_t addChildTableDataPointsToInsertSql(char* cTableName, char* sTable
       TAOS_SML_KV* kv = tagKVs[i];
       size_t       beforeLen = totalLen;
       int32_t      len = 0;
+
+      if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
+        int64_t ts = *(int64_t*)(kv->value);
+        ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, sTableSchema->precision);
+        converToStr(sql + beforeLen, kv->type, &ts, kv->length, &len);
+      }else{
         converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len);
+      }
       totalLen += len;
 
       ret = smlSnprintf(sql, &totalLen, capacity, ",");
@@ -914,7 +921,13 @@ static int32_t addChildTableDataPointsToInsertSql(char* cTableName, char* sTable
         TAOS_SML_KV* kv = colKVs[i];
         size_t       beforeLen = totalLen;
         int32_t      len = 0;
+        if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) {
+          int64_t ts = *(int64_t*)(kv->value);
+          ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, sTableSchema->precision);
+          converToStr(sql + beforeLen, kv->type, &ts, kv->length, &len);
+        }else{
           converToStr(sql + beforeLen, kv->type, kv->value, kv->length, &len);
+        }
         totalLen += len;
         ret = smlSnprintf(sql, &totalLen, capacity, ",");
         if (ret != 0) {
@@ -965,7 +978,7 @@ static void insertCallback(void *param, TAOS_RES *res, int32_t notUsedCode) {
     batch->tryAgain = true;
   }
 
-  if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
+  if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID || code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
     batch->resetQueryCache = true;
     if (batch->tryAgain) {
       batch->sleep = true;
@@ -1105,6 +1118,9 @@ static int32_t applyDataPointsWithSqlInsert(TAOS* taos, TAOS_SML_DATA_POINT* poi
     SSmlSqlInsertBatch* b = info->batches + i;
     if (b->code != 0) {
       code = b->code;
+      if(code == TSDB_CODE_MND_INVALID_TABLE_NAME){
+        break;
+      }
     }
   }
 
@@ -1221,6 +1237,8 @@ int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLine
 
   info->affectedRows = 0;
 
+  int32_t tableNotExistTryTimes = 3;
+  while(tableNotExistTryTimes){
     if (numPoint == 1) {
       TAOS_SML_DATA_POINT* point = points + 0;
       code = doSmlInsertOneDataPoint(taos, point, info);
@@ -1250,13 +1268,21 @@ int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLine
       tscError("SML:0x%"PRIx64" error apply data points : %s", info->id, tstrerror(code));
     }
 
-clean_up:
+  clean_up:
     for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) {
       SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i);
       taosArrayDestroy(&schema->fields);
       taosArrayDestroy(&schema->tags);
     }
     taosArrayDestroy(&stableSchemas);
+
+    if(code == TSDB_CODE_MND_INVALID_TABLE_NAME){
+      tableNotExistTryTimes--;
+    }else{
+      tableNotExistTryTimes = 0;
+    }
+  }
+
   return code;
 }
 
@@ -2090,7 +2116,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **idx, int32_t sqlLen,
       switch (tag_state) {
       case tag_common:
         if (back_slash == true) {
-          if (*cur != ',' && *cur != '=' && *cur != ' ' && *cur != 'n' ) {
+          if (*cur != ',' && *cur != '=' && *cur != ' ' && *cur != 'n' && *cur != 't' && *cur != 'r') {
             tscError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur);
             ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
             goto error;
@@ -2150,7 +2176,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **idx, int32_t sqlLen,
         break;
       case tag_lqoute:
         if (back_slash == true) {
-          if (*cur != ',' && *cur != '=' && *cur != ' ' && *cur != 'n') {
+          if (*cur != ',' && *cur != '=' && *cur != ' ' && *cur != 'n' && *cur != 't' && *cur != 'r') {
             tscError("SML:0x%"PRIx64" tag value: state(%d), incorrect character(%c) escaped", info->id, tag_state, *cur);
             ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
             goto error;
@@ -2205,7 +2231,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **idx, int32_t sqlLen,
       switch (val_state) {
       case val_common:
         if (back_slash == true) {
-          if (*cur != '\\' && *cur != '"' && *cur != 'n') {
+          if (*cur != '\\' && *cur != '"' && *cur != 'n' && *cur != 't' && *cur != 'r') {
             tscError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur);
             ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
             goto error;
@@ -2293,7 +2319,7 @@ static int32_t parseSmlValue(TAOS_SML_KV *pKV, const char **idx, int32_t sqlLen,
         break;
       case val_lqoute:
         if (back_slash == true) {
-          if (*cur != '\\' && *cur != '"' && *cur != 'n') {
+          if (*cur != '\\' && *cur != '"' && *cur != 'n' && *cur != 't' && *cur != 'r') {
             tscError("SML:0x%"PRIx64" field value: state(%d), incorrect character(%c) escaped", info->id, val_state, *cur);
             ret = TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
             goto error;

src/client/src/tscSQLParser.c

@@ -862,7 +862,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       const char* msg3 = "name too long";
       const char* msg5 = "invalid user rights";
       const char* msg7 = "not support options";
-      const char* msg8 = "tags filter length must over 3 bytes.";
+      const char* msg8 = "tags filter string length must less than 255 bytes.";
 
       pCmd->command = pInfo->type;
 
@@ -903,8 +903,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
           }
         } else if (pUser->type == TSDB_ALTER_USER_TAGS) {
           SStrToken* pTags = &pUser->tags;
-          if(pTags->n < 4)
-            return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
+          if (pTags->n > TSDB_TAGS_LEN - 1 ) return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
         } else {
           return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
         }

src/client/src/tscSql.c

@@ -21,6 +21,7 @@
 #include "tnote.h"
 #include "trpc.h"
 #include "tscLog.h"
+#include "tscBatchWrite.h"
 #include "tscSubquery.h"
 #include "tscUtil.h"
 #include "tsclient.h"
@@ -164,8 +165,23 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
   pSql->param     = param;
   pSql->cmd.command = TSDB_SQL_CONNECT;
   
+  pObj->dispatcherManager = NULL;
+  if (tsWriteBatchSize > 1 && !tscEmbedded) {
+    pObj->dispatcherManager = createDispatcherManager(pObj, tsWriteBatchSize, tsWriteBatchTimeout, tsWriteBatchThreadLocal);
+    if (!pObj->dispatcherManager) {
+      terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      tscReleaseRpc(pRpcObj);
+      free(pSql);
+      free(pObj);
+      return NULL;
+    }
+  }
+  
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    if (pObj->dispatcherManager) {
+      destroyDispatcherManager(pObj->dispatcherManager);
+    }
     tscReleaseRpc(pRpcObj);
     free(pSql);
     free(pObj);

src/client/src/tscStream.c

@@ -339,7 +339,7 @@ bool sqlBufSend(TAOS *taos, char *sqlBuf) {
   } while(++sleepCnt < 20);
 
   strcat(sqlBuf, ";");
-  taos_query_ra(taos, sqlBuf, cbSendValues, NULL);
+  taos_query_ra(taos, sqlBuf, cbSendValues, NULL, false);
   return true;
 }
 

src/client/src/tscSystem.c

@@ -14,18 +14,19 @@
  */
 
 #include "os.h"
+#include "qScript.h"
 #include "taosmsg.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tnote.h"
 #include "tref.h"
 #include "trpc.h"
-#include "tnote.h"
-#include "ttimer.h"
-#include "tsched.h"
+#include "tscBatchWrite.h"
 #include "tscLog.h"
+#include "tsched.h"
 #include "tsclient.h"
-#include "tglobal.h"
-#include "tconfig.h"
+#include "ttimer.h"
 #include "ttimezone.h"
-#include "qScript.h"
 
 // global, not configurable
 #define TSC_VAR_NOT_RELEASE 1
@@ -319,6 +320,49 @@ void taos_cleanup(void) {
   taosTmrCleanUp(p);
 }
 
+/**
+ * Set the option value (int32, uint16, int16, int8).
+ * @param cfg   the config.
+ * @param str   the value string.
+ * @return      whether is set or not.
+ */
+static bool taos_set_option_int(SGlobalCfg * cfg, const char* str) {
+  if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
+    char* p = NULL;
+    errno = 0;
+    long value = strtol(str, &p, 10);
+    
+    if (errno != 0 || p == str) {
+      tscError("failed to parse option: %s, value: %s", cfg->option, str);
+      return false;
+    }
+    
+    if ((float) value < cfg->minValue || (float) value > cfg->maxValue) {
+      tscError("failed to set option: %s, setValue: %ld, minValue: %f, maxValue: %f", cfg->option, value, cfg->minValue, cfg->maxValue);
+      return false;
+    }
+    
+    if (cfg->valType == TAOS_CFG_VTYPE_INT32) {
+      *((int32_t*) cfg->ptr) = (int32_t) value;
+    } else if (cfg->valType == TAOS_CFG_VTYPE_UINT16) {
+      *((uint16_t*) cfg->ptr) = (uint16_t) value;
+    } else if (cfg->valType == TAOS_CFG_VTYPE_INT16) {
+      *((int16_t*) cfg->ptr) = (int16_t) value;
+    } else if (cfg->valType == TAOS_CFG_VTYPE_INT8) {
+      *((int8_t*) cfg->ptr) = (int8_t) value;
+    } else {
+      tscError("failed to set option: %s, type expected %d", cfg->option, cfg->valType);
+      return false;
+    }
+    
+    cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
+    tscDebug("config option: %s has set to %s", cfg->option, str);
+    return true;
+  }
+  tscWarn("config option: %s, is configured by %s", cfg->option, tsCfgStatusStr[cfg->cfgStatus]);
+  return false;
+}
+
 static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
   SGlobalCfg *cfg = NULL;
 
@@ -474,6 +518,27 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       }
       break;
       
+    case TSDB_WRITE_BATCH_SIZE: {
+      cfg = taosGetConfigOption("writeBatchSize");
+      assert(cfg != NULL);
+      taos_set_option_int(cfg, pStr);
+      break;
+    }
+      
+    case TSDB_WRITE_BATCH_TIMEOUT: {
+      cfg = taosGetConfigOption("writeBatchTimeout");
+      assert(cfg != NULL);
+      taos_set_option_int(cfg, pStr);
+      break;
+    }  
+      
+    case TSDB_WRITE_BATCH_THREAD_LOCAL: {
+      cfg = taosGetConfigOption("writeBatchThreadLocal");
+      assert(cfg != NULL);
+      taos_set_option_int(cfg, pStr);
+      break;
+    }
+
     default:
       // TODO return the correct error code to client in the format for taos_errstr()
       tscError("Invalid option %d", option);

src/client/src/tscUtil.c

@@ -20,6 +20,7 @@
 #include "texpr.h"
 #include "tkey.h"
 #include "tmd5.h"
+#include "tscBatchWrite.h"
 #include "tscGlobalmerge.h"
 #include "tscLog.h"
 #include "tscProfile.h"
@@ -1841,32 +1842,36 @@ void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) {
   tfree(pColInfo->colIdxInfo);
 }
 
+void destroySTableDataBlocks(STableDataBlocks* pDataBlocks) {
+  if (!pDataBlocks) {
+    return;
+  }
+  tfree(pDataBlocks->pData);
+  if (!pDataBlocks->cloned) {
+    tfree(pDataBlocks->params);
+    
+    // free the refcount for metermeta
+    if (pDataBlocks->pTableMeta != NULL) {
+      tfree(pDataBlocks->pTableMeta);
+    }
+
+    tscDestroyBoundColumnInfo(&pDataBlocks->boundColumnInfo);
+  }
+  tfree(pDataBlocks);
+}
+
 void tscDestroyDataBlock(SSqlObj *pSql, STableDataBlocks* pDataBlock, bool removeMeta) {
   if (pDataBlock == NULL) {
     return;
   }
   
-  tfree(pDataBlock->pData);
-
   if (removeMeta) {
     char name[TSDB_TABLE_FNAME_LEN] = {0};
     tNameExtractFullName(&pDataBlock->tableName, name);
-
     taosHashRemove(UTIL_GET_TABLEMETA(pSql), name, strnlen(name, TSDB_TABLE_FNAME_LEN));
   }
 
-  if (!pDataBlock->cloned) {
-    tfree(pDataBlock->params);
-
-    // free the refcount for metermeta
-    if (pDataBlock->pTableMeta != NULL) {
-      tfree(pDataBlock->pTableMeta);
-    }
-
-    tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
-  }
-
-  tfree(pDataBlock);
+  destroySTableDataBlocks(pDataBlock);
 }
 
 SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
@@ -1893,18 +1898,22 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
   return param;
 }
 
-void*  tscDestroyBlockArrayList(SSqlObj *pSql, SArray* pDataBlockList) {
-  if (pDataBlockList == NULL) {
-    return NULL;
+void destroySTableDataBlocksList(SArray* pDataBlocks) {
+  if (!pDataBlocks) {
+    return;
   }
-
-  size_t size = taosArrayGetSize(pDataBlockList);
-  for (int32_t i = 0; i < size; i++) {
-    void* d = taosArrayGetP(pDataBlockList, i);
-    tscDestroyDataBlock(pSql, d, false);
+  size_t nBlocks = taosArrayGetSize(pDataBlocks);
+  for (size_t i = 0; i < nBlocks; ++i) {
+    STableDataBlocks * pDataBlock = taosArrayGetP(pDataBlocks, i);
+    if (pDataBlock) {
+      destroySTableDataBlocks(pDataBlock);
     }
+  }
+  taosArrayDestroy(&pDataBlocks);
+}
 
-  taosArrayDestroy(&pDataBlockList);
+void*  tscDestroyBlockArrayList(SSqlObj *pSql, SArray* pDataBlockList) {
+  destroySTableDataBlocksList(pDataBlockList);
   return NULL;
 }
 
@@ -2202,42 +2211,35 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) {
   return result;
 }
 
-static void extractTableNameList(SSqlObj *pSql, SInsertStatementParam *pInsertParam, bool freeBlockMap) {
-  pInsertParam->numOfTables = (int32_t) taosHashGetSize(pInsertParam->pTableBlockHashList);
-  if (pInsertParam->pTableNameList == NULL) {
-    pInsertParam->pTableNameList = malloc(pInsertParam->numOfTables * POINTER_BYTES);
-  }
-
-  STableDataBlocks **p1 = taosHashIterate(pInsertParam->pTableBlockHashList, NULL);
-  int32_t i = 0;
-  while(p1) {
-    STableDataBlocks* pBlocks = *p1;
-    //tfree(pInsertParam->pTableNameList[i]);
-
-    pInsertParam->pTableNameList[i++] = tNameDup(&pBlocks->tableName);
-    p1 = taosHashIterate(pInsertParam->pTableBlockHashList, p1);
-  }
-
-  if (freeBlockMap) {
-    pInsertParam->pTableBlockHashList = tscDestroyBlockHashTable(pSql, pInsertParam->pTableBlockHashList, false);
-  }
-}
-
 int32_t tscMergeTableDataBlocks(SSqlObj *pSql, SInsertStatementParam *pInsertParam, bool freeBlockMap) {
   const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
   int       code = 0;
   bool      isRawPayload = IS_RAW_PAYLOAD(pInsertParam->payloadType);
-  void*     pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
-  SArray*   pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
+  size_t    initialSize = taosHashGetSize(pInsertParam->pTableBlockHashList);
+  initialSize = initialSize > 128 ? 128 : initialSize;
   
-  STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL);
+  void*     pVnodeDataBlockHashList = taosHashInit(initialSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  SArray*   pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
   
-  STableDataBlocks* pOneTableBlock = *p;
+  // alloc table name list.
+  size_t numOfTables = taosHashGetSize(pInsertParam->pTableBlockHashList);
+  if (pInsertParam->pTableNameList) {
+    destroyTableNameList(pInsertParam);
+  }
+  pInsertParam->pTableNameList = calloc(numOfTables, sizeof(SName*));
+  pInsertParam->numOfTables = (int32_t) numOfTables;
   
+  size_t tail = 0;
   SBlockKeyInfo blkKeyInfo = {0};  // share by pOneTableBlock
-
-  while(pOneTableBlock) {
+  STableDataBlocks** iter = taosHashIterate(pInsertParam->pTableBlockHashList, NULL);
+  while (iter) {
+    STableDataBlocks* pOneTableBlock = *iter;
     SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
+    iter = taosHashIterate(pInsertParam->pTableBlockHashList, iter);
+    
+    // extract table name list.
+    pInsertParam->pTableNameList[tail++] = tNameDup(&pOneTableBlock->tableName);
+    
     if (pBlocks->numOfRows > 0) {
       // the maximum expanded size in byte when a row-wise data is converted to SDataRow format
       int32_t           expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
@@ -2320,19 +2322,19 @@ int32_t tscMergeTableDataBlocks(SSqlObj *pSql, SInsertStatementParam *pInsertPar
       dataBuf->numOfTables += 1;
 
       pBlocks->numOfRows = 0;
-    }else {
+    } else {
       tscDebug("0x%"PRIx64" table %s data block is empty", pInsertParam->objectId, pOneTableBlock->tableName.tname);
     }
     
-    p = taosHashIterate(pInsertParam->pTableBlockHashList, p);
-    if (p == NULL) {
-      break;
+    if (freeBlockMap) {
+      tscDestroyDataBlock(pSql, pOneTableBlock, false);
     }
-
-    pOneTableBlock = *p;
   }
   
-  extractTableNameList(pSql, pInsertParam, freeBlockMap);
+  if (freeBlockMap) {
+    taosHashCleanup(pInsertParam->pTableBlockHashList);
+    pInsertParam->pTableBlockHashList = NULL;
+  }
 
   // free the table data blocks;
   pInsertParam->pDataBlocks = pVnodeDataBlockList;
@@ -2354,6 +2356,9 @@ void tscCloseTscObj(void *param) {
   pthread_mutex_destroy(&pObj->mutex);
   tscReleaseClusterInfo(pObj->clusterId);
   
+  destroyDispatcherManager(pObj->dispatcherManager);
+  pObj->dispatcherManager = NULL;
+
   tfree(pObj);
 }
 

src/common/inc/tglobal.h

@@ -92,6 +92,9 @@ extern int32_t tsRetryStreamCompDelay;
 extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the whole time window
 extern int32_t tsProjectExecInterval;
 extern int64_t tsMaxRetentWindow;
+extern bool    tsWriteBatchThreadLocal;
+extern int32_t tsWriteBatchSize;
+extern int32_t tsWriteBatchTimeout;
 
 // db parameters in client
 extern int32_t tsCacheBlockSize;
@@ -151,6 +154,10 @@ extern char    tsMonitorDbName[];
 extern char    tsInternalPass[];
 extern int32_t tsMonitorInterval;
 
+// audit
+extern int8_t  tsEnableAudit;
+extern char    tsAuditDbName[];
+
 // stream
 extern int8_t tsEnableStream;
 

src/common/src/tglobal.c

@@ -127,6 +127,11 @@ int8_t tsSortWhenGroupBy = 1;
 int32_t tsProjectExecInterval = 10000;   // every 10sec, the projection will be executed once
 int64_t tsMaxRetentWindow = 24 * 3600L;  // maximum time window tolerance
 
+// The tsc async write batching feature (using ABWD).
+bool    tsWriteBatchThreadLocal = false;    // if thread local enable, each thread will allocate a dispatcher.
+int32_t tsWriteBatchSize = 0;               // suggest: 64 - 512, default 0, 0 means disable batching.
+int32_t tsWriteBatchTimeout = 10;           // suggest: 2 - 100 (unit: milliseconds)
+
 // the maximum allowed query buffer size during query processing for each data node.
 // -1 no limit (default)
 // 0  no query allowed, queries are disabled
@@ -200,6 +205,10 @@ char    tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char    tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+// audit
+int8_t  tsEnableAudit = 0;
+char    tsAuditDbName[TSDB_DB_NAME_LEN] = "audit";
+
 // stream
 int8_t tsEnableStream = 1;
 
@@ -1287,6 +1296,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "audit";
+  cfg.ptr = &tsEnableAudit;
+  cfg.valType = TAOS_CFG_VTYPE_INT8;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 1;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "stream";
   cfg.ptr = &tsEnableStream;
   cfg.valType = TAOS_CFG_VTYPE_INT8;
@@ -1860,6 +1879,37 @@ static void doInitGlobalConfig(void) {
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
+  
+  cfg.option = "writeBatchThreadLocal";
+  cfg.ptr = &tsWriteBatchThreadLocal;
+  cfg.valType = TAOS_CFG_VTYPE_INT8;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "writeBatchSize";
+  cfg.ptr = &tsWriteBatchSize;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 4096;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "writeBatchTimeout";
+  cfg.ptr = &tsWriteBatchTimeout;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 1;
+  cfg.maxValue = 2048;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM);
 #else
   // if TD_TSZ macro define, have 5 count configs, so must add 5

src/inc/monitor.h

@@ -27,6 +27,27 @@ extern "C" {
   monSaveDnodeLog(level, __VA_ARGS__); \
 }
 
+typedef enum {
+  // create
+  MON_DDL_CMD_CREATE_DATABASE,
+  MON_DDL_CMD_CREATE_TABLE,
+  MON_DDL_CMD_CREATE_CHILD_TABLE,
+  MON_DDL_CMD_CREATE_SUPER_TABLE,
+  // drop
+  MON_DDL_CMD_DROP_DATABASE,
+  MON_DDL_CMD_DROP_TABLE,
+  MON_DDL_CMD_DROP_CHILD_TABLE,
+  MON_DDL_CMD_DROP_SUPER_TABLE,
+  // alter
+  MON_DDL_CMD_ALTER_DATABASE,
+  MON_DDL_CMD_ADD_COLUMN,
+  MON_DDL_CMD_DROP_COLUMN,
+  MON_DDL_CMD_MODIFY_COLUMN,
+  MON_DDL_CMD_ADD_TAG,
+  MON_DDL_CMD_DROP_TAG,
+  MON_DDL_CMD_CHANGE_TAG,
+} EMonDDLCmdType;
+
 typedef struct {
   const char * name;
   int32_t code;
@@ -62,6 +83,7 @@ int32_t monInitSystem();
 int32_t monStartSystem();
 void    monStopSystem();
 void    monCleanupSystem();
+void    monSaveAuditLog(int8_t type, const char *user, const char *obj, bool result);
 void    monSaveAcctLog(SAcctMonitorObj *pMonObj);
 void    monSaveLog(int32_t level, const char *const format, ...);
 void    monSaveDnodeLog(int32_t level, const char *const format, ...);

src/inc/taos.h

@@ -60,7 +60,10 @@ typedef enum {
   TSDB_OPTION_TIMEZONE,
   TSDB_OPTION_CONFIGDIR,
   TSDB_OPTION_SHELL_ACTIVITY_TIMER,
-  TSDB_MAX_OPTIONS
+  TSDB_MAX_OPTIONS,
+  TSDB_WRITE_BATCH_SIZE,
+  TSDB_WRITE_BATCH_TIMEOUT,
+  TSDB_WRITE_BATCH_THREAD_LOCAL
 } TSDB_OPTION;
 
 typedef struct taosField {

src/kit/shell/src/shellAuto.c

@@ -31,7 +31,7 @@
 
 
 // extern function
-void insertChar(Command *cmd, char *c, int size);
+void insertStr(Command *cmd, char *str, int size);
 
 
 typedef struct SAutoPtr {
@@ -533,26 +533,16 @@ void parseCommand(SWords * command, bool pattern) {
 }
 
 // free Command
-void freeCommand(SWords * command) {
-  SWord * word = command->head;
-  if (word == NULL) {
-    return ;
-  }
-
+void freeCommand(SWords* command) {
+  SWord* item = command->head;
   // loop
-  while (word->next) {
-    SWord * tmp = word;
-    word = word->next;
+  while (item) {
+    SWord* tmp = item;
+    item = item->next;
     // if malloc need free
-    if(tmp->free && tmp->word)
-      free(tmp->word);
+    if (tmp->free && tmp->word) free(tmp->word);
     free(tmp);
   }
-
-  // if malloc need free
-  if(word->free && word->word)
-    free(word->word);
-  free(word);
 }
 
 void GenerateVarType(int type, char** p, int count) {
@@ -1087,7 +1077,7 @@ void printScreen(TAOS * con, Command * cmd, SWords * match) {
   }
   
   // insert new
-  insertChar(cmd, (char *)str, strLen);
+  insertStr(cmd, (char *)str, strLen);
 }
 
 
@@ -1178,11 +1168,11 @@ bool nextMatchCommand(TAOS * con, Command * cmd, SWords * firstMatch) {
   printScreen(con, cmd, match);
 
   // free
+  freeCommand(input);
   if (input->source) {
     free(input->source);
     input->source = NULL;
   }
-  freeCommand(input);
   free(input);
 
   return true;
@@ -1202,7 +1192,7 @@ bool fillWithType(TAOS * con, Command * cmd, char* pre, int type) {
 
   // show
   int count = strlen(part);
-  insertChar(cmd, part, count);
+  insertStr(cmd, part, count);
   cntDel = count; // next press tab delete current append count
 
   free(str);
@@ -1230,7 +1220,7 @@ bool fillTableName(TAOS * con, Command * cmd, char* pre) {
 
   // show
   int count = strlen(part);
-  insertChar(cmd, part, count);
+  insertStr(cmd, part, count);
   cntDel = count; // next press tab delete current append count
   
   free(str);
@@ -1354,7 +1344,7 @@ bool appendAfterSelect(TAOS * con, Command * cmd, char* sql, int32_t len) {
     bool fieldEnd = fieldsInputEnd(p);
     // cheeck fields input end then insert from keyword
     if (fieldEnd && p[len-1] == ' ') {
-      insertChar(cmd, "from", 4);
+      insertStr(cmd, "from", 4);
       free(p);
       return true;
     }
@@ -1537,7 +1527,7 @@ bool matchOther(TAOS * con, Command * cmd) {
   if (p[len - 1] == '\\') {
     // append '\G'
     char a[] = "G;";
-    insertChar(cmd, a, 2);
+    insertStr(cmd, a, 2);
     return true;
   }
 

src/kit/shell/src/shellCheck.c

@@ -131,7 +131,7 @@ static void *shellCheckThreadFp(void *arg) {
     char *tbname = tbNames[t];
     if (tbname == NULL) break;
 
-    snprintf(sql, SHELL_SQL_LEN, "select count(*) from %s;", tbname);
+    snprintf(sql, SHELL_SQL_LEN, "select last_row(_c0) from `%s`;", tbname);
 
     TAOS_RES *pSql = taos_query(pThread->taos, sql);
     int32_t   code = taos_errno(pSql);

src/kit/shell/src/shellCommand.c

@@ -79,11 +79,22 @@ void insertChar(Command *cmd, char *c, int size) {
   /* update the values */
   cmd->commandSize += size;
   cmd->cursorOffset += size;
-  for (int i = 0; i < size; i++) {
-    mbtowc(&wc, c + i, size);
   cmd->screenOffset += wcwidth(wc);
   cmd->endOffset += wcwidth(wc);
-  } 
+  showOnScreen(cmd);
+}
+
+void insertStr(Command *cmd, char *str, int size) {
+  clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size);
+  /* update the buffer */
+  memmove(cmd->command + cmd->cursorOffset + size, cmd->command + cmd->cursorOffset,
+          cmd->commandSize - cmd->cursorOffset);
+  memcpy(cmd->command + cmd->cursorOffset, str, size);
+  /* update the values */
+  cmd->commandSize += size;
+  cmd->cursorOffset += size;
+  cmd->screenOffset += size;
+  cmd->endOffset += size; 
   showOnScreen(cmd);
 }
 

src/kit/shell/src/shellEngine.c

@@ -377,7 +377,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) {
   } else {
     int num_rows_affacted = taos_affected_rows(pSql);
     et = taosGetTimestampUs();
-    printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6);
+    printf("Query OK, %d row(s) affected in set (%.6fs)\n", num_rows_affacted, (et - st) / 1E6);
 
 #ifndef WINDOWS
     // call auto tab

src/kit/shell/src/tire.c

@@ -325,30 +325,26 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) {
 }
 
 SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) {
-    if(match == NULL) {
-        match = (SMatch* )tmalloc(sizeof(SMatch));
-        memset(match, 0, sizeof(SMatch));
+  SMatch* rMatch = match; // define return match
+  if (rMatch == NULL) {
+    rMatch = (SMatch*)malloc(sizeof(SMatch));
+    memset(rMatch, 0, sizeof(SMatch));
   }
 
   switch (tire->type) {
     case TIRE_TREE:
-        matchPrefixFromTree(tire, prefix, match);
+      matchPrefixFromTree(tire, prefix, rMatch);
+      break;
     case TIRE_LIST:
-        matchPrefixFromList(tire, prefix, match);
+      matchPrefixFromList(tire, prefix, rMatch);
+      break;
     default:
       break;
   }
 
-    // return if need
-    if (match->count == 0) {
-        freeMatch(match);
-        match = NULL;
-    }    
-
-    return match;  
+  return rMatch;
 }
 
-
 // get all items from tires tree
 void enumFromList(STire* tire, SMatch* match) {
     StrName * item = tire->head;
@@ -395,8 +391,10 @@ SMatch* enumAll(STire* tire) {
     switch (tire->type) {
       case TIRE_TREE: 
         enumFromTree(tire, match);
+        break;
       case TIRE_LIST:
         enumFromList(tire, match);
+        break;
       default:
         break;
     }

taos-tools @ d0fb461c

-Subproject commit 217a267b5365b7a4089eb6ee2d4b2ceedfc4df66
+Subproject commit d0fb461c232a5d6f5e863d954293a8af9cd2c718

src/mnode/src/mnodeDb.c

@@ -409,6 +409,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) {
   } else {
     mError("db:%s, failed to create by %s, reason:%s", pDb->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
   }
+  monSaveAuditLog(MON_DDL_CMD_CREATE_DATABASE, mnodeGetUserFromMsg(pMsg), pDb->name, !code);
 
   return code;
 }
@@ -1013,9 +1014,9 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) {
     newCfg.daysToKeep2 = daysToKeep2;
   }
 
-  if (minRows > 0 && minRows != pDb->cfg.minRowsPerFileBlock) {
-    mError("db:%s, can't alter minRows option", pDb->name);
-    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+  if (minRows >= 0 && minRows != pDb->cfg.minRowsPerFileBlock) {
+    mDebug("db:%s, minRows:%d change to %d", pDb->name, pDb->cfg.minRowsPerFileBlock, minRows);
+    newCfg.minRowsPerFileBlock = minRows;
   }
 
   if (maxRows > 0 && maxRows != pDb->cfg.maxRowsPerFileBlock) {
@@ -1127,6 +1128,8 @@ static int32_t mnodeAlterDbFp(SMnodeMsg *pMsg) {
   mDebug("db:%s, all vgroups is altered", pDb->name);
   mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
 
+  monSaveAuditLog(MON_DDL_CMD_ALTER_DATABASE, mnodeGetUserFromMsg(pMsg), pDb->name, true);
+
   // in case there is no vnode for this db currently(no table in db,etc.)
   if (pMsg->expected == 0) {
     SSdbRow row = {
@@ -1183,7 +1186,7 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
     if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mError("db:%s, failed to alter, reason:%s", pDb->name, tstrerror(code));
     }
-}
+  }
 
   return code;
 }
@@ -1213,6 +1216,7 @@ static int32_t mnodeDropDbCb(SMnodeMsg *pMsg, int32_t code) {
   } else {
     mLInfo("db:%s, is dropped by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
   }
+  monSaveAuditLog(MON_DDL_CMD_DROP_DATABASE, mnodeGetUserFromMsg(pMsg), pDb->name, !code);
 
   return code;
 }

src/mnode/src/mnodeTable.c

@@ -38,6 +38,7 @@
 #include "mnodeSdb.h"
 #include "mnodeShow.h"
 #include "mnodeTable.h"
+#include "mnodeUser.h"
 #include "mnodeVgroup.h"
 #include "mnodeWrite.h"
 #include "mnodeRead.h"
@@ -1056,6 +1057,7 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
     if(pMsg->pBatchMasterMsg)
       pMsg->pBatchMasterMsg->received ++;
   }
+  monSaveAuditLog(MON_DDL_CMD_CREATE_SUPER_TABLE, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, !code);
 
   // if super table create by batch msg, check done and send finished to client
   if(pMsg->pBatchMasterMsg) {
@@ -1171,11 +1173,12 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
   if (code != TSDB_CODE_SUCCESS) {
     mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
-
+    monSaveAuditLog(MON_DDL_CMD_DROP_SUPER_TABLE, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return code;
   }
 
   mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
+  monSaveAuditLog(MON_DDL_CMD_DROP_SUPER_TABLE, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, true);
 
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
@@ -1250,6 +1253,8 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+  monSaveAuditLog(MON_DDL_CMD_ADD_TAG, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
+
   return code;
 }
 
@@ -1308,6 +1313,8 @@ static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+  monSaveAuditLog(MON_DDL_CMD_DROP_TAG, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
+
   return code;
 }
 
@@ -1345,6 +1352,8 @@ static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+  monSaveAuditLog(MON_DDL_CMD_CHANGE_TAG, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
+
   return code;
 }
 
@@ -1403,6 +1412,8 @@ static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+
+  monSaveAuditLog(MON_DDL_CMD_ADD_COLUMN, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
   return code;
 }
 
@@ -1474,6 +1485,8 @@ static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+  monSaveAuditLog(MON_DDL_CMD_DROP_COLUMN, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
+
   return code;
 }
 
@@ -1522,6 +1535,8 @@ static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     code = mnodeGetSuperTableMeta(pMsg);
   }
+  monSaveAuditLog(MON_DDL_CMD_MODIFY_COLUMN, mnodeGetUserFromMsg(pMsg), pStable->info.tableId, !code);
+
   return code;
 }
 
@@ -2073,6 +2088,9 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
   SCreateTableMsg *pCreate = (SCreateTableMsg*) ((char*)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg));
   assert(pTable);
 
+  monSaveAuditLog((pTable->info.type == TSDB_CHILD_TABLE) ? MON_DDL_CMD_CREATE_CHILD_TABLE : MON_DDL_CMD_CREATE_TABLE,
+                  mnodeGetUserFromMsg(pMsg), pTable->info.tableId, !code);
+
   if (code == TSDB_CODE_SUCCESS) {
     if (pCreate->getMeta) {
       mDebug("msg:%p, app:%p table:%s, created in dnode and continue to get meta, thandle:%p", pMsg,
@@ -2101,6 +2119,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
   } else {
     mError("msg:%p, app:%p table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg,
            pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code));
+
     SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb};
     sdbDeleteRow(&desc);
 
@@ -2294,6 +2313,8 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
   if (pDrop == NULL) {
     mError("msg:%p, app:%p ctable:%s, failed to drop ctable, no enough memory", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId);
+    monSaveAuditLog((pTable->info.type == TSDB_CHILD_TABLE) ? MON_DDL_CMD_DROP_CHILD_TABLE : MON_DDL_CMD_DROP_TABLE,
+                    mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
@@ -2308,6 +2329,9 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
   mInfo("msg:%p, app:%p ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle,
         pDrop->tableFname, pTable->vgId, pTable->tid, pTable->uid);
 
+  monSaveAuditLog((pTable->info.type == TSDB_CHILD_TABLE) ? MON_DDL_CMD_DROP_CHILD_TABLE : MON_DDL_CMD_DROP_TABLE,
+                  mnodeGetUserFromMsg(pMsg), pTable->info.tableId, true);
+
   SRpcMsg rpcMsg = {
     .ahandle = pMsg,
     .pCont   = pDrop,
@@ -2327,6 +2351,8 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
     mError("msg:%p, app:%p ctable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
+    monSaveAuditLog((pTable->info.type == TSDB_CHILD_TABLE) ? MON_DDL_CMD_DROP_CHILD_TABLE : MON_DDL_CMD_DROP_TABLE,
+                    mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return code;
   }
 
@@ -2414,6 +2440,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
   SDbObj *pDb = pMsg->pDb;
   if (ncols <= 0) {
     mError("msg:%p, app:%p ctable:%s, add column, ncols:%d <= 0", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, ncols);
+    monSaveAuditLog(MON_DDL_CMD_ADD_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return TSDB_CODE_MND_APP_ERROR;
   }
 
@@ -2421,6 +2448,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
     if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) {
       mError("msg:%p, app:%p ctable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
              pTable->info.tableId, schema[i].name);
+      monSaveAuditLog(MON_DDL_CMD_ADD_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
       return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
     }
   }
@@ -2445,6 +2473,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
   }
 
   mInfo("msg:%p, app:%p ctable %s, start to add column", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
+  monSaveAuditLog(MON_DDL_CMD_ADD_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, true);
 
   SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
@@ -2464,6 +2493,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
   if (col <= 0) {
     mError("msg:%p, app:%p ctable:%s, drop column, column:%s not exist", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId, colName);
+    monSaveAuditLog(MON_DDL_CMD_DROP_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
 
@@ -2478,6 +2508,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
   }
 
   mInfo("msg:%p, app:%p ctable %s, start to drop column %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, colName);
+  monSaveAuditLog(MON_DDL_CMD_DROP_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, true);
 
   SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
@@ -2498,15 +2529,18 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg) {
   if (col < 0) {
     mError("msg:%p, app:%p ctable:%s, change column, name: %s", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId, name);
+    monSaveAuditLog(MON_DDL_CMD_MODIFY_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, false);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
 
   SSchema *schema = (SSchema *) (pTable->schema + col);
   ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
   schema->bytes = pAlter->schema[0].bytes;
+  ++pTable->sversion;
 
   mInfo("msg:%p, app:%p ctable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
          name, schema->bytes);
+  monSaveAuditLog(MON_DDL_CMD_MODIFY_COLUMN, mnodeGetUserFromMsg(pMsg), pTable->info.tableId, true);
 
   SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,

src/mnode/src/mnodeUser.c

@@ -97,7 +97,7 @@ static int32_t mnodeUserActionDecode(SSdbRow *pRow) {
   SUserObj *pUser = (SUserObj *)calloc(1, sizeof(SUserObj));
   if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pUser, pRow->rowData, tsUserUpdateSize);
+  memcpy(pUser, pRow->rowData, pRow->rowSize);
   pRow->pObj = pUser;
   return TSDB_CODE_SUCCESS;
 }

src/plugins/monitor/src/monMain.c

@@ -41,6 +41,9 @@
 #define DNODE_INFO_LEN      128
 #define QUERY_ID_LEN        24
 #define CHECK_INTERVAL      1000
+#define AUDIT_MAX_RETRIES   10
+#define MAX_DDL_TYPE_LEN    32
+#define MAX_DDL_OBJ_LEN     512
 
 #define SQL_STR_FMT "\"%s\""
 
@@ -161,6 +164,7 @@ typedef struct {
 } SMonStat;
 
 static void *monHttpStatusHashTable;
+static void *auditConn;
 
 static SMonConn tsMonitor = {0};
 static SMonStat tsMonStat = {{0}};
@@ -176,6 +180,7 @@ static void  monSaveGrantsInfo();
 static void  monSaveHttpReqInfo();
 static void  monGetSysStats();
 static void  *monThreadFunc(void *param);
+static void  *monAuditFunc(void *param);
 static void  monBuildMonitorSql(char *sql, int32_t cmd);
 static void  monInitHttpStatusHashTable();
 static void  monCleanupHttpStatusHashTable();
@@ -185,6 +190,13 @@ extern void    (*monStopSystemFp)();
 extern void    (*monExecuteSQLFp)(char *sql);
 extern char *  strptime(const char *buf, const char *fmt, struct tm *tm); //make the compilation pass
 
+#ifdef _STORAGE
+  char *keepValue = "30,30,30";
+#else
+  char *keepValue = "30";
+#endif
+
+
 int32_t monInitSystem() {
   if (tsMonitor.ep[0] == 0) {
     strcpy(tsMonitor.ep, tsLocalEp);
@@ -203,12 +215,20 @@ int32_t monInitSystem() {
   pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
   if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) {
-    monError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+    monError("failed to create thread for monitor module, reason:%s", strerror(errno));
+    return -1;
+  }
+  monDebug("monitor thread is launched");
+
+  pthread_t auditThread;
+  pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_DETACHED);
+  if (pthread_create(&auditThread, &thAttr, monAuditFunc, NULL)) {
+    monError("failed to create audit thread, reason:%s", strerror(errno));
     return -1;
   }
 
+  monDebug("audit thread is launched");
   pthread_attr_destroy(&thAttr);
-  monDebug("monitor thread is launched");
 
   monStartSystemFp = monStartSystem;
   monStopSystemFp = monStopSystem;
@@ -250,6 +270,70 @@ SMonHttpStatus *monGetHttpStatusHashTableEntry(int32_t code) {
   return (SMonHttpStatus*)taosHashGet(monHttpStatusHashTable, &code, sizeof(int32_t));
 }
 
+static void *monAuditFunc(void *param) {
+  if (!tsEnableAudit) {
+    return NULL;
+  }
+
+  monDebug("starting to initialize audit database...");
+  setThreadName("audit");
+  taosMsleep(1000);
+
+  int32_t try = 0;
+  for (; try < AUDIT_MAX_RETRIES; ++try) {
+    auditConn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
+    if (auditConn == NULL) {
+      monDebug("audit retry connect, tries: %d", try);
+      taosMsleep(1000);
+    } else {
+      monDebug("audit successfuly connect to database");
+      break;
+    }
+  }
+
+  if (try == AUDIT_MAX_RETRIES) {
+    monError("audit failed to connect to database, reason:%s", tstrerror(terrno));
+    return NULL;
+  }
+
+  // create database
+  char sql[512] = {0};
+  snprintf(sql, sizeof(sql),
+           "create database if not exists %s replica 1 days 10 keep %s cache %d "
+           "blocks %d precision 'us'",
+           tsAuditDbName, keepValue, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MIN_TOTAL_BLOCKS);
+
+  void *res = taos_query(auditConn, sql);
+  int32_t code = taos_errno(res);
+  taos_free_result(res);
+
+  if (code != 0) {
+    monError("failed to create database: %s, sql:%s, reason:%s", tsAuditDbName, sql, tstrerror(code));
+    return NULL;
+  }
+
+  // create table
+  memset(sql, 0, sizeof(sql));
+  snprintf(sql, sizeof(sql),
+           "create table if not exists %s.ddl(ts timestamp"
+           ", user_name binary(%d), ip_addr binary(%d), type binary(%d)"
+           ", object binary(%d), result binary(10)"
+           ")",
+           tsAuditDbName, TSDB_USER_LEN, IP_LEN_STR,
+           MAX_DDL_TYPE_LEN, MAX_DDL_OBJ_LEN);
+
+  res = taos_query(auditConn, sql);
+  code = taos_errno(res);
+  taos_free_result(res);
+
+  if (code != 0) {
+    monError("failed to create table: ddl, exec sql:%s, reason:%s", sql, tstrerror(code));
+    return NULL;
+  }
+
+  return NULL;
+}
+
 static void *monThreadFunc(void *param) {
   monDebug("starting to initialize monitor module ...");
   setThreadName("monitor");
@@ -335,12 +419,6 @@ static void *monThreadFunc(void *param) {
 static void monBuildMonitorSql(char *sql, int32_t cmd) {
   memset(sql, 0, SQL_LENGTH);
 
-#ifdef _STORAGE
-  char *keepValue = "30,30,30";
-#else
-  char *keepValue = "30";
-#endif
-
   if (cmd == MON_CMD_CREATE_DB) {
     snprintf(sql, SQL_LENGTH,
              "create database if not exists %s replica %d days 10 keep %s cache %d "
@@ -494,6 +572,11 @@ void monCleanupSystem() {
     pthread_join(tsMonitor.thread, NULL);
   }
 
+  if (auditConn != NULL) {
+    taos_close(tsMonitor.conn);
+    auditConn = NULL;
+  }
+
   if (tsMonitor.conn != NULL) {
     taos_close(tsMonitor.conn);
     tsMonitor.conn = NULL;
@@ -1323,6 +1406,116 @@ static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
   taos_free_result(result);
 }
 
+static bool monConvDDLType2Str(int8_t type, char *buf, int32_t len) {
+  if (buf == NULL) {
+    return false;
+  }
+
+  switch (type) {
+    case MON_DDL_CMD_CREATE_DATABASE: {
+      strncpy(buf, "CREATE DATABASE", len);
+      break;
+    }
+    case MON_DDL_CMD_CREATE_TABLE: {
+      strncpy(buf, "CREATE TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_CREATE_CHILD_TABLE: {
+      strncpy(buf, "CREATE CHILD TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_CREATE_SUPER_TABLE: {
+      strncpy(buf, "CREATE SUPER TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_DATABASE: {
+      strncpy(buf, "DROP DATABASE", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_TABLE: {
+      strncpy(buf, "DROP TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_CHILD_TABLE: {
+      strncpy(buf, "DROP CHILD TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_SUPER_TABLE: {
+      strncpy(buf, "DROP SUPER TABLE", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_COLUMN: {
+      strncpy(buf, "DROP COLUMN", len);
+      break;
+    }
+    case MON_DDL_CMD_DROP_TAG: {
+      strncpy(buf, "DROP TAG", len);
+      break;
+    }
+    case MON_DDL_CMD_ALTER_DATABASE: {
+      strncpy(buf, "ALTER DATABASE", len);
+      break;
+    }
+    case MON_DDL_CMD_ADD_COLUMN: {
+      strncpy(buf, "ADD COLUMN", len);
+      break;
+    }
+    case MON_DDL_CMD_ADD_TAG: {
+      strncpy(buf, "ADD TAG", len);
+      break;
+    }
+    case MON_DDL_CMD_MODIFY_COLUMN: {
+      strncpy(buf, "MODIFY COLUMN/TAG LENGTH", len);
+      break;
+    }
+    case MON_DDL_CMD_CHANGE_TAG: {
+      strncpy(buf, "CHANGE TAG NAME", len);
+      break;
+    }
+    default: {
+      return false;
+    }
+  }
+  return true;
+}
+
+void monSaveAuditLog(int8_t type, const char *user, const char *obj, bool result) {
+  if (tsEnableAudit == 0) { //audit not enabled
+    return;
+  }
+
+  char sql[1024] = {0};
+  char typeStr[64] = {0};
+
+  if (!monConvDDLType2Str(type, typeStr, (int32_t)sizeof(typeStr))) {
+    monError("unknown DDL type: %d ", type);
+    return;
+  }
+
+  snprintf(sql, 1023,
+          "insert into %s.ddl values(now, "
+          SQL_STR_FMT", "SQL_STR_FMT", "
+          SQL_STR_FMT", "SQL_STR_FMT", "
+          SQL_STR_FMT")",
+          tsAuditDbName,
+          (user != NULL) ? user : "NULL",
+          tsLocalEp,
+          typeStr,
+          (obj != NULL) ? obj : "NULL",
+          result ? "success" : "fail");
+
+  monDebug("save ddl info, sql:%s", sql);
+  void *res = taos_query(auditConn, sql);
+  int32_t code = taos_errno(res);
+  taos_free_result(res);
+
+  if (code != 0) {
+    monError("failed to save audit ddl info, reason:%s, sql:%s", tstrerror(code), sql);
+  } else {
+    monDebug("successfully save audit ddl info, sql:%s", sql);
+  }
+}
+
 void monSaveAcctLog(SAcctMonitorObj *pMon) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
@@ -1399,6 +1592,7 @@ void monSaveDnodeLog(int32_t level, const char *const format, ...) {
   taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log");
 }
 
+
 void monExecuteSQL(char *sql) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 

taosadapter @ 688d7ec6

-Subproject commit 1e0a02097f8433f557125898c756f293c97da88d
+Subproject commit 688d7ec69fa53d2736cc95db08941f9c5c48056f

src/query/inc/sql.y

@@ -311,6 +311,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X).      { Y = Z; Y.numOfBlocks = s
 alter_db_optr(Y) ::= alter_db_optr(Z) comp(X).        { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) update(X).      { Y = Z; Y.update = strtol(X.z, NULL, 10); }
 alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X).   { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); }
+alter_db_optr(Y) ::= alter_db_optr(Z) minrows(X).     { Y = Z; Y.minRowsPerBlock = strtol(X.z, NULL, 10); }
 
 // dynamically update the following two parameters are not allowed.
 //alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X).       { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); }

src/query/src/qAggMain.c

@@ -5912,6 +5912,11 @@ static void mode_func_finalizer(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   SModeFuncInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
 
+  if (pRes->num == 0) {
+    setNull(pCtx->pOutput, type, 0);
+    goto _mode_over;
+  }
+
   size_t size = sizeof(ModeUnit) + bytes;
 
   char *tvp = pRes->res;
@@ -5919,20 +5924,18 @@ static void mode_func_finalizer(SQLFunctionCtx *pCtx) {
   int64_t maxCount = 0;
   for (int32_t i = 0; i < pRes->num; ++i) {
     int64_t count = ((ModeUnit*)tvp)->count;
-    if (count > maxCount){
+    if (count >= maxCount){
       maxCount = count;
       result = tvp;
-    }else if(count == maxCount){
-      result = NULL;
     }
+
     tvp += size;
   }
 
-  if (result){
   memcpy(pCtx->pOutput, result + sizeof(ModeUnit), bytes);
-  }else{
-    setNull(pCtx->pOutput, type, 0);
-  }
+
+_mode_over:
+
   pResInfo->numOfRes = 1;
   doFinalizer(pCtx);
 }

src/query/src/qExecutor.c

@@ -2837,6 +2837,10 @@ static bool isCachedLastQuery(SQueryAttr* pQueryAttr) {
     return false;
   }
 
+  if (pQueryAttr->pFilters != NULL || pQueryAttr->pFilterInfo != NULL) {
+    return false;
+  }
+
   return true;
 }
 
@@ -3165,7 +3169,7 @@ static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlock
       return true;
     }
 
-    while(w.skey < pBlockInfo->window.ekey) {
+    while (w.skey < pBlockInfo->window.ekey) {
       // add one slding
       if (pQueryAttr->interval.slidingUnit == 'n' || pQueryAttr->interval.slidingUnit == 'y')
         ekey = taosTimeAdd(ekey, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision);
@@ -3180,7 +3184,7 @@ static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlock
       getAlignQueryTimeWindow(pQueryAttr, ekey, sk, ek, &w);
     }
 
-    while(1) {
+    while (1) {
       if (w.ekey < pBlockInfo->window.skey) {
         break;
       }
@@ -6984,7 +6988,7 @@ static bool doEveryInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlo
     return false;
   }
   int32_t nRows = pBlock ? pBlock->info.rows : 0;
-  int32_t startPos = binarySearchForKey((char*) tsCols, nRows, pCtx->startTs, pQueryAttr->order.order);
+  int32_t startPos = binarySearchForKey((char*)tsCols, nRows, pCtx->startTs, pQueryAttr->order.order);
 
   if (ascQuery && pQueryAttr->fillType != TSDB_FILL_NEXT && pCtx->start.key == INT64_MIN) {
     if (startPos < 0) {
@@ -10171,7 +10175,8 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
     pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES);
     pTableqinfo->numOfTables = pTableGroupInfo->numOfTables;
 
-    pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+    pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true,
+                                    HASH_ENTRY_LOCK);
   }
 
   pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo));
@@ -10373,7 +10378,7 @@ static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) {
 
   taosArrayDestroy(&pTableqinfoGroupInfo->pGroupList);
 
-  SHashObj *pmap = pTableqinfoGroupInfo->map;
+  SHashObj* pmap = pTableqinfoGroupInfo->map;
   if (pmap == atomic_val_compare_exchange_ptr(&pTableqinfoGroupInfo->map, pmap, NULL)) {
     taosHashCleanup(pmap);
   }

src/rpc/src/rpcMain.c

@@ -1080,7 +1080,9 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) {
     pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     pContext->pConn = NULL;
     pConn->pReqMsg = NULL;
-    taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl);
+    int64_t *rid = malloc(sizeof(int64_t));
+    *rid = pContext->rid;
+    taosTmrStart(rpcProcessConnError, 0, rid, pRpc->tmrCtrl);
   }
    
   if (pConn->inType) rpcReportBrokenLinkToServer(pConn); 
@@ -1293,7 +1295,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte
       rpcFreeCont(rpcMsg.pCont);
     } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || pHead->code == TSDB_CODE_DND_EXITING) {
       pContext->code = pHead->code;
-      rpcProcessConnError(pContext, NULL);
+      int64_t *rid = malloc(sizeof(int64_t));
+      *rid = pContext->rid;      
+      rpcProcessConnError(rid, NULL);
       rpcFreeCont(rpcMsg.pCont);
     } else {
       rpcNotifyClient(pContext, &rpcMsg);
@@ -1395,7 +1399,9 @@ static TBOOL rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
   if (pConn == NULL) {
     pContext->code = terrno;
     // in rpcProcessConnError if numOfTry over limit, could call rpcNotifyClient to stop query
-    taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl);
+    int64_t *rid = malloc(sizeof(int64_t));
+    *rid = pContext->rid;
+    taosTmrStart(rpcProcessConnError, 1, rid, pRpc->tmrCtrl);
     return BOOL_ASYNC;
   }
 
@@ -1442,7 +1448,9 @@ static TBOOL rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
     // try next ip again
     pContext->code = terrno;
     // in rpcProcessConnError if numOfTry over limit, could call rpcNotifyClient to stop query
-    taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl);
+    int64_t *rid = malloc(sizeof(int64_t));
+    *rid = pContext->rid;
+    taosTmrStart(rpcProcessConnError, 1, rid, pRpc->tmrCtrl);
     return BOOL_ASYNC;
   }
 
@@ -1480,11 +1488,23 @@ static bool rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) {
 }
 
 static void rpcProcessConnError(void *param, void *id) {
-  SRpcReqContext *pContext = (SRpcReqContext *)param;
+  if (NULL == param) {
+    return;
+  }
+  
+  int64_t *rid = (int64_t*)param;
+  SRpcReqContext *pContext = (SRpcReqContext *)taosAcquireRef(tsRpcRefId, *rid);
+  if (NULL == pContext) {
+    free(param);
+    return;
+  }
+  
   SRpcInfo       *pRpc = pContext->pRpc;
   SRpcMsg         rpcMsg;
  
   if (pRpc == NULL) {
+    taosReleaseRef(tsRpcRefId, *rid);
+    free(param);
     return;
   }
   
@@ -1504,6 +1524,9 @@ static void rpcProcessConnError(void *param, void *id) {
     pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps;
     rpcSendReqToServer(pRpc, pContext);
   }
+
+  taosReleaseRef(tsRpcRefId, *rid);  
+  free(param);  
 }
 
 static void rpcProcessRetryTimer(void *param, void *tmrId) {
@@ -1528,7 +1551,9 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) {
         pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
         pConn->pContext->pConn = NULL;
         pConn->pReqMsg = NULL;
-        taosTmrStart(rpcProcessConnError, 1, pConn->pContext, pRpc->tmrCtrl);
+        int64_t *rid = malloc(sizeof(int64_t));
+        *rid = pConn->pContext->rid;
+        taosTmrStart(rpcProcessConnError, 1, rid, pRpc->tmrCtrl);
         rpcReleaseConn(pConn);
       }
     }

src/tsdb/src/tsdbCommit.c

@@ -1080,7 +1080,7 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDFile
   int         rowsToWrite = pDataCols->numOfRows;
 
   ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock);
-  ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock);
+  // ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock);
 
   // Make buffer space
   if (tsdbMakeRoom(ppBuf, tsdbBlockStatisSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) {

src/tsdb/src/tsdbCommitQueue.c

@@ -132,13 +132,16 @@ static void tsdbApplyRepoConfig(STsdbRepo *pRepo) {
   pRepo->config.keep2 = pRepo->save_config.keep2;
   pRepo->config.cacheLastRow = pRepo->save_config.cacheLastRow;
   pRepo->config.totalBlocks = pRepo->save_config.totalBlocks;
+  pRepo->config.minRowsPerFileBlock = pRepo->save_config.minRowsPerFileBlock;
 
   pthread_mutex_unlock(&pRepo->save_mutex);
 
-  tsdbInfo("vgId:%d apply new config: compression(%d), keep(%d,%d,%d), totalBlocks(%d), cacheLastRow(%d->%d),totalBlocks(%d->%d)",
-    REPO_ID(pRepo),
-    pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2,
-    pSaveCfg->totalBlocks, oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks);
+  tsdbInfo(
+      "vgId:%d apply new config: "
+      "compression(%d),keep(%d,%d,%d),totalBlocks(%d),cacheLastRow(%d->%d),totalBlocks(%d->%d),minRows(%d->%d)",
+      REPO_ID(pRepo), pSaveCfg->compression, pSaveCfg->keep, pSaveCfg->keep1, pSaveCfg->keep2, pSaveCfg->totalBlocks,
+      oldCfg.cacheLastRow, pSaveCfg->cacheLastRow, oldTotalBlocks, pSaveCfg->totalBlocks, oldCfg.minRowsPerFileBlock,
+      pSaveCfg->minRowsPerFileBlock);
 
   int err = tsdbExpandPool(pRepo, oldTotalBlocks);
   if (!TAOS_SUCCEEDED(err)) {

src/tsdb/src/tsdbMain.c

@@ -233,7 +233,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) {
   ASSERT(pRCfg->tsdbId == pCfg->tsdbId);
   ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize);
   ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile);
-  ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock);
+  // ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock);
   ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock);
   ASSERT(pRCfg->precision == pCfg->precision);
 
@@ -256,6 +256,9 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) {
   if (pRCfg->totalBlocks != pCfg->totalBlocks) {
     configChanged = true;
   }
+  if (pRCfg->minRowsPerFileBlock != pCfg->minRowsPerFileBlock) {
+    configChanged = true;
+  }
 
   if (!configChanged) {
     tsdbError("vgId:%d no config changed", REPO_ID(repo));
@@ -277,15 +280,16 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) {
   pSaveCfg->keep2 = pCfg->keep2;
   pSaveCfg->cacheLastRow = pCfg->cacheLastRow;
   pSaveCfg->totalBlocks = pCfg->totalBlocks;
+  pSaveCfg->minRowsPerFileBlock = pCfg->minRowsPerFileBlock;
 
-  tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)",
+  tsdbInfo("vgId:%d old config: compression(%d),keep(%d,%d,%d),cacheLastRow(%d),totalBlocks(%d),minRows(%d)",
     REPO_ID(repo),
     pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2,
-    pRCfg->cacheLastRow, pRCfg->totalBlocks);
-  tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)",
+    pRCfg->cacheLastRow, pRCfg->totalBlocks, pRCfg->minRowsPerFileBlock);
+  tsdbInfo("vgId:%d new config: compression(%d),keep(%d,%d,%d),cacheLastRow(%d),totalBlocks(%d),minRows(%d)",
     REPO_ID(repo),
     pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2,
-    pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks);
+    pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks,pSaveCfg->minRowsPerFileBlock);
 
   repo->config_changed = true;
 
@@ -641,6 +645,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
     // tsdbFreeMemTable(pRepo->imem);
     tsem_destroy(&(pRepo->readyToCommit));
     pthread_mutex_destroy(&pRepo->mutex);
+    pthread_mutex_destroy(&pRepo->save_mutex);
     free(pRepo);
   }
 }

src/util/inc/tconfig.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define TSDB_CFG_MAX_NUM    137
+#define TSDB_CFG_MAX_NUM    141
 #define TSDB_CFG_PRINT_LEN  23
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41

src/util/src/tutil.c

@@ -225,66 +225,59 @@ char *tstrstr(char *src, char *dst, bool ignoreInEsc) {
 }
 
 char* strtolower(char *dst, const char *src) {
-  int esc = 0;
-  char quote = 0, *p = dst, c;
-
-  assert(dst != NULL);
+  if (src == NULL || dst == NULL) {
+    return dst;
+  }
+  char* const ret = dst;
+  while (*src) {
+    const char ch = *(src++);
+    *(dst++) = (ch >= 'A' && ch <= 'Z') ? ch - 'A' + 'a' : ch;
          
-  for (c = *src++; c; c = *src++) {
-    if (esc) {
-      esc = 0;
-    } else if (quote) {
-      if (c == '\\') {
-        esc = 1;
-      } else if (c == quote) {
-        quote = 0;
+    if (ch == '\'' || ch == '"') {
+      char prev = ch;
+      while (*src) {
+        const char next = *(src++);
+        *(dst++) = next;
+        if (prev != '\\' && next == ch) break;
+        prev = next;
       }
-    } else if (c >= 'A' && c <= 'Z') {
-      c -= 'A' - 'a';
-    } else if (c == '\'' || c == '"') {
-      quote = c;
+    } else if (ch == '`') {
+      while (*src) {
+        const char next = *(src++);
+        *(dst++) = next;
+        if (next == ch) break;
       }
-    *p++ = c;
     }
-
-  *p = 0;
-  return dst;
+  }
+  *(dst) = 0;
+  return ret;
 }
 
 char* strntolower(char *dst, const char *src, int32_t n) {
-  int esc = 0, inEsc = 0;
-  char quote = 0, *p = dst, c;
-
   assert(dst != NULL);
-  if (n == 0) {
-    *p = 0;
-    return dst;
+  char* const end = dst + n;
+  while (dst != end) {
+    const char ch = *(src++);
+    *(dst++) = (ch >= 'A' && ch <= 'Z') ? ch - 'A' + 'a' : ch;
+    
+    if (ch == '\'' || ch == '"') {
+      char prev = ch;
+      while (dst != end) {
+        const char next = *(src++);
+        *(dst++) = next;
+        if (prev != '\\' && next == ch) break;
+        prev = next;
       }
-  for (c = *src++; n-- > 0; c = *src++) {
-    if (esc) {
-      esc = 0;
-    } else if (quote) {
-      if (c == '\\') {
-        esc = 1;
-      } else if (c == quote) {
-        quote = 0;
+    } else if (ch == '`') {
+      while (dst != end) {
+        const char next = *(src++);
+        *(dst++) = next;
+        if (next == ch) break;
       }
-    } else if (inEsc) {
-      if (c == '`') {
-        inEsc = 0;
     }
-    } else if (c >= 'A' && c <= 'Z') {
-      c -= 'A' - 'a';
-    } else if (inEsc == 0 && (c == '\'' || c == '"')) {
-      quote = c;
-    } else if (c == '`' && quote == 0) {
-      inEsc = 1;
   }
-    *p++ = c;
-  }
-
-  *p = 0;
-  return dst;
+  *(dst) = 0;
+  return dst - n;
 }
 
 char* strntolower_s(char *dst, const char *src, int32_t n) {

src/vnode/src/vnodeWrite.c

@@ -161,6 +161,8 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
 
   vTrace("vgId:%d, submit msg is processed", pVnode->vgId);
 
+  int64_t submitStartUs = taosGetTimestampUs();
+
   if (pVnode->dbType == TSDB_DB_TYPE_TOPIC && pVnode->role == TAOS_SYNC_ROLE_MASTER) {
     tpUpdateTs(pVnode->vgId, &pVnode->sequence, pCont);
   }
@@ -186,6 +188,11 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR
     atomic_fetch_add_64(&tsSubmitRowSucNum, ntohl(pRsp->affectedRows));
   }
 
+  int64_t submitEndUs = taosGetTimestampUs();
+  if (submitEndUs - submitStartUs > 10 * 1000000) {
+    vWarn("vgId: %d, submit msg process takes more than 10s", pVnode->vgId);
+  }
+
   return code;
 }
 

tests/develop-test/2-query/function_mode.py

@@ -57,10 +57,10 @@ class TDTestCase:
         tdSql.checkData(0, 1, 2)
         tdSql.query('select mode(num) from d001')
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, -32767)
         tdSql.query('select mode(dbool) from d001')
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, False)
         tdSql.query('select mode(dtiny) from d001')
         tdSql.checkRows(1)
         tdSql.checkData(0, 0, None)
@@ -87,7 +87,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 1)
         tdSql.query('select mode(num),mode(voltage) from smode')
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 3276)
         tdSql.checkData(0, 1, 1)
         tdSql.query('select mode(dbool) from smode')
         tdSql.checkRows(1)
@@ -97,7 +97,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, None)
         tdSql.query('select mode(dfloat) from smode')
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 3.3232219219207764)
         tdSql.query('select mode(ddouble) from smode')
         tdSql.checkRows(1)
         tdSql.checkData(0, 0, 4.982392323)
@@ -112,16 +112,16 @@ class TDTestCase:
         #group by column
         tdSql.query('select mode(num) from d002 group by dbinary')
         tdSql.checkRows(2)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 3276)
         tdSql.checkData(1, 0, None)
         tdSql.execute('insert into D002 values("2021-11-17 20:31:31", 1, 3276, true, NULL, 3.32322, 4.982392323, "你好吗", "sdf", 333)')
         tdSql.query('select mode(num) from d002 group by dbinary')
         tdSql.checkRows(2)
         tdSql.checkData(1, 0, 3276)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 3276)
         tdSql.query('select mode(dfloat) from d002 group by dbinary')
         tdSql.checkRows(2)
-        tdSql.checkData(1, 0, None)
+        tdSql.checkData(1, 0, 3.3232200145721436)
         tdSql.query('select mode(dchar) from d002 group by dbinary')
         tdSql.checkRows(2)
         tdSql.checkData(1, 0, "你好吗")
@@ -138,7 +138,7 @@ class TDTestCase:
         #group by tbname
         tdSql.query('select mode(dchar) from smode group by tbname')
         tdSql.checkRows(2)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, "你好吗")
         tdSql.checkData(0, 1, "d001")
         tdSql.checkData(1, 0, "你好吗")
         tdSql.checkData(1, 1, "d002")
@@ -148,7 +148,7 @@ class TDTestCase:
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, 4.982392323)
         tdSql.checkData(0, 1, "Beijing.haidian")
-        tdSql.checkData(1, 0, None)
+        tdSql.checkData(1, 0, 4.982392323)
         tdSql.checkData(1, 1, "Beijing.Chaoyang")
 
         #group by tag,column
@@ -169,7 +169,7 @@ class TDTestCase:
         tdSql.checkRows(2)
         tdSql.checkData(0, 0, "你好吗")
         tdSql.checkData(0, 1, "d002")
-        tdSql.checkData(1, 0, None)
+        tdSql.checkData(1, 0, "你好吗")
         tdSql.checkData(1, 1, "d001")
 
         #where
@@ -178,7 +178,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 19)
         tdSql.query('select mode(voltage) from smode where num > 9')
         tdSql.checkRows(1)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 19)
 
         #interval
         tdSql.query('select mode(voltage) from smode interval(1s)')
@@ -188,7 +188,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, "2021-01-01 00:00:00")
         tdSql.checkData(0, 1, 1)
         tdSql.checkData(1, 0, "2022-01-01 00:00:00")
-        tdSql.checkData(1, 1, None)
+        tdSql.checkData(1, 1, 19)
         tdSql.query('select mode(voltage) from smode interval(1n)')
         tdSql.checkRows(4)
         tdSql.checkData(0, 0, "2021-10-01 00:00:00")
@@ -198,7 +198,7 @@ class TDTestCase:
         tdSql.checkData(2, 0, "2021-12-01 00:00:00")
         tdSql.checkData(2, 1, 2)
         tdSql.checkData(3, 0, "2022-01-01 00:00:00")
-        tdSql.checkData(3, 1, None)
+        tdSql.checkData(3, 1, 19)
 
         tdSql.query('select mode(voltage) from smode where ts > "2021-09-01 00:00:00" and ts <"2022-02-02 00:00:00" interval(1n) fill(prev)')
         tdSql.checkRows(6)
@@ -207,7 +207,7 @@ class TDTestCase:
         tdSql.checkData(3, 0, "2021-12-01 00:00:00")
         tdSql.checkData(3, 1, 2)
         tdSql.checkData(5, 0, "2022-02-01 00:00:00")
-        tdSql.checkData(5, 1, 2)
+        tdSql.checkData(5, 1, 19)
 
         #session
         tdSql.query('select mode(voltage) from d002 session(ts,1w)')
@@ -222,7 +222,7 @@ class TDTestCase:
         #state_window
         tdSql.query('select mode(dfloat) from d002 state_window(voltage)')
         tdSql.checkRows(3)
-        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 0, 3.3232200145721436)
         tdSql.checkData(1, 0, None)
 
         #slimit/soffset
@@ -237,9 +237,9 @@ class TDTestCase:
 
         #having
         tdSql.query('select mode(ddouble) from smode group by location having mode(ddouble)>3')
-        tdSql.checkRows(1)
+        tdSql.checkRows(2)
         tdSql.checkData(0, 0, 4.982392323)
-        tdSql.checkData(0, 1, "Beijing.haidian")
+        tdSql.checkData(0, 1, "Beijing.Chaoyang")
 
         #subquery
         tdSql.query('select mode(ddouble) from (select * from smode where voltage = 1)')

tests/pytest/query/query1970YearsAf.py

@@ -25,8 +25,8 @@ from util.cases import *
 from util.dnodes import *
 from util.dnodes import TDDnode
 
-class TDTestCase:
 
+class TDTestCase:
     def __init__(self):
         self.path = ""
 
@@ -68,7 +68,7 @@ class TDTestCase:
             "cachelast": 0,
             "quorum": 1,
             "fsync": 3000,
-            "update": 0
+            "update": 0,
         }
 
         # set stable schema
@@ -99,8 +99,7 @@ class TDTestCase:
                 {"type": "TINYINT", "count": 2},
                 {"type": "BOOL", "count": 2},
                 {"type": "NCHAR", "len": 3, "count": 1},
-                {"type": "BINARY", "len": 8, "count": 1}
-
+                {"type": "BINARY", "len": 8, "count": 1},
             ],
             "tags": [
                 {"type": "INT", "count": 2},
@@ -111,17 +110,14 @@ class TDTestCase:
                 {"type": "TINYINT", "count": 2},
                 {"type": "BOOL", "count": 2},
                 {"type": "NCHAR", "len": 3, "count": 1},
-                {"type": "BINARY", "len": 8, "count": 1}
-            ]
+                {"type": "BINARY", "len": 8, "count": 1},
+            ],
         }
 
         # create different stables like stable1 and add to list super_tables
         super_tables = []
         super_tables.append(stable1)
-        database = {
-            "dbinfo": dbinfo,
-            "super_tables": super_tables
-        }
+        database = {"dbinfo": dbinfo, "super_tables": super_tables}
 
         cfgdir = self.getCfgDir()
         create_table = {
@@ -137,33 +133,60 @@ class TDTestCase:
             "confirm_parameter_prompt": "no",
             "insert_interval": 0,
             "num_of_records_per_req": 100,
-            "databases": [database]
+            "databases": [database],
         }
         return create_table
 
+    def getPath(self, tool="taosBenchmark"):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if "community" in selfPath:
+            projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
+        elif "/tests/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tests/")]
+        else:
+            tdLog.exit("path: %s is not supported" % selfPath)
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if (tool) in files:
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if "packaging" not in rootRealPath:
+                    paths.append(os.path.join(root, tool))
+                    break
+        if len(paths) == 0:
+            return ""
+        return paths[0]
+
     def createinsertfile(self):
         create_table = self.creatcfg()
         date = datetime.datetime.now().strftime("%Y%m%d%H%M")
         file_create_table = f"/tmp/insert_{date}.json"
 
-        with open(file_create_table, 'w') as f:
+        with open(file_create_table, "w") as f:
             json.dump(create_table, f)
         return file_create_table
 
     def inserttable(self, filepath):
-        create_table_cmd = f"taosdemo -f {filepath}  > /dev/null 2>&1"
+        binPath = self.getPath("taosBenchmark")
+        if binPath == "":
+            tdLog.exit("taosBenchmark not found!")
+        else:
+            tdLog.info("taosBenchmark found in %s" % binPath)
+
+        create_table_cmd = "%s -f %s  > /dev/null 2>&1" % (binPath, filepath)
         _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8")
 
     def sqlsquery(self):
         # stable query
-        tdSql.query(
-            "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(43200)
 
-        tdSql.query(
-            "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(6800)
 
         tdSql.query(
@@ -172,14 +195,10 @@ class TDTestCase:
         tdSql.checkRows(3590)
 
         # child-tables query
-        tdSql.query(
-            "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from t0 where t0.ts < '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(4320)
 
-        tdSql.query(
-            "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(680)
 
         tdSql.query(
@@ -192,9 +211,7 @@ class TDTestCase:
         )
         tdSql.checkRows(680)
 
-        tdSql.query(
-            "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(679)
 
         tdSql.query(
@@ -203,9 +220,7 @@ class TDTestCase:
         tdSql.checkRows(43200)
 
         # query with timestamp in 'where ...'
-        tdSql.query(
-            "select * from stb2 where stb2.ts > -28800000 "
-        )
+        tdSql.query("select * from stb2 where stb2.ts > -28800000 ")
         tdSql.checkRows(6790)
 
         tdSql.query(
@@ -219,14 +234,16 @@ class TDTestCase:
         tdSql.checkRows(3590)
 
     def run(self):
-        s = 'reset query cache'
+        s = "reset query cache"
         tdSql.execute(s)
-        s = 'create database if not exists db'
+        s = "create database if not exists db"
         tdSql.execute(s)
-        s = 'use db'
+        s = "use db"
         tdSql.execute(s)
 
-        tdLog.info("==========step1:create table stable and child table,then insert data automatically")
+        tdLog.info(
+            "==========step1:create table stable and child table,then insert data automatically"
+        )
         insertfile = self.createinsertfile()
         self.inserttable(insertfile)
 

tests/pytest/query/queryInterval.py

@@ -197,6 +197,25 @@ class TDTestCase:
         else:
             tdLog.exit("sql:%s, column : ts is not sorted in accending order as expected" % (tdSql.sql))
 
+        # TS-1582
+        tdSql.execute("create table stb1(ts timestamp, c1 int) tags(t1 int)")
+        tdSql.execute("insert into tb1 using stb1 tags(1) values(%d, 1)(%d, 5)(%d, 10)" % (self.ts, self.ts + 1000, self.ts + 2000))
+        tdSql.execute("insert into tb2 using stb1 tags(2) values(%d, 1)(%d, 5)(%d, 10)" % (self.ts, self.ts + 1000, self.ts + 2000))
+
+        tdSql.query("select t1, last(c1) from stb1 where ts between 1593548685000 and 1593548688000 interval(2s) fill(NULL) group by tbname")
+        tdSql.checkRows(6)
+        tdSql.checkData(2, 2, None)
+        tdSql.checkData(2, 3, "tb1")
+        tdSql.checkData(5, 2, None)
+        tdSql.checkData(5, 3, "tb2")
+
+        tdSql.query("select t1, last(c1) - 1 from stb1 where ts between 1593548685000 and 1593548688000 interval(2s) fill(NULL) group by tbname")
+        tdSql.checkRows(6)
+        tdSql.checkData(2, 2, None)
+        tdSql.checkData(2, 3, "tb1")
+        tdSql.checkData(5, 2, None)
+        tdSql.checkData(5, 3, "tb2")
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/util/dnodes.py

@@ -142,6 +142,7 @@ class TDDnode:
             "numOfMnodes": "3",
             "numOfThreadsPerCore": "2.0",
             "monitor": "0",
+            "audit": "0",
             "maxVnodeConnections": "30000",
             "maxMgmtConnections": "30000",
             "maxMeterConnections": "30000",

tests/script/general/alter/column_width_commit.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
+system sh/exec.sh -n dnode1 -s start
+
+sleep 2000
+sql connect
+
+print ======== step1 create db 
+sql drop database if exists db1;
+sql CREATE DATABASE db1;
+sql use db1;
+
+print ======== step2 create normal table  
+sql CREATE TABLE normal_t1('ts' TIMESTAMP, 'c1' BINARY(1));
+
+print ======== step3 alter column width 
+sql alter table db1.normal_t1 modify column c1 binary(2400);
+
+print ======== step4 insert into 20 rows
+$ms = 0
+$cnt = 0
+while $cnt < 20
+   $ms = $cnt . m
+   sql insert into normal_t1(ts,c1) values(now + $ms , '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789'); 
+   $cnt = $cnt + 1
+ endw  
+
+########################################################
+print ===> step 4 stop dnode1
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+
+########################################################
+print ===> step 5 start dnode1
+system sh/exec.sh -n dnode1 -s start
+
+sleep 1000
+
+sql select count(*) from db1.normal_t1;
+print $data00
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 20 then
+  return -1
+endi
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/general/db/alter_option.sim

@@ -9,7 +9,7 @@ sleep 2000
 sql connect
 
 print ============= create database
-sql create database db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1
+sql create database db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 1200 ctime 120 precision 'ms' comp 2 wal 1 replica 1
 sql show databases
 if $data00 != db then 
   return -1
@@ -182,9 +182,13 @@ sql_error alter database db blocks -1
 sql_error alter database db blocks 10001
 
 print ============== step minrows
+sql_error alter database db minrows 0
 sql_error alter database db minrows 1
-sql_error alter database db minrows 100
-sql_error alter database db minrows 1000
+sql_error alter database db minrows -1
+sql_error alter database db minrows 9
+sql_error alter database db minrows 1001
+sql alter database db minrows 100
+sql alter database db minrows 1000
 
 print ============== step maxrows
 sql_error alter database db maxrows 1

tests/script/general/db/topic1.sim

@@ -471,8 +471,9 @@ sql_error alter topic db blocks 10001
 
 print ============== step minrows
 sql_error alter database db minrows 1
-sql_error alter database db minrows 100
-sql_error alter database db minrows 1000
+sql alter database db minrows 100
+sql alter database db minrows 399
+sql_error alter database db minrows 1001
 
 sql_error alter topic db minrows 1
 sql_error alter topic db minrows 100

tests/script/jenkins/basic.txt

@@ -193,6 +193,7 @@ cd ../../../debug; make
 ./test.sh -f general/alter/insert2.sim
 ./test.sh -f general/alter/metrics.sim
 ./test.sh -f general/alter/table.sim
+./test.sh -f general/alter/column_width_commit.sim
 ./test.sh -f general/cache/new_metrics.sim
 ./test.sh -f general/cache/restart_metrics.sim
 ./test.sh -f general/cache/restart_table.sim

tests/script/sh/deploy.bat

@@ -85,6 +85,7 @@ echo wdebugFlag             135                >> %TAOS_CFG%
 echo cqdebugFlag            135                >> %TAOS_CFG%
 echo monitor                0                  >> %TAOS_CFG%
 echo monitorInterval        1                  >> %TAOS_CFG%
+echo audit                  0                  >> %TAOS_CFG%
 echo http                   0                  >> %TAOS_CFG%
 echo slaveQuery             0                  >> %TAOS_CFG%
 echo numOfThreadsPerCore    2.0                >> %TAOS_CFG%

tests/script/sh/deploy.sh

@@ -139,6 +139,7 @@ echo "wdebugFlag             143"                >> $TAOS_CFG
 echo "cqdebugFlag            143"                >> $TAOS_CFG
 echo "monitor                0"                  >> $TAOS_CFG
 echo "monitorInterval        1"                  >> $TAOS_CFG
+echo "audit                  0"                  >> $TAOS_CFG
 echo "http                   0"                  >> $TAOS_CFG
 echo "slaveQuery             0"                  >> $TAOS_CFG
 echo "numOfThreadsPerCore    2.0"                >> $TAOS_CFG

tests/system-test/0-others/audit.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import os
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+
+    updatecfgDict = {'audit': 1}
+
+    def caseDescription(self):
+        '''
+        TS-1887 Create Audit db for DDL storage
+        '''
+        return
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        now = time.time()
+        self.ts = int(round(now * 1000))
+
+    def run(self):
+
+        #tdSql.prepare()
+        time.sleep(3)
+
+        print("==============step1 test CREATE DDL")
+
+        # CREATE DATABASE
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CREATE DATABASE')
+        tdSql.checkData(0, 4, '0.db')
+        tdSql.checkData(0, 5, 'success')
+
+        # CREATE NORMAL TABLE
+        tdSql.execute("create table tb (ts timestamp, c0 int)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CREATE TABLE')
+        tdSql.checkData(0, 4, '0.db.tb')
+        tdSql.checkData(0, 5, 'success')
+
+        # CREATE SUPER TABLE
+        tdSql.execute("create table stb (ts timestamp, c0 int) tags (t0 int)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CREATE SUPER TABLE')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # CREATE CHILD TABLE
+        tdSql.execute("create table ctb using stb tags (1)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CREATE CHILD TABLE')
+        tdSql.checkData(0, 4, '0.db.ctb')
+        tdSql.checkData(0, 5, 'success')
+
+        # CREATE CHILD TABLE(AUTO)
+        tdSql.execute("insert into ctb_auto using stb tags (2) values (now, 2)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CREATE CHILD TABLE')
+        tdSql.checkData(0, 4, '0.db.ctb_auto')
+        tdSql.checkData(0, 5, 'success')
+
+        print("==============step2 test ALTER DDL")
+
+        # ALTER ATABASE
+        tdSql.execute("alter database db keep 354")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'ALTER DATABASE')
+        tdSql.checkData(0, 4, '0.db')
+        tdSql.checkData(0, 5, 'success')
+
+        tdSql.execute("alter database db cachelast 1")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'ALTER DATABASE')
+        tdSql.checkData(0, 4, '0.db')
+        tdSql.checkData(0, 5, 'success')
+
+        # ADD COLUMN NORMAL TABLE
+        tdSql.execute("alter table tb add column c1 binary(4)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'ADD COLUMN')
+        tdSql.checkData(0, 4, '0.db.tb')
+        tdSql.checkData(0, 5, 'success')
+
+        # MODIFY COLUMN NORMAL TABLE
+        tdSql.execute("alter table tb modify column c1 binary(10)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'MODIFY COLUMN/TAG LENGTH')
+        tdSql.checkData(0, 4, '0.db.tb')
+        tdSql.checkData(0, 5, 'success')
+
+        # ADD COLUMN SUPER TABLE
+        tdSql.execute("alter table stb add column c1 binary(4)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'ADD COLUMN')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # ADD TAG SUPER TABLE
+        tdSql.execute("alter table stb add tag t1 binary(4)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'ADD TAG')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # MODIFY COLUMN SUPER TABLE
+        tdSql.execute("alter table stb modify column c1 binary(10)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'MODIFY COLUMN/TAG LENGTH')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # MODIFY TAG SUPER TABLE
+        tdSql.execute("alter table stb modify tag t1 binary(10)")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'MODIFY COLUMN/TAG LENGTH')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # CHANGE TAG NAME SUPER TABLE
+        tdSql.execute("alter table stb change tag t1 t2")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'CHANGE TAG NAME')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+
+        print("==============step3 test DROP DDL")
+        # DROP COLUMN NORMAL TABLE
+        tdSql.execute("alter table tb drop column c1")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP COLUMN')
+        tdSql.checkData(0, 4, '0.db.tb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP COLUMN SUPER TABLE
+        tdSql.execute("alter table stb drop column c1")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP COLUMN')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP TAG SUPER TABLE
+        tdSql.execute("alter table stb drop tag t2")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP TAG')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP NORMAL TABLE
+        tdSql.execute("drop table tb")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP TABLE')
+        tdSql.checkData(0, 4, '0.db.tb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP CHILD TABLE
+        tdSql.execute("drop table ctb")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP CHILD TABLE')
+        tdSql.checkData(0, 4, '0.db.ctb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP SUPER TABLE
+        tdSql.execute("drop table stb")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP SUPER TABLE')
+        tdSql.checkData(0, 4, '0.db.stb')
+        tdSql.checkData(0, 5, 'success')
+
+        # DROP DATABASE
+        tdSql.execute("drop database db")
+
+        tdSql.query("select last(*) from audit.ddl");
+        tdSql.checkData(0, 3, 'DROP DATABASE')
+        tdSql.checkData(0, 4, '0.db')
+        tdSql.checkData(0, 5, 'success')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py

@@ -24,36 +24,44 @@ class TDTestCase:
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
-    def getBuildPath(self):
+    def getPath(self, tool="taosBenchmark"):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
-        if ("community" in selfPath):
-            projPath = selfPath[:selfPath.find("community")]
+        if "community" in selfPath:
+            projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
+        elif "/tests/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tests/")]
         else:
-            projPath = selfPath[:selfPath.find("tests")]
+            tdLog.exit("path: %s is not supported" % selfPath)
 
+        paths = []
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if (tool) in files:
                 rootRealPath = os.path.dirname(os.path.realpath(root))
-                if ("packaging" not in rootRealPath):
-                    buildPath = root[:len(root)-len("/build/bin")]
+                if "packaging" not in rootRealPath:
+                    paths.append(os.path.join(root, tool))
                     break
-        return buildPath
+        if len(paths) == 0:
+            return ""
+        return paths[0]
 
     def run(self):
-        buildPath = self.getBuildPath()
-        if (buildPath == ""):
-            tdLog.exit("taosd not found!")
+        binPath = self.getPath()
+        if binPath == "":
+            tdLog.exit("taosBenchmark not found!")
         else:
-            tdLog.info("taosd found in %s" % buildPath)
-        binPath = buildPath+ "/build/bin/"
+            tdLog.info("taosBenchmark found in %s" % binPath)
 
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert*_res.txt*")
-        os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename )         
+        os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename)
 
         # insert: create one  or mutiple tables per sql and insert multiple rows per sql
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 11)
@@ -69,7 +77,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 2000)
 
         # restful connector insert data
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 10)
@@ -86,17 +94,17 @@ class TDTestCase:
 
         # default values json files
         tdSql.execute("drop database if exists db")
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath)
         tdSql.query("show databases;")
         for i in range(tdSql.queryRows):
-            if tdSql.queryResult[i][0] == 'db':
+            if tdSql.queryResult[i][0] == "db":
                 tdSql.checkData(i, 2, 100)
                 tdSql.checkData(i, 4, 1)
                 tdSql.checkData(i, 6, 10)
-                tdSql.checkData(i, 16, 'ms')           
+                tdSql.checkData(i, 16, "ms")
 
         # insert: create  mutiple tables per sql and insert one rows per sql .
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 10)
@@ -113,7 +121,9 @@ class TDTestCase:
 
         # insert: using parament "insert_interval to controls spped  of insert.
         # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath)
+        os.system(
+            "%s -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath
+        )
         tdSql.execute("use db")
         tdSql.query("show stables")
         tdSql.checkData(0, 4, 10)
@@ -131,11 +141,6 @@ class TDTestCase:
         # rm useless files
         os.system("rm -rf ./insert*_res.txt*")
 
-
-        
-        
-        
-
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/system-test/fulltest-others.sh

 python3 ./test.py -f 0-others/create_col_tag.py
+python3 ./test.py -f 0-others/audit.py