Merge branch 'feature/TD-4399' of https://github.com/taosdata/TDengine into feature/TD-4399

documentation20/cn/03.architecture/02.replica/docs.md

@@ -107,9 +107,9 @@ TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性
 
 ![replica-forward.png](page://images/architecture/replica-forward.png)
 
-1. 应用对写请求做基本的合法性检查，通过，则给改请求包打上一个版本号(version, 单调递增）
+1. 应用对写请求做基本的合法性检查，通过，则给该请求包打上一个版本号(version, 单调递增）
 2. 应用将打上版本号的写请求封装一个WAL Head, 写入WAL(Write Ahead Log)
-3. 应用调用API syncForwardToPeer，如多vnode B是slave状态，sync模块将包含WAL Head的数据包通过Forward消息发送给vnode B，否则就不转发。
+3. 应用调用API syncForwardToPeer，如果vnode B是slave状态，sync模块将包含WAL Head的数据包通过Forward消息发送给vnode B，否则就不转发。
 4. vnode B收到Forward消息后，调用回调函数writeToCache, 交给应用处理
 5. vnode B应用在写入成功后，都需要调用syncAckForward通知sync模块已经写入成功。
 6. 如果quorum大于1，vnode B需要等待应用的回复确认，收到确认后，vnode B发送Forward Response消息给node A。
@@ -219,7 +219,7 @@ Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系
 
 不同之处：
 
-- 选举流程不一样：Raft里任何一个节点是candidate时，主动向其他节点发出vote request, 如果超过半数回答Yes, 这个candidate就成为Leader,开始一个新的term. 而TDengine的实现里，节点上线、离线或角色改变都会触发状态消息在节点组类传播，等节点组里状态稳定一致之后才触发选举流程，因为状态稳定一致，基于同样的状态信息，每个节点做出的决定会是一致的，一旦某个节点符合成为master的条件，无需其他节点认可，它会自动将自己设为master。TDengine里，任何一个节点检测到其他节点或自己的角色发生改变，就会给节点组内其他节点进行广播的。Raft里不存在这样的机制，因此需要投票来解决。
+- 选举流程不一样：Raft里任何一个节点是candidate时，主动向其他节点发出vote request，如果超过半数回答Yes，这个candidate就成为Leader，开始一个新的term。而TDengine的实现里，节点上线、离线或角色改变都会触发状态消息在节点组内传播，等节点组里状态稳定一致之后才触发选举流程，因为状态稳定一致，基于同样的状态信息，每个节点做出的决定会是一致的，一旦某个节点符合成为master的条件，无需其他节点认可，它会自动将自己设为master。TDengine里，任何一个节点检测到其他节点或自己的角色发生改变，就会向节点组内其他节点进行广播。Raft里不存在这样的机制，因此需要投票来解决。
 - 对WAL的一条记录，Raft用term + index来做唯一标识。但TDengine只用version（类似index)，在TDengine实现里，仅仅用version是完全可行的, 因为TDengine的选举机制，没有term的概念。
 
 如果整个虚拟节点组全部宕机，重启，但不是所有虚拟节点都上线，这个时候TDengine是不会选出master的，因为未上线的节点有可能有最高version的数据。而RAFT协议，只要超过半数上线，就会选出Leader。

documentation20/cn/03.architecture/docs.md

@@ -343,7 +343,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久
 
 对于采集的数据，一般有保留时长，这个时长由系统配置参数keep决定。超过这个设置天数的数据文件，将被系统自动删除，释放存储空间。
 
-给定days与keep两个参数，一个vnode总的数据文件数为：keep/days。总的数据文件个数不宜过大，也不宜过小。10到100以内合适。基于这个原则，可以设置合理的days。 目前的版本，参数keep可以修改，但对于参数days，一但设置后，不可修改。
+给定days与keep两个参数，一个典型工作状态的vnode中总的数据文件数为：`向上取整(keep/days)+1`个。总的数据文件个数不宜过大，也不宜过小。10到100以内合适。基于这个原则，可以设置合理的days。 目前的版本，参数keep可以修改，但对于参数days，一但设置后，不可修改。
 
 在每个数据文件里，一张表的数据是一块一块存储的。一张表可以有一到多个数据文件块。在一个文件块里，数据是列式存储的，占用的是一片连续的存储空间，这样大大提高读取速度。文件块的大小由系统参数maxRows（每块最大记录条数）决定，缺省值为4096。这个值不宜过大，也不宜过小。过大，定位具体时间段的数据的搜索时间会变长，影响读取速度；过小，数据块的索引太大，压缩效率偏低，也影响读取速度。
 

documentation20/cn/08.connector/docs.md

@@ -516,7 +516,7 @@ conn.close()
 - _TDengineCursor_ 类
 
   参考python中help(taos.TDengineCursor)。
-  这个类对应客户端进行的写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能夸线程共享使用，否则会导致返回结果出现错误。
+  这个类对应客户端进行的写入、查询操作。在客户端多线程的场景下，这个游标实例必须保持线程独享，不能跨线程共享使用，否则会导致返回结果出现错误。
 
 - _connect_ 方法
 

documentation20/cn/12.taos-sql/docs.md

@@ -37,7 +37,7 @@ taos> DESCRIBE meters;
 - Epoch Time：时间戳也可以是一个长整数，表示从 1970-01-01 08:00:00.000 开始的毫秒数
 - 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降频操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n(自然月) 和 y(自然年)。
 
-TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableMicrosecond 就可以支持微秒。
+TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传递的 PRECISION 参数就可以支持微秒。
 
 在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 
 
@@ -400,6 +400,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableM
                 tb2_name (tb2_field1_name, ...) [USING stb2_name TAGS (tag_value2, ...)] VALUES (field1_value1, ...) (field1_value2, ...) ...;
     ```
     以自动建表的方式，同时向表tb1_name和tb2_name中按列分别插入多条记录。  
+    说明：`(tb1_field1_name, ...)`的部分可以省略掉，这样就是使用全列模式写入——也即在 VALUES 部分提供的数据，必须为数据表的每个列都显式地提供数据。全列写入速度会远快于指定列，因此建议尽可能采用全列写入方式，此时空列可以填入NULL。  
     从 2.0.20.5 版本开始，子表的列名可以不跟在子表名称后面，而是可以放在 TAGS 和 VALUES 之间，例如像下面这样写：
     ```mysql
     INSERT INTO tb1_name [USING stb1_name TAGS (tag_value1, ...)] (tb1_field1_name, ...) VALUES (field1_value1, ...) (field1_value2, ...) ...;
@@ -423,9 +424,9 @@ Query OK, 1 row(s) in set (0.001029s)
 taos> SHOW TABLES;
 Query OK, 0 row(s) in set (0.000946s)
 
-taos> INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2);
+taos> INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a');
 
-DB error: invalid SQL: keyword VALUES or FILE required
+DB error: invalid SQL: 'a' (invalid timestamp) (0.039494s)
 
 taos> SHOW TABLES;
            table_name           |      created_time       | columns |          stable_name           |

src/client/inc/tscLocalMerge.h → src/client/inc/tscGlobalmerge.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSCLOCALMERGE_H
-#define TDENGINE_TSCLOCALMERGE_H
+#ifndef TDENGINE_TSCGLOBALMERGE_H
+#define TDENGINE_TSCGLOBALMERGE_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,7 +24,7 @@ extern "C" {
 #include "qFill.h"
 #include "taosmsg.h"
 #include "tlosertree.h"
-#include "tsclient.h"
+#include "qExecutor.h"
 
 #define MAX_NUM_OF_SUBQUERY_RETRY 3
   
@@ -38,7 +38,7 @@ typedef struct SLocalDataSource {
   tFilePage      filePage;
 } SLocalDataSource;
 
-typedef struct SLocalMerger {
+typedef struct SGlobalMerger {
   SLocalDataSource     **pLocalDataSrc;
   int32_t                numOfBuffer;
   int32_t                numOfCompleted;
@@ -48,20 +48,22 @@ typedef struct SLocalMerger {
   tOrderDescriptor      *pDesc;
   tExtMemBuffer        **pExtMemBuffer;    // disk-based buffer
   char                  *buf;              // temp buffer
-} SLocalMerger;
+} SGlobalMerger;
+
+struct SSqlObj;
 
 typedef struct SRetrieveSupport {
   tExtMemBuffer **  pExtMemBuffer;     // for build loser tree
   tOrderDescriptor *pOrderDescriptor;
   int32_t           subqueryIndex;     // index of current vnode in vnode list
-  SSqlObj *         pParentSql;
+  struct SSqlObj   *pParentSql;
   tFilePage *       localBuffer;       // temp buffer, there is a buffer for each vnode to
   uint32_t          numOfRetry;        // record the number of retry times
 } SRetrieveSupport;
 
-int32_t tscLocalReducerEnvCreate(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id);
+int32_t tscCreateGlobalMergerEnv(SQueryInfo* pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub, tOrderDescriptor **pDesc, uint32_t nBufferSize, int64_t id);
 
-void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes);
+void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes);
 
 int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePage *pPage, void *data,
                      int32_t numOfRows, int32_t orderType);
@@ -71,13 +73,13 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF
 /*
  * create local reducer to launch the second-stage reduce process at client site
  */
-int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                          SQueryInfo *pQueryInfo, SLocalMerger **pMerger, int64_t id);
+int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
+                          SQueryInfo *pQueryInfo, SGlobalMerger **pMerger, int64_t id);
 
-void tscDestroyLocalMerger(SLocalMerger* pLocalMerger);
+void tscDestroyGlobalMerger(SGlobalMerger* pMerger);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_TSCLOCALMERGE_H
+#endif  // TDENGINE_TSCGLOBALMERGE_H

src/client/inc/tscUtil.h

@@ -20,13 +20,13 @@
 extern "C" {
 #endif
 
-#include "tsched.h"
 #include "exception.h"
 #include "os.h"
 #include "qExtbuffer.h"
 #include "taosdef.h"
 #include "tbuffer.h"
-#include "tscLocalMerge.h"
+#include "tscGlobalmerge.h"
+#include "tsched.h"
 #include "tsclient.h"
 
 #define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
@@ -63,7 +63,7 @@ typedef struct SJoinSupporter {
   SArray*         exprList;
   SFieldInfo      fieldsInfo;
   STagCond        tagCond;
-  SGroupbyExpr groupInfo;       // group by info
+  SGroupbyExpr    groupInfo;       // group by info
   struct STSBuf*  pTSBuf;          // the TSBuf struct that holds the compressed timestamp array
   FILE*           f;               // temporary file in order to create TSBuf
   char            path[PATH_MAX];  // temporary file path, todo dynamic allocate memory
@@ -110,7 +110,7 @@ void*   tscDestroyBlockArrayList(SArray* pDataBlockList);
 void*   tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
 
 int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
-int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap);
+int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap);
 int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta,
                                 STableDataBlocks** dataBlocks, SArray* pBlockList);
 
@@ -284,6 +284,7 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
 SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd);
 
 void registerSqlObj(SSqlObj* pSql);
+void tscInitResForMerge(SSqlRes* pRes);
 
 SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql);
 void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
@@ -328,12 +329,15 @@ SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
 int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr);
 
 void tsCreateSQLFunctionCtx(SQueryInfo* pQueryInfo, SQLFunctionCtx* pCtx, SSchema* pSchema);
-void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage);
+void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage);
 
 void* malloc_throw(size_t size);
 void* calloc_throw(size_t nmemb, size_t size);
 char* strdup_throw(const char* str);
 
+bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
+SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/client/inc/tschemautil.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TSCHEMAUTIL_H
-#define TDENGINE_TSCHEMAUTIL_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "taosmsg.h"
-#include "tsclient.h"
-#include "ttoken.h"
-
-/**
- * get the number of tags of this table
- * @param pTableMeta
- * @return
- */
-int32_t tscGetNumOfTags(const STableMeta* pTableMeta);
-
-/**
- * get the number of columns of this table
- * @param pTableMeta
- * @return
- */
-int32_t tscGetNumOfColumns(const STableMeta* pTableMeta);
-
-/**
- * get the basic info of this table
- * @param pTableMeta
- * @return
- */
-STableComInfo tscGetTableInfo(const STableMeta* pTableMeta);
-
-/**
- * get the schema
- * @param pTableMeta
- * @return
- */
-SSchema* tscGetTableSchema(const STableMeta* pTableMeta);
-
-/**
- * get the tag schema
- * @param pMeta
- * @return
- */
-SSchema *tscGetTableTagSchema(const STableMeta *pMeta);
-
-/**
- * get the column schema according to the column index
- * @param pMeta
- * @param colIndex
- * @return
- */
-SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
-
-/**
- * get the column schema according to the column id
- * @param pTableMeta
- * @param colId
- * @return
- */
-SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
-
-/**
- * create the table meta from the msg
- * @param pTableMetaMsg
- * @param size size of the table meta
- * @return
- */
-STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg);
-
-bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src);
-SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_TSCHEMAUTIL_H

src/client/inc/tsclient.h

@@ -40,17 +40,9 @@ extern "C" {
 
 // forward declaration
 struct SSqlInfo;
-struct SLocalMerger;
 
 typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int32_t numOfRows);
 
-typedef struct STableComInfo {
-  uint8_t numOfTags;
-  uint8_t precision;
-  int16_t numOfColumns;
-  int32_t rowSize;
-} STableComInfo;
-
 typedef struct SNewVgroupInfo {
   int32_t    vgId;
   int8_t     inUse;
@@ -66,34 +58,6 @@ typedef struct CChildTableMeta {
   uint64_t       suid;                              // super table id
 } CChildTableMeta;
 
-typedef struct STableMeta {
-  int32_t        vgId;
-  STableId       id;
-  uint8_t        tableType;
-  char           sTableName[TSDB_TABLE_FNAME_LEN];  // super table name
-  uint64_t       suid;       // super table id
-  int16_t        sversion;
-  int16_t        tversion;
-  STableComInfo  tableInfo;
-  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
-} STableMeta;
-
-typedef struct STableMetaInfo {
-  STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
-  uint32_t       tableMetaSize;
-  SVgroupsInfo  *vgroupList;
-  SArray        *pVgroupTables;   // SArray<SVgroupTableInfo>
-  
-  /*
-   * 1. keep the vgroup index during the multi-vnode super table projection query
-   * 2. keep the vgroup index for multi-vnode insertion
-   */
-  int32_t       vgroupIndex;
-  SName         name;
-  char          aliasName[TSDB_TABLE_NAME_LEN];    // alias name of table specified in query sql
-  SArray       *tagColList;                        // SArray<SColumn*>, involved tag columns
-} STableMetaInfo;
-
 typedef struct SColumnIndex {
   int16_t tableIndex;
   int16_t columnIndex;
@@ -111,44 +75,6 @@ typedef struct SInternalField {
   SExprInfo      *pExpr;
 } SInternalField;
 
-typedef struct SFieldInfo {
-  int16_t      numOfOutput;   // number of column in result
-  TAOS_FIELD*  final;
-  SArray      *internalField; // SArray<SInternalField>
-} SFieldInfo;
-
-typedef struct SCond {
-  uint64_t uid;
-  int32_t  len;  // length of tag query condition data
-  char *   cond;
-} SCond;
-
-typedef struct SJoinNode {
-  uint64_t uid;
-  int16_t  tagColId;
-  SArray*  tsJoin;
-  SArray*  tagJoin;
-} SJoinNode;
-
-typedef struct SJoinInfo {
-  bool       hasJoin;
-  SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM];
-} SJoinInfo;
-
-typedef struct STagCond {
-  // relation between tbname list and query condition, including : TK_AND or TK_OR
-  int16_t relType;
-
-  // tbname query condition, only support tbname query condition on one table
-  SCond tbnameCond;
-
-  // join condition, only support two tables join currently
-  SJoinInfo joinInfo;
-
-  // for different table, the query condition must be seperated
-  SArray *pCond;
-} STagCond;
-
 typedef struct SParamInfo {
   int32_t  idx;
   uint8_t  type;
@@ -191,57 +117,6 @@ typedef struct STableDataBlocks {
   SParamInfo *params;
 } STableDataBlocks;
 
-typedef struct SQueryInfo {
-  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
-  uint32_t         type;          // query/insert type
-  STimeWindow      window;        // the whole query time window
-
-  SInterval        interval;      // tumble time window
-  SSessionWindow   sessionWindow; // session time window
-
-  SGroupbyExpr     groupbyExpr;   // groupby tags info
-  SArray *         colList;       // SArray<SColumn*>
-  SFieldInfo       fieldsInfo;
-  SArray *         exprList;      // SArray<SExprInfo*>
-  SArray *         exprList1;     // final exprlist in case of arithmetic expression exists
-  SLimitVal        limit;
-  SLimitVal        slimit;
-  STagCond         tagCond;
-
-  SOrderVal        order;
-  int16_t          fillType;      // final result fill type
-  int16_t          numOfTables;
-  STableMetaInfo **pTableMetaInfo;
-  struct STSBuf   *tsBuf;
-  int64_t *        fillVal;       // default value for fill
-  char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
-  int64_t          clauseLimit;   // limit for current sub clause
-
-  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
-  int64_t          vgroupLimit;    // table limit in case of super table projection query + global order + limit
-
-  int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
-  int16_t          resColumnId;   // result column id
-  bool             distinctTag;   // distinct tag or not
-  int32_t          round;         // 0/1/....
-  int32_t          bufLen;
-  char*            buf;
-  SQInfo*          pQInfo;      // global merge operator
-  SQueryAttr*      pQueryAttr;     // query object
-
-  struct SQueryInfo *sibling;     // sibling
-  SArray            *pUpstream;   // SArray<struct SQueryInfo>
-  struct SQueryInfo *pDownstream;
-  int32_t            havingFieldNum;
-  bool               stableQuery;
-  bool               groupbyColumn;
-  bool               simpleAgg;
-  bool               arithmeticOnAgg;
-  bool               projectionQuery;
-  bool               hasFilter;
-  bool               onlyTagQuery;
-} SQueryInfo;
-
 typedef struct {
   STableMeta   *pTableMeta;
   SVgroupsInfo *pVgroupInfo;
@@ -255,9 +130,13 @@ typedef struct SInsertStatementParam {
   int8_t       schemaAttached;          // denote if submit block is built with table schema or not
   STagData     tagData;                 // NOTE: pTagData->data is used as a variant length array
 
+  int32_t      batchSize;               // for parameter ('?') binding and batch processing
+  int32_t      numOfParams;
+
   char         msg[512];                // error message
-  char        *sql;                     // current sql statement position
   uint32_t     insertType;              // insert data from [file|sql statement| bound statement]
+  uint64_t     objectId;                // sql object id
+  char        *sql;                     // current sql statement position
 } SInsertStatementParam;
 
 // TODO extract sql parser supporter
@@ -266,13 +145,9 @@ typedef struct {
   uint8_t msgType;
   SInsertStatementParam insertParam;
   char    reserve1[3];        // fix bus error on arm32
-  union {
-    int32_t count;
-  };
+  int32_t count;   // todo remove it
 
-  char *       curSql;       // current sql, resume position of sql after parsing paused
   char         reserve2[3];        // fix bus error on arm32
-
   int16_t      numOfCols;
   char         reserve3[2];        // fix bus error on arm32
   uint32_t     allocSize;
@@ -283,8 +158,6 @@ typedef struct {
   SQueryInfo  *pQueryInfo;
   SQueryInfo  *active;         // current active query info
   int32_t      batchSize;      // for parameter ('?') binding and batch processing
-  int32_t      numOfParams;
-  STagData     tagData;        // NOTE: pTagData->data is used as a variant length array
   int32_t      resColumnId;
 } SSqlCmd;
 
@@ -320,7 +193,7 @@ typedef struct {
 
   TAOS_FIELD*           final;
   SArithmeticSupport   *pArithSup;   // support the arithmetic expression calculation on agg functions
-  struct SLocalMerger  *pLocalMerger;
+  struct SGlobalMerger *pMerger;
 } SSqlRes;
 
 typedef struct {
@@ -447,7 +320,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo);
 void tscSetResRawPtrRv(SSqlRes* pRes, SQueryInfo* pQueryInfo, SSDataBlock* pBlock);
 
 void handleDownstreamOperator(SSqlObj** pSqlList, int32_t numOfUpstream, SQueryInfo* px, SSqlRes* pOutput);
-void destroyTableNameList(SSqlCmd* pCmd);
+void destroyTableNameList(SInsertStatementParam* pInsertParam);
 
 void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta);
 
@@ -479,7 +352,7 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code);
 void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen);
 
 void tscImportDataFromFile(SSqlObj *pSql);
-void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
+struct SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id);
 bool tscIsUpdateQuery(SSqlObj* pSql);
 char* tscGetSqlStr(SSqlObj* pSql);
 bool tscIsQueryWithLimit(SSqlObj* pSql);
@@ -489,7 +362,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32
 
 char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
 
-int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
+int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql);
 int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo,  const char* sql);
 
 int32_t tscValidateSqlInfo(SSqlObj *pSql, struct SSqlInfo *pInfo);

src/client/src/tscAsync.c

@@ -22,7 +22,7 @@
 #include "tscSubquery.h"
 #include "tscUtil.h"
 #include "tsched.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "tsclient.h"
 
 static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
@@ -58,7 +58,6 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para
   strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen);
 
   tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr);
-  pCmd->curSql = pSql->sqlstr;
   pCmd->resColumnId = TSDB_RES_COL_ID;
 
   int32_t code = tsParseSql(pSql, true);
@@ -221,7 +220,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) {
 
   tscResetForNextRetrieve(pRes);
   
-  // handle the sub queries of join query
+  // handle outer query based on the already retrieved nest query results.
   SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
   if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) {
     SSchedMsg schedMsg = {0};

src/client/src/tscLocalMerge.c → src/client/src/tscGlobalmerge.c

@@ -13,15 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tscLocalMerge.h"
-#include "tscSubquery.h"
 #include "os.h"
 #include "texpr.h"
 #include "tlosertree.h"
+
+#include "tscGlobalmerge.h"
+#include "tscSubquery.h"
 #include "tscLog.h"
-#include "tsclient.h"
 #include "qUtil.h"
 
+#define COLMODEL_GET_VAL(data, schema, rowId, colId) \
+  (data + (schema)->pFields[colId].offset * ((schema)->capacity) + (rowId) * (schema)->pFields[colId].field.bytes)
+
+
 typedef struct SCompareParam {
   SLocalDataSource **pLocalData;
   tOrderDescriptor * pDesc;
@@ -29,9 +33,18 @@ typedef struct SCompareParam {
   int32_t            groupOrderType;
 } SCompareParam;
 
-bool needToMergeRv(SSDataBlock* pBlock, SArray* columnIndex, int32_t index, char **buf);
+static bool needToMerge(SSDataBlock* pBlock, SArray* columnIndexList, int32_t index, char **buf) {
+  int32_t ret = 0;
+  size_t  size = taosArrayGetSize(columnIndexList);
+  if (size > 0) {
+    ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, index, buf, TSDB_ORDER_ASC);
+  }
+
+  // if ret == 0, means the result belongs to the same group
+  return (ret == 0);
+}
 
-int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
+static int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
   int32_t pLeftIdx = *(int32_t *)pLeft;
   int32_t pRightIdx = *(int32_t *)pRight;
 
@@ -57,16 +70,16 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) {
   }
 }
 
-int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
-                             SQueryInfo* pQueryInfo, SLocalMerger **pMerger, int64_t id) {
+int32_t tscCreateGlobalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc,
+                             SQueryInfo* pQueryInfo, SGlobalMerger **pMerger, int64_t id) {
   if (pMemBuffer == NULL) {
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     tscError("0x%"PRIx64" %p pMemBuffer is NULL", id, pMemBuffer);
     return TSDB_CODE_TSC_APP_ERROR;
   }
  
   if (pDesc->pColumnModel == NULL) {
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     tscError("0x%"PRIx64" no local buffer or intermediate result format model", id);
     return  TSDB_CODE_TSC_APP_ERROR;
   }
@@ -83,7 +96,7 @@ int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tO
   }
 
   if (numOfFlush == 0 || numOfBuffer == 0) {
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     tscDebug("0x%"PRIx64" no data to retrieve", id);
     return TSDB_CODE_SUCCESS;
   }
@@ -92,15 +105,15 @@ int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tO
     tscError("0x%"PRIx64" Invalid value of buffer capacity %d and page size %d ", id, pDesc->pColumnModel->capacity,
              pMemBuffer[0]->pageSize);
 
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     return TSDB_CODE_TSC_APP_ERROR;
   }
 
-  *pMerger = (SLocalMerger *) calloc(1, sizeof(SLocalMerger));
+  *pMerger = (SGlobalMerger *) calloc(1, sizeof(SGlobalMerger));
   if ((*pMerger) == NULL) {
     tscError("0x%"PRIx64" failed to create local merge structure, out of memory", id);
 
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -160,7 +173,7 @@ int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tO
   // no data actually, no need to merge result.
   if (idx == 0) {
     tscDebug("0x%"PRIx64" retrieved no data", id);
-    tscLocalReducerEnvDestroy(pMemBuffer, pDesc, numOfBuffer);
+    tscDestroyGlobalMergerEnv(pMemBuffer, pDesc, numOfBuffer);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -198,7 +211,7 @@ int32_t tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tO
   }
 
   // restore the limitation value at the last stage
-  if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+  if (pQueryInfo->orderProjectQuery) {
     pQueryInfo->limit.limit = pQueryInfo->clauseLimit;
     pQueryInfo->limit.offset = pQueryInfo->prjOffset;
   }
@@ -297,28 +310,28 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa
   return 0;
 }
 
-void tscDestroyLocalMerger(SLocalMerger* pLocalMerger) {
-  if (pLocalMerger == NULL) {
+void tscDestroyGlobalMerger(SGlobalMerger* pMerger) {
+  if (pMerger == NULL) {
     return;
   }
 
-  for (int32_t i = 0; i < pLocalMerger->numOfBuffer; ++i) {
-    tfree(pLocalMerger->pLocalDataSrc[i]);
+  for (int32_t i = 0; i < pMerger->numOfBuffer; ++i) {
+    tfree(pMerger->pLocalDataSrc[i]);
   }
 
-  pLocalMerger->numOfBuffer = 0;
-  tscLocalReducerEnvDestroy(pLocalMerger->pExtMemBuffer, pLocalMerger->pDesc, pLocalMerger->numOfVnode);
+  pMerger->numOfBuffer = 0;
+  tscDestroyGlobalMergerEnv(pMerger->pExtMemBuffer, pMerger->pDesc, pMerger->numOfVnode);
 
-  pLocalMerger->numOfCompleted = 0;
+  pMerger->numOfCompleted = 0;
 
-  if (pLocalMerger->pLoserTree) {
-    tfree(pLocalMerger->pLoserTree->param);
-    tfree(pLocalMerger->pLoserTree);
+  if (pMerger->pLoserTree) {
+    tfree(pMerger->pLoserTree->param);
+    tfree(pMerger->pLoserTree);
   }
 
-  tfree(pLocalMerger->buf);
-  tfree(pLocalMerger->pLocalDataSrc);
-  free(pLocalMerger);
+  tfree(pMerger->buf);
+  tfree(pMerger->pLocalDataSrc);
+  free(pMerger);
 }
 
 static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo* pQueryInfo, SColumnModel *pModel) {
@@ -329,7 +342,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo*
   }
 
   // primary timestamp column is involved in final result
-  if (pQueryInfo->interval.interval != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
+  if (pQueryInfo->interval.interval != 0 || pQueryInfo->orderProjectQuery) {
     numOfGroupByCols++;
   }
 
@@ -392,7 +405,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SQueryInfo*
   }
 }
 
-int32_t tscLocalReducerEnvCreate(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub,
+int32_t tscCreateGlobalMergerEnv(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBuffer, int32_t numOfSub,
                                  tOrderDescriptor **pOrderDesc, uint32_t nBufferSizes, int64_t id) {
   SSchema      *pSchema = NULL;
   SColumnModel *pModel = NULL;
@@ -456,7 +469,7 @@ int32_t tscLocalReducerEnvCreate(SQueryInfo *pQueryInfo, tExtMemBuffer ***pMemBu
  * @param pDesc
  * @param numOfVnodes
  */
-void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes) {
+void tscDestroyGlobalMergerEnv(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDesc, int32_t numOfVnodes) {
   tOrderDescDestroy(pDesc);
   for (int32_t i = 0; i < numOfVnodes; ++i) {
     pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]);
@@ -467,12 +480,12 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe
 
 /**
  *
- * @param pLocalMerge
+ * @param pMerger
  * @param pOneInterDataSrc
  * @param treeList
  * @return the number of remain input source. if ret == 0, all data has been handled
  */
-int32_t loadNewDataFromDiskFor(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc,
+int32_t loadNewDataFromDiskFor(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc,
                                bool *needAdjustLoserTree) {
   pOneInterDataSrc->rowIdx = 0;
   pOneInterDataSrc->pageId += 1;
@@ -489,17 +502,17 @@ int32_t loadNewDataFromDiskFor(SLocalMerger *pLocalMerge, SLocalDataSource *pOne
 #endif
     *needAdjustLoserTree = true;
   } else {
-    pLocalMerge->numOfCompleted += 1;
+    pMerger->numOfCompleted += 1;
 
     pOneInterDataSrc->rowIdx = -1;
     pOneInterDataSrc->pageId = -1;
     *needAdjustLoserTree = true;
   }
 
-  return pLocalMerge->numOfBuffer;
+  return pMerger->numOfBuffer;
 }
 
-void adjustLoserTreeFromNewData(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc,
+void adjustLoserTreeFromNewData(SGlobalMerger *pMerger, SLocalDataSource *pOneInterDataSrc,
                                 SLoserTreeInfo *pTree) {
   /*
    * load a new data page into memory for intermediate dataset source,
@@ -507,7 +520,7 @@ void adjustLoserTreeFromNewData(SLocalMerger *pLocalMerge, SLocalDataSource *pOn
    */
   bool needToAdjust = true;
   if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) {
-    loadNewDataFromDiskFor(pLocalMerge, pOneInterDataSrc, &needToAdjust);
+    loadNewDataFromDiskFor(pMerger, pOneInterDataSrc, &needToAdjust);
   }
 
   /*
@@ -515,7 +528,7 @@ void adjustLoserTreeFromNewData(SLocalMerger *pLocalMerge, SLocalDataSource *pOn
    * if the loser tree is rebuild completed, we do not need to adjust
    */
   if (needToAdjust) {
-    int32_t leafNodeIdx = pTree->pNode[0].index + pLocalMerge->numOfBuffer;
+    int32_t leafNodeIdx = pTree->pNode[0].index + pMerger->numOfBuffer;
 
 #ifdef _DEBUG_VIEW
     printf("before adjust:\t");
@@ -567,7 +580,7 @@ static void setTagValueForMultipleRows(SQLFunctionCtx* pCtx, int32_t numOfOutput
   }
 }
 
-static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) {
+static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) {
   SMultiwayMergeInfo* pInfo = pOperator->info;
   SQLFunctionCtx* pCtx = pInfo->binfo.pCtx;
 
@@ -579,7 +592,7 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S
 
   for(int32_t i = 0; i < pBlock->info.rows; ++i) {
     if (pInfo->hasPrev) {
-      if (needToMergeRv(pBlock, pInfo->orderColumnList, i, pInfo->prevRow)) {
+      if (needToMerge(pBlock, pInfo->orderColumnList, i, pInfo->prevRow)) {
         for (int32_t j = 0; j < numOfExpr; ++j) {
           pCtx[j].pInput = add[j] + pCtx[j].inputBytes * i;
         }
@@ -654,45 +667,27 @@ static void doExecuteFinalMergeRv(SOperatorInfo* pOperator, int32_t numOfExpr, S
   tfree(add);
 }
 
-bool needToMergeRv(SSDataBlock* pBlock, SArray* columnIndexList, int32_t index, char **buf) {
-  int32_t ret = 0;
-  size_t  size = taosArrayGetSize(columnIndexList);
-  if (size > 0) {
-    ret = compare_aRv(pBlock, columnIndexList, (int32_t) size, index, buf, TSDB_ORDER_ASC);
-  }
-
-  // if ret == 0, means the result belongs to the same group
-  return (ret == 0);
+static bool isAllSourcesCompleted(SGlobalMerger *pMerger) {
+  return (pMerger->numOfBuffer == pMerger->numOfCompleted);
 }
 
-static bool isAllSourcesCompleted(SLocalMerger *pLocalMerge) {
-  return (pLocalMerge->numOfBuffer == pLocalMerge->numOfCompleted);
-}
-
-void tscInitResObjForLocalQuery(SSqlObj *pSql, int32_t numOfRes, int32_t rowLen) {
-  SSqlRes *pRes = &pSql->res;
-  if (pRes->pLocalMerger != NULL) {
-    tscDestroyLocalMerger(pRes->pLocalMerger);
-    pRes->pLocalMerger = NULL;
-    tscDebug("0x%"PRIx64" free local reducer finished", pSql->self);
+SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint64_t id) {
+  SGlobalMerger *pMerger = calloc(1, sizeof(SGlobalMerger));
+  if (pMerger == NULL) {
+    tscDebug("0x%"PRIx64" free local reducer finished", id);
+    return NULL;
   }
 
-  pRes->qId = 1;  // hack to pass the safety check in fetch_row function
-  pRes->numOfRows = 0;
-  pRes->row = 0;
-
-  pRes->rspType = 0;  // used as a flag to denote if taos_retrieved() has been called yet
-  pRes->pLocalMerger = (SLocalMerger *)calloc(1, sizeof(SLocalMerger));
-
   /*
    * One more byte space is required, since the sprintf function needs one additional space to put '\0' at
    * the end of string
    */
   size_t size = numOfRes * rowLen + 1;
-  pRes->pLocalMerger->buf = calloc(1, size);
-  pRes->data = pRes->pLocalMerger->buf;
+  pMerger->buf = calloc(1, size);
+  return pMerger;
 }
 
+// todo remove it
 int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
   int32_t maxRowSize = MAX(rowSize, finalRowSize);
   char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize));
@@ -736,9 +731,6 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_
   return offset;
 }
 
-#define COLMODEL_GET_VAL(data, schema, rowId, colId) \
-  (data + (schema)->pFields[colId].offset * ((schema)->capacity) + (rowId) * (schema)->pFields[colId].field.bytes)
-
 static void appendOneRowToDataBlock(SSDataBlock *pBlock, char *buf, SColumnModel *pModel, int32_t rowIndex,
                                     int32_t maxRows) {
   for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
@@ -760,7 +752,7 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) {
 
   SMultiwayMergeInfo *pInfo = pOperator->info;
 
-  SLocalMerger   *pMerger = pInfo->pMerge;
+  SGlobalMerger   *pMerger = pInfo->pMerge;
   SLoserTreeInfo *pTree   = pMerger->pLoserTree;
 
   pInfo->binfo.pRes->info.rows = 0;
@@ -844,7 +836,7 @@ SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup) {
   return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL;
 }
 
-static bool isSameGroupRv(SArray* orderColumnList, SSDataBlock* pBlock, char** dataCols) {
+static bool isSameGroup(SArray* orderColumnList, SSDataBlock* pBlock, char** dataCols) {
   int32_t numOfCols = (int32_t) taosArrayGetSize(orderColumnList);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColIndex *pIndex = taosArrayGet(orderColumnList, i);
@@ -892,7 +884,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
         }
       }
 
-      doExecuteFinalMergeRv(pOperator, pOperator->numOfOutput, pAggInfo->pExistBlock);
+      doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pAggInfo->pExistBlock);
 
       savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pAggInfo->pExistBlock, 0,
                            &pAggInfo->hasGroupColData);
@@ -913,7 +905,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
     }
 
     if (pAggInfo->hasGroupColData) {
-      bool sameGroup = isSameGroupRv(pAggInfo->groupColumnList, pBlock, pAggInfo->currentGroupColData);
+      bool sameGroup = isSameGroup(pAggInfo->groupColumnList, pBlock, pAggInfo->currentGroupColData);
       if (!sameGroup) {
         *newgroup = true;
         pAggInfo->hasDataBlockForNewGroup = true;
@@ -927,7 +919,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
     setInputDataBlock(pOperator, pAggInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC);
     updateOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor);
 
-    doExecuteFinalMergeRv(pOperator, pOperator->numOfOutput, pBlock);
+    doExecuteFinalMerge(pOperator, pOperator->numOfOutput, pBlock);
     savePrevOrderColumns(pAggInfo->currentGroupColData, pAggInfo->groupColumnList, pBlock, 0, &pAggInfo->hasGroupColData);
     handleData = true;
   }

src/client/src/tscLocal.c

@@ -20,7 +20,7 @@
 #include "tname.h"
 #include "tscLog.h"
 #include "tscUtil.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "tsclient.h"
 #include "taos.h"
 #include "tscSubquery.h"
@@ -71,7 +71,9 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
     numOfRows = numOfRows + tscGetNumOfTags(pMeta);
   }
 
-  tscInitResObjForLocalQuery(pSql, totalNumOfRows, rowLen);
+  pSql->res.pMerger = tscInitResObjForLocalQuery(totalNumOfRows, rowLen, pSql->self);
+  tscInitResForMerge(&pSql->res);
+
   SSchema *pSchema = tscGetTableSchema(pMeta);
 
   for (int32_t i = 0; i < numOfRows; ++i) {
@@ -433,7 +435,8 @@ static int32_t tscSCreateSetValueToResObj(SSqlObj *pSql, int32_t rowLen, const c
   if (strlen(ddl) == 0) {
     
   }
-  tscInitResObjForLocalQuery(pSql, numOfRows, rowLen);
+  pSql->res.pMerger = tscInitResObjForLocalQuery(numOfRows, rowLen, pSql->self);
+  tscInitResForMerge(&pSql->res);
 
   TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0);
   char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * numOfRows;
@@ -882,7 +885,8 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa
   TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength);
   tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
 
-  tscInitResObjForLocalQuery(pSql, 1, (int32_t)valueLength);
+  pSql->res.pMerger = tscInitResObjForLocalQuery(1, (int32_t)valueLength, pSql->self);
+  tscInitResForMerge(&pSql->res);
 
   SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0);
   pInfo->pExpr = taosArrayGetP(pQueryInfo->exprList, 0);

src/client/src/tscPrepare.c

@@ -1085,7 +1085,7 @@ static int insertStmtExecute(STscStmt* stmt) {
 
   fillTablesColumnsNull(stmt->pSql);
 
-  int code = tscMergeTableDataBlocks(stmt->pSql, false);
+  int code = tscMergeTableDataBlocks(&stmt->pSql->cmd.insertParam, false);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -1185,7 +1185,7 @@ static int insertBatchStmtExecute(STscStmt* pStmt) {
 
   fillTablesColumnsNull(pStmt->pSql);
 
-  if ((code = tscMergeTableDataBlocks(pStmt->pSql, false)) != TSDB_CODE_SUCCESS) {
+  if ((code = tscMergeTableDataBlocks(&pStmt->pSql->cmd.insertParam, false)) != TSDB_CODE_SUCCESS) {
     return code;
   }
 
@@ -1213,7 +1213,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) {
   }
 
   int32_t index = 0;
-  SStrToken sToken = tStrGetToken(pCmd->curSql, &index, false);
+  SStrToken sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
   if (sToken.n == 0) {
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
@@ -1232,7 +1232,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) {
 
     pStmt->mtb.tagSet = true;
 
-    sToken = tStrGetToken(pCmd->curSql, &index, false);
+    sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
     if (sToken.n > 0 && sToken.type == TK_VALUES) {
       return TSDB_CODE_SUCCESS;
     }
@@ -1241,18 +1241,18 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
-    sToken = tStrGetToken(pCmd->curSql, &index, false);
+    sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
     if (sToken.n <= 0 || ((sToken.type != TK_ID) && (sToken.type != TK_STRING))) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
     pStmt->mtb.stbname = sToken;
 
-    sToken = tStrGetToken(pCmd->curSql, &index, false);
+    sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
     if (sToken.n <= 0 || sToken.type != TK_TAGS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
-    sToken = tStrGetToken(pCmd->curSql, &index, false);
+    sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
     if (sToken.n <= 0 || sToken.type != TK_LP) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
@@ -1262,7 +1262,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) {
     int32_t loopCont = 1;
 
     while (loopCont) {
-      sToken = tStrGetToken(pCmd->curSql, &index, false);
+      sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
       if (sToken.n <= 0) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
       }
@@ -1285,7 +1285,7 @@ int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
-    sToken = tStrGetToken(pCmd->curSql, &index, false);
+    sToken = tStrGetToken(pCmd->insertParam.sql, &index, false);
     if (sToken.n <= 0 || sToken.type != TK_VALUES) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
@@ -1468,7 +1468,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   if (tscIsInsertData(pSql->sqlstr)) {
     pStmt->isInsert = true;
 
-    pSql->cmd.numOfParams = 0;
+    pSql->cmd.insertParam.numOfParams = 0;
     pSql->cmd.batchSize   = 0;
 
     registerSqlObj(pSql);
@@ -1565,7 +1565,7 @@ int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags
 
   tscDebug("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr);
 
-  pSql->cmd.numOfParams = 0;
+  pSql->cmd.insertParam.numOfParams = 0;
   pSql->cmd.batchSize   = 0;
 
   if (taosHashGetSize(pCmd->insertParam.pTableBlockHashList) > 0) {
@@ -1851,8 +1851,8 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
 
   if (pStmt->isInsert) {
     SSqlObj* pSql = pStmt->pSql;
-    SSqlCmd *pCmd    = &pSql->cmd;
-    *nums = pCmd->numOfParams;
+    SSqlCmd *pCmd = &pSql->cmd;
+    *nums = pCmd->insertParam.numOfParams;
     return TSDB_CODE_SUCCESS;
   } else {
     SNormalStmt* normal = &pStmt->normal;

src/client/src/tscSQLParser.c

@@ -28,7 +28,7 @@
 #include "tname.h"
 #include "tscLog.h"
 #include "tscUtil.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "tsclient.h"
 #include "tstrbuild.h"
 #include "ttoken.h"
@@ -41,11 +41,11 @@
 
 #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey))
 
-// -1 is tbname column index, so here use the -3 as the initial value
-#define COLUMN_INDEX_INITIAL_VAL (-3)
+// -1 is tbname column index, so here use the -2 as the initial value
+#define COLUMN_INDEX_INITIAL_VAL (-2)
 #define COLUMN_INDEX_INITIALIZER \
   { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL }
-#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_BLOCK_DIST_COLUMN_INDEX))
+#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX))
 #define TBNAME_LIST_SEP ","
 
 typedef struct SColumnList {  // todo refactor
@@ -195,7 +195,7 @@ static bool validateDebugFlag(int32_t v) {
  * is not needed in the final error message.
  */
 static int32_t invalidOperationMsg(char* dstBuffer, const char* errMsg) {
-  return tscInvalidSQLErrMsg(dstBuffer, errMsg, NULL);
+  return tscInvalidOperationMsg(dstBuffer, errMsg, NULL);
 }
 
 static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tVariant* pVar) {
@@ -1873,9 +1873,6 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
       SSchema* colSchema = tGetTbnameColumnSchema();
       tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG, getNewResColId(pCmd));
-    } else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) {
-      SSchema colSchema = tGetBlockDistColumnSchema();
-      tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG, getNewResColId(pCmd));
     } else {
       STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
       STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
@@ -2487,7 +2484,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
-      SColumnIndex index = {.tableIndex = 0, .columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX,};
+      SColumnIndex index = {.tableIndex = 0, .columnIndex = 0,};
       pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
 
       SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY};
@@ -2495,10 +2492,16 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       int16_t resType = 0;
       int16_t bytes   = 0;
       getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0);
-
       s.bytes = bytes;
       s.type = (uint8_t)resType;
-      SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, &s, TSDB_COL_TAG, getNewResColId(pCmd));
+
+      SExprInfo* pExpr = tscExprInsert(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, resType,
+                                         bytes, getNewResColId(pCmd), bytes, 0);
+      tstrncpy(pExpr->base.aliasName, s.name, sizeof(pExpr->base.aliasName));
+
+      SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex);
+      insertResultField(pQueryInfo, 0, &ids, bytes, s.type, s.name, pExpr);
+
       pExpr->base.numOfParams = 1;
       pExpr->base.param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize;
       pExpr->base.param[0].nType = TSDB_DATA_TYPE_BIGINT;
@@ -2545,14 +2548,6 @@ static bool isTablenameToken(SStrToken* token) {
 
   return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L));
 }
-static bool isTableBlockDistToken(SStrToken* token) {
-  SStrToken tmpToken = *token;
-  SStrToken tableToken = {0};
-
-  extractTableNameFromToken(&tmpToken, &tableToken);
-
-  return (strncasecmp(TSQL_BLOCK_DIST, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_BLOCK_DIST_L));
-}
 
 static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) {
   STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta;
@@ -2582,8 +2577,6 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu
 
   if (isTablenameToken(pToken)) {
     pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX;
-  } else if (isTableBlockDistToken(pToken))  {
-    pIndex->columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX; 
   } else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) {
     pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX;
   } else {
@@ -6693,7 +6686,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
 
         if (!findColumnIndex) {
           tdDestroyKVRowBuilder(&kvRowBuilder);
-          return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken->z);
+          return tscInvalidOperationMsg(pCmd->payload, "invalid tag name", sToken->z);
         }
       }
     } else {
@@ -7546,6 +7539,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
+    // validate the query filter condition info
     if (pSqlNode->pWhere != NULL) {
       if (validateWhereNode(pQueryInfo, &pSqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_OPERATION;
@@ -7557,6 +7551,16 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
         pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
       }
     }
+
+    // validate the interval info
+    if (validateIntervalNode(pSql, pQueryInfo, pSqlNode) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    } else {
+      if (isTimeWindowQuery(pQueryInfo) &&
+          (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    }
   } else {
     pQueryInfo->command = TSDB_SQL_SELECT;
 
@@ -7702,6 +7706,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
     pQueryInfo->groupbyColumn = tscGroupbyColumn(pQueryInfo);
 
     pQueryInfo->arithmeticOnAgg = tsIsArithmeticQueryOnAggResult(pQueryInfo);
+    pQueryInfo->orderProjectQuery = tscOrderedProjectionQueryOnSTable(pQueryInfo, 0);
 
     SExprInfo** p = NULL;
     int32_t numOfExpr = 0;

src/client/src/tscServer.c

@@ -15,15 +15,15 @@
 
 #include "os.h"
 #include "tcmdtype.h"
+#include "tlockfree.h"
 #include "trpc.h"
-#include "tscLocalMerge.h"
+#include "tscGlobalmerge.h"
 #include "tscLog.h"
 #include "tscProfile.h"
 #include "tscUtil.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "tsclient.h"
 #include "ttimer.h"
-#include "tlockfree.h"
 #include "qPlan.h"
 
 int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
@@ -1598,7 +1598,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
     return code;
   }
 
-  if (pRes->pLocalMerger == NULL) { // no result from subquery, so abort here directly.
+  if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly.
     (*pSql->fp)(pSql->param, pSql, pRes->numOfRows);
     return code;
   }
@@ -1615,15 +1615,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) {
     taosArrayPush(group, &tableKeyInfo);
     taosArrayPush(tableGroupInfo.pGroupList, &group);
 
-    // todo remove it
-    SExprInfo* list = calloc(tscNumOfExprs(pQueryInfo), sizeof(SExprInfo));
-    for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) {
-      SExprInfo* pExprInfo = tscExprGet(pQueryInfo, i);
-      list[i] = *pExprInfo;
-    }
-
-    pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, list, &tableGroupInfo, NULL, NULL, pRes->pLocalMerger, MERGE_STAGE);
-    tfree(list);
+    pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE);
   }
 
   uint64_t localQueryId = 0;
@@ -2402,8 +2394,8 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn
     char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
 
     // tag data exists
-    if (autocreate && pSql->cmd.tagData.dataLen != 0) {
-      pMsg = serializeTagData(&pSql->cmd.tagData, pMsg);
+    if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) {
+      pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg);
     }
 
     pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg);

src/client/src/tscStream.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <tschemautil.h>
 #include "os.h"
 #include "taosmsg.h"
 #include "tscLog.h"

src/client/src/tscSubquery.c

@@ -21,7 +21,7 @@
 #include "tcompare.h"
 #include "tscLog.h"
 #include "tscSubquery.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "tsclient.h"
 #include "qUtil.h"
 #include "qPlan.h"
@@ -2446,7 +2446,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
 
   assert(pState->numOfSub > 0);
   
-  int32_t ret = tscLocalReducerEnvCreate(pQueryInfo, &pMemoryBuf, pSql->subState.numOfSub, &pDesc, nBufferSize, pSql->self);
+  int32_t ret = tscCreateGlobalMergerEnv(pQueryInfo, &pMemoryBuf, pSql->subState.numOfSub, &pDesc, nBufferSize, pSql->self);
   if (ret != 0) {
     pRes->code = ret;
     tscAsyncResultOnError(pSql);
@@ -2459,7 +2459,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   if (pSql->pSubs == NULL) {
     tfree(pSql->pSubs);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc,pState->numOfSub);
+    tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc,pState->numOfSub);
 
     tscAsyncResultOnError(pSql);
     return ret;
@@ -2526,13 +2526,13 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
     tscError("0x%"PRIx64" failed to prepare subquery structure and launch subqueries", pSql->self);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pState->numOfSub);
+    tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub);
     doCleanupSubqueries(pSql, i);
     return pRes->code;   // free all allocated resource
   }
   
   if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) {
-    tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pState->numOfSub);
+    tscDestroyGlobalMergerEnv(pMemoryBuf, pDesc, pState->numOfSub);
     doCleanupSubqueries(pSql, i);
     return pRes->code;
   }
@@ -2697,7 +2697,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
       tstrerror(pParentSql->res.code));
 
   // release allocated resource
-  tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor,
+  tscDestroyGlobalMergerEnv(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor,
                             pState->numOfSub);
   
   tscFreeRetrieveSup(pSql);
@@ -2772,7 +2772,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
   SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd);
   tscClearInterpInfo(pPQueryInfo);
   
-  code = tscCreateLocalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pLocalMerger, pParentSql->self);
+  code = tscCreateGlobalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pMerger, pParentSql->self);
   pParentSql->res.code = code;
 
   if (code == TSDB_CODE_SUCCESS && trsupport->pExtMemBuffer == NULL) {
@@ -3499,9 +3499,8 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
   return hasData;
 }
 
-// todo remove pExprs
-void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo,
-                               SOperatorInfo* pSourceOperator, char* sql, void* merger, int32_t stage) {
+void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pSourceOperator,
+    char* sql, void* merger, int32_t stage) {
   assert(pQueryInfo != NULL);
   SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo));
   if (pQInfo == NULL) {

src/common/inc/tname.h

@@ -92,8 +92,6 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len);
 
 void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
 
-SSchema tGetBlockDistColumnSchema();
-
 SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name);
 
 bool tscValidateTableNameLength(size_t len);

src/common/src/tname.c

@@ -33,15 +33,6 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len) {
   return strlen(prefix);
 }
 
-SSchema tGetBlockDistColumnSchema() {
-  SSchema s = {0};
-  s.bytes = TSDB_MAX_BINARY_LEN;;
-  s.type  = TSDB_DATA_TYPE_BINARY;
-  s.colId = TSDB_BLOCK_DIST_COLUMN_INDEX;
-  tstrncpy(s.name, TSQL_BLOCK_DIST_L, TSDB_COL_NAME_LEN);
-  return s;
-}
-
 SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) {
   SSchema s = {0};
 

go @ 050667e5

-Subproject commit 8ce6d86558afc8c0b50c10f990fd2b4270cf06fc
+Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f

grafanaplugin @ 32e2c97a

-Subproject commit 3530c6df097134a410bacec6b3cd013ef38a61aa
+Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df

src/inc/taosdef.h

@@ -244,7 +244,6 @@ do { \
 #define TSDB_MAX_REPLICA          5
 
 #define TSDB_TBNAME_COLUMN_INDEX        (-1)
-#define TSDB_BLOCK_DIST_COLUMN_INDEX    (-2)
 #define TSDB_UD_COLUMN_INDEX            (-1000)
 #define TSDB_RES_COL_ID                 (-5000)
 

src/inc/tsdb.h

@@ -215,7 +215,7 @@ typedef struct SDataBlockInfo {
 } SDataBlockInfo;
 
 typedef struct SFileBlockInfo {
-  int32_t numOfRows;
+  int32_t numBlocksOfStep;
 } SFileBlockInfo;
 
 typedef struct {
@@ -229,11 +229,15 @@ typedef struct {
   SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
+#define TSDB_BLOCK_DIST_STEP_ROWS 16
 typedef struct {
   uint16_t  rowSize;
   uint16_t  numOfFiles;
   uint32_t  numOfTables;
   uint64_t  totalSize;
+  uint64_t  totalRows;
+  int32_t   maxRows;
+  int32_t   minRows;
   int32_t   firstSeekTimeUs;
   uint32_t  numOfRowsInMemTable;
   SArray   *dataBlockInfos;

src/kit/taosdemo/taosdemo.c

@@ -2910,8 +2910,8 @@ static void* createTable(void *sarg)
   int buff_len;
   buff_len = BUFFER_SIZE / 8;
 
-  char *buffer = calloc(buff_len, 1);
-  if (buffer == NULL) {
+  pThreadInfo->buffer = calloc(buff_len, 1);
+  if (pThreadInfo->buffer == NULL) {
     errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
     exit(-1);
   }
@@ -2926,7 +2926,7 @@ static void* createTable(void *sarg)
   for (uint64_t i = pThreadInfo->start_table_from;
           i <= pThreadInfo->end_table_to; i++) {
     if (0 == g_Dbs.use_metric) {
-      snprintf(buffer, buff_len,
+      snprintf(pThreadInfo->buffer, buff_len,
               "create table if not exists %s.%s%"PRIu64" %s;",
               pThreadInfo->db_name,
               g_args.tb_prefix, i,
@@ -2935,13 +2935,13 @@ static void* createTable(void *sarg)
       if (superTblInfo == NULL) {
         errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                   __func__, __LINE__);
-        free(buffer);
+        free(pThreadInfo->buffer);
         exit(-1);
       } else {
         if (0 == len) {
           batchNum = 0;
-          memset(buffer, 0, buff_len);
-          len += snprintf(buffer + len,
+          memset(pThreadInfo->buffer, 0, buff_len);
+          len += snprintf(pThreadInfo->buffer + len,
                   buff_len - len, "create table ");
         }
         char* tagsValBuf = NULL;
@@ -2953,10 +2953,10 @@ static void* createTable(void *sarg)
                   i % superTblInfo->tagSampleCount);
         }
         if (NULL == tagsValBuf) {
-          free(buffer);
+          free(pThreadInfo->buffer);
           return NULL;
         }
-        len += snprintf(buffer + len,
+        len += snprintf(pThreadInfo->buffer + len,
                 buff_len - len,
                 "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
                 pThreadInfo->db_name, superTblInfo->childTblPrefix,
@@ -2973,9 +2973,10 @@ static void* createTable(void *sarg)
     }
 
     len = 0;
-    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
-      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
-      free(buffer);
+    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
+                NO_INSERT_TYPE, false)){
+      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
+      free(pThreadInfo->buffer);
       return NULL;
     }
 
@@ -2988,12 +2989,13 @@ static void* createTable(void *sarg)
   }
 
   if (0 != len) {
-    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
-      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
+    if (0 != queryDbExec(pThreadInfo->taos, pThreadInfo->buffer,
+                NO_INSERT_TYPE, false)) {
+      errorPrint( "queryDbExec() failed. buffer:\n%s\n", pThreadInfo->buffer);
     }
   }
 
-  free(buffer);
+  free(pThreadInfo->buffer);
   return NULL;
 }
 
@@ -4399,8 +4401,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
 
         cJSON* resubAfterConsume =
             cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
-        if (resubAfterConsume
-                && resubAfterConsume->type == cJSON_Number) {
+        if ((resubAfterConsume)
+                && (resubAfterConsume->type == cJSON_Number)
+                && (resubAfterConsume->valueint >= 0)) {
             g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                 = resubAfterConsume->valueint;
         } else if (!resubAfterConsume) {
@@ -4561,14 +4564,15 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
     }
 
     cJSON* superResubAfterConsume =
-            cJSON_GetObjectItem(superQuery, "endAfterConsume");
-    if (superResubAfterConsume
-            && superResubAfterConsume->type == cJSON_Number) {
-        g_queryInfo.superQueryInfo.endAfterConsume =
+            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
+    if ((superResubAfterConsume)
+            && (superResubAfterConsume->type == cJSON_Number)
+            && (superResubAfterConsume->valueint >= 0)) {
+        g_queryInfo.superQueryInfo.resubAfterConsume =
             superResubAfterConsume->valueint;
     } else if (!superResubAfterConsume) {
         // default value is -1, which mean do not resub
-        g_queryInfo.superQueryInfo.endAfterConsume = -1;
+        g_queryInfo.superQueryInfo.resubAfterConsume = -1;
     }
 
     // supert table sqls
@@ -4932,7 +4936,7 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k)
 
         case REST_IFACE:
             if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
-                        pThreadInfo->buffer, NULL /* not set result file */)) {
+                        pThreadInfo->buffer, pThreadInfo)) {
                 affectedRows = -1;
                 printf("========restful return fail, threadID[%d]\n",
                         pThreadInfo->threadID);

src/query/inc/qExecutor.h

@@ -22,6 +22,7 @@
 #include "qFill.h"
 #include "qResultbuf.h"
 #include "qSqlparser.h"
+#include "qTableMeta.h"
 #include "qTsbuf.h"
 #include "query.h"
 #include "taosdef.h"
@@ -70,14 +71,6 @@ typedef struct SResultRowPool {
   SArray* pData;    // SArray<void*>
 } SResultRowPool;
 
-typedef struct SGroupbyExpr {
-  int16_t tableIndex;
-  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
-  int16_t numOfGroupCols;  // todo remove it
-  int16_t orderIndex;  // order by column index
-  int16_t orderType;   // order by type: asc/desc
-} SGroupbyExpr;
-
 typedef struct SResultRow {
   int32_t       pageId;      // pageId & rowId is the position of current result in disk-based output buffer
   int32_t       offset:29;    // row index in buffer page
@@ -216,7 +209,7 @@ typedef struct SQueryAttr {
   int32_t          intermediateResultRowSize; // intermediate result row size, in case of top-k query.
   int32_t          maxTableColumnWidth;
   int32_t          tagLen;           // tag value length of current query
-  SGroupbyExpr* pGroupbyExpr;
+  SGroupbyExpr    *pGroupbyExpr;
 
   SExprInfo*       pExpr1;
   SExprInfo*       pExpr2;
@@ -475,10 +468,10 @@ typedef struct SDistinctOperatorInfo {
   int64_t           outputCapacity;
 } SDistinctOperatorInfo;
 
-struct SLocalMerger;
+struct SGlobalMerger;
 
 typedef struct SMultiwayMergeInfo {
-  struct SLocalMerger *pMerge;
+  struct SGlobalMerger *pMerge;
   SOptrBasicInfo       binfo;
   int32_t              bufCapacity;
   int64_t              seed;
@@ -559,6 +552,8 @@ int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId);
 void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters);
 
 STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf);
+STableQueryInfo* createTmpTableQueryInfo(STimeWindow win);
+
 int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg);
 
 bool isQueryKilled(SQInfo *pQInfo);

src/query/inc/qPlan.h

@@ -16,6 +16,8 @@
 #ifndef TDENGINE_QPLAN_H
 #define TDENGINE_QPLAN_H
 
+#include "qExecutor.h"
+
 struct SQueryInfo;
 
 typedef struct SQueryNodeBasicInfo {

src/query/inc/qSqlparser.h

@@ -142,7 +142,7 @@ typedef struct SCreateTableSql {
   } colInfo;
 
   SArray            *childTableInfo;        // SArray<SCreatedTableInfo>
-  SSqlNode     *pSelect;
+  SSqlNode          *pSelect;
 } SCreateTableSql;
 
 typedef struct SAlterTableInfo {
@@ -258,7 +258,6 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int
 SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
 
 SRelationInfo *setTableNameList(SRelationInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias);
-//SRelationInfo *setSubquery(SRelationInfo* pFromInfo, SRelElementPair* p);
 void          *destroyRelationInfo(SRelationInfo* pFromInfo);
 SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias);
 

src/query/inc/qTableMeta.h

+#ifndef TDENGINE_QTABLEUTIL_H
+#define TDENGINE_QTABLEUTIL_H
+
+#include "tsdb.h"  //todo tsdb should not be here
+#include "qSqlparser.h"
+
+typedef struct SFieldInfo {
+  int16_t      numOfOutput;   // number of column in result
+  TAOS_FIELD*  final;
+  SArray      *internalField; // SArray<SInternalField>
+} SFieldInfo;
+
+typedef struct SCond {
+  uint64_t uid;
+  int32_t  len;  // length of tag query condition data
+  char *   cond;
+} SCond;
+
+typedef struct SJoinNode {
+  uint64_t uid;
+  int16_t  tagColId;
+  SArray*  tsJoin;
+  SArray*  tagJoin;
+} SJoinNode;
+
+typedef struct SJoinInfo {
+  bool       hasJoin;
+  SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM];
+} SJoinInfo;
+
+typedef struct STagCond {
+  // relation between tbname list and query condition, including : TK_AND or TK_OR
+  int16_t relType;
+
+  // tbname query condition, only support tbname query condition on one table
+  SCond tbnameCond;
+
+  // join condition, only support two tables join currently
+  SJoinInfo joinInfo;
+
+  // for different table, the query condition must be seperated
+  SArray *pCond;
+} STagCond;
+
+typedef struct SGroupbyExpr {
+  int16_t tableIndex;
+  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
+  int16_t numOfGroupCols;  // todo remove it
+  int16_t orderIndex;  // order by column index
+  int16_t orderType;   // order by type: asc/desc
+} SGroupbyExpr;
+
+typedef struct STableComInfo {
+  uint8_t numOfTags;
+  uint8_t precision;
+  int16_t numOfColumns;
+  int32_t rowSize;
+} STableComInfo;
+
+typedef struct STableMeta {
+  int32_t        vgId;
+  STableId       id;
+  uint8_t        tableType;
+  char           sTableName[TSDB_TABLE_FNAME_LEN];  // super table name
+  uint64_t       suid;       // super table id
+  int16_t        sversion;
+  int16_t        tversion;
+  STableComInfo  tableInfo;
+  SSchema        schema[];  // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
+} STableMeta;
+
+typedef struct STableMetaInfo {
+  STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
+  uint32_t       tableMetaSize;
+  SVgroupsInfo  *vgroupList;
+  SArray        *pVgroupTables;   // SArray<SVgroupTableInfo>
+
+  /*
+   * 1. keep the vgroup index during the multi-vnode super table projection query
+   * 2. keep the vgroup index for multi-vnode insertion
+   */
+  int32_t       vgroupIndex;
+  SName         name;
+  char          aliasName[TSDB_TABLE_NAME_LEN];    // alias name of table specified in query sql
+  SArray       *tagColList;                        // SArray<SColumn*>, involved tag columns
+} STableMetaInfo;
+
+struct   SQInfo;      // global merge operator
+struct   SQueryAttr;     // query object
+
+typedef struct SQueryInfo {
+  int16_t          command;       // the command may be different for each subclause, so keep it seperately.
+  uint32_t         type;          // query/insert type
+  STimeWindow      window;        // the whole query time window
+
+  SInterval        interval;      // tumble time window
+  SSessionWindow   sessionWindow; // session time window
+
+  SGroupbyExpr     groupbyExpr;   // groupby tags info
+  SArray *         colList;       // SArray<SColumn*>
+  SFieldInfo       fieldsInfo;
+  SArray *         exprList;      // SArray<SExprInfo*>
+  SArray *         exprList1;     // final exprlist in case of arithmetic expression exists
+  SLimitVal        limit;
+  SLimitVal        slimit;
+  STagCond         tagCond;
+
+  SOrderVal        order;
+  int16_t          fillType;      // final result fill type
+  int16_t          numOfTables;
+  STableMetaInfo **pTableMetaInfo;
+  struct STSBuf   *tsBuf;
+  int64_t *        fillVal;       // default value for fill
+  char *           msg;           // pointer to the pCmd->payload to keep error message temporarily
+  int64_t          clauseLimit;   // limit for current sub clause
+
+  int64_t          prjOffset;     // offset value in the original sql expression, only applied at client side
+  int64_t          vgroupLimit;    // table limit in case of super table projection query + global order + limit
+
+  int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
+  int16_t          resColumnId;   // result column id
+  bool             distinctTag;   // distinct tag or not
+  int32_t          round;         // 0/1/....
+  int32_t          bufLen;
+  char*            buf;
+  struct SQInfo*          pQInfo;      // global merge operator
+  struct SQueryAttr*      pQueryAttr;     // query object
+
+  struct SQueryInfo *sibling;     // sibling
+  SArray            *pUpstream;   // SArray<struct SQueryInfo>
+  struct SQueryInfo *pDownstream;
+  int32_t            havingFieldNum;
+  bool               stableQuery;
+  bool               groupbyColumn;
+  bool               simpleAgg;
+  bool               arithmeticOnAgg;
+  bool               projectionQuery;
+  bool               hasFilter;
+  bool               onlyTagQuery;
+  bool               orderProjectQuery;
+} SQueryInfo;
+
+/**
+ * get the number of tags of this table
+ * @param pTableMeta
+ * @return
+ */
+int32_t tscGetNumOfTags(const STableMeta* pTableMeta);
+
+/**
+ * get the number of columns of this table
+ * @param pTableMeta
+ * @return
+ */
+int32_t tscGetNumOfColumns(const STableMeta* pTableMeta);
+
+/**
+ * get the basic info of this table
+ * @param pTableMeta
+ * @return
+ */
+STableComInfo tscGetTableInfo(const STableMeta* pTableMeta);
+
+/**
+ * get the schema
+ * @param pTableMeta
+ * @return
+ */
+SSchema* tscGetTableSchema(const STableMeta* pTableMeta);
+
+/**
+ * get the tag schema
+ * @param pMeta
+ * @return
+ */
+SSchema *tscGetTableTagSchema(const STableMeta *pMeta);
+
+/**
+ * get the column schema according to the column index
+ * @param pMeta
+ * @param colIndex
+ * @return
+ */
+SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex);
+
+/**
+ * get the column schema according to the column id
+ * @param pTableMeta
+ * @param colId
+ * @return
+ */
+SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId);
+
+/**
+ * create the table meta from the msg
+ * @param pTableMetaMsg
+ * @param size size of the table meta
+ * @return
+ */
+STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg);
+
+#endif  // TDENGINE_QTABLEUTIL_H

src/query/src/qAggMain.c

@@ -19,6 +19,7 @@
 #include "texpr.h"
 #include "ttype.h"
 #include "tsdb.h"
+#include "tglobal.h"
 
 #include "qAggMain.h"
 #include "qFill.h"
@@ -4828,51 +4829,81 @@ void blockInfo_func(SQLFunctionCtx* pCtx) {
   pResInfo->hasResult = DATA_SET_FLAG;
 }
 
-static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) {
+static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) {
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
   assert(pDist != NULL && pSrc != NULL);
+
   pDist->numOfTables += pSrc->numOfTables;
   pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable;
   pDist->numOfFiles += pSrc->numOfFiles;
   pDist->totalSize += pSrc->totalSize;
+  pDist->totalRows += pSrc->totalRows;
 
-  if (pDist->dataBlockInfos == NULL) {
-    pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo));
+  if (pResInfo->hasResult == DATA_SET_FLAG) {
+    pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows);
+    pDist->minRows = MIN(pDist->minRows, pSrc->minRows);
+  } else {
+    pDist->maxRows = pSrc->maxRows;
+    pDist->minRows = pSrc->minRows;
+    
+    int32_t numSteps = tsMaxRowsInFileBlock/TSDB_BLOCK_DIST_STEP_ROWS;
+    pDist->dataBlockInfos = taosArrayInit(numSteps, sizeof(SFileBlockInfo));
+    taosArraySetSize(pDist->dataBlockInfos, numSteps);
   }
 
-  taosArrayAddBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos));
+  size_t steps = taosArrayGetSize(pDist->dataBlockInfos);
+  for (int32_t i = 0; i < steps; ++i) {
+    int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep;
+    SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i);
+    blockInfo->numBlocksOfStep += srcNumBlocks;
+  }
 }
 
 void block_func_merge(SQLFunctionCtx* pCtx) {
-  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-
-  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
   STableBlockDist info = {0};
-
   int32_t len = *(int32_t*) pCtx->pInput;
   blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info);
 
-  mergeTableBlockDist(pDist, &info);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  mergeTableBlockDist(pResInfo, &info);
+
+  pResInfo->numOfRes = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
 }
 
-static int32_t doGetPercentile(const SArray* pArray, double rate) {
-  int32_t len = (int32_t)taosArrayGetSize(pArray);
-  if (len <= 0) {
-    return 0;
+void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
+                    double* percents, int32_t* percentiles) {
+  if (totalBlocks == 0) {
+    for (int32_t i = 0; i < numOfPercents; ++i) {
+      percentiles[i] = 0;
+    }
+    return;
   }
 
-  assert(rate >= 0 && rate <= 1.0);
-  int idx = (int32_t)((len - 1) * rate);
+  SArray *blocksInfos = pTableBlockDist->dataBlockInfos;
+  size_t  numSteps = taosArrayGetSize(blocksInfos);
+  size_t  cumulativeBlocks = 0;
 
-  return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows;
-}
+  int percentIndex = 0;
+  for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) {
+    int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep;
+    if (numStepBlocks == 0) continue;
+    cumulativeBlocks += numStepBlocks;
 
-static int compareBlockInfo(const void *pLeft, const void *pRight) {
-  int32_t left  = ((SFileBlockInfo *)pLeft)->numOfRows;
-  int32_t right = ((SFileBlockInfo *)pRight)->numOfRows;
+    while (percentIndex < numOfPercents) {
+      double blockRank = totalBlocks * percents[percentIndex];
+      if (blockRank <= cumulativeBlocks) {
+        percentiles[percentIndex] = indexStep;
+        ++percentIndex;
+      } else {
+        break;
+      }
+    }
+  }
 
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
+  for (int32_t i = 0; i < numOfPercents; ++i) {
+    percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2;
+  }
 }
 
 void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
@@ -4880,40 +4911,41 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
     return;
   }
 
-  int64_t min = INT64_MAX, max = INT64_MIN, avg = 0;
-  SArray* blockInfos= pTableBlockDist->dataBlockInfos;
-  int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos);
+  SArray* blockInfos = pTableBlockDist->dataBlockInfos;
+  uint64_t totalRows = pTableBlockDist->totalRows;
+  size_t   numSteps = taosArrayGetSize(blockInfos);
+  int64_t totalBlocks = 0;
+  int64_t min = -1, max = -1, avg = 0;
 
-  for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) {
+  for (int32_t i = 0; i < numSteps; i++) {
     SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i);
-    int64_t rows = blockInfo->numOfRows;
-
-    min = MIN(min, rows);
-    max = MAX(max, rows);
-    totalRows += rows;
+    int64_t blocks = blockInfo->numBlocksOfStep;
+    totalBlocks += blocks;
   }
 
   avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0;
-  taosArraySort(blockInfos, compareBlockInfo);
+  min = totalBlocks > 0 ? pTableBlockDist->minRows : 0;
+  max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0;
+
+  double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99};
+  int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
+  assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t));
+  getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles);
 
   uint64_t totalLen = pTableBlockDist->totalSize;
   int32_t rowSize = pTableBlockDist->rowSize;
-
+  double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1;
   int sz = sprintf(result + VARSTR_HEADER_SIZE,
                    "summary: \n\t "
                    "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t "
                    "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t "
                    "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t "
-                   "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t "
+                   "Rows=[%"PRIu64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f]\n\t "
                    "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]",
-                   doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10),
-                   doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30),
-                   doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50),
-                   doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70),
-                   doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90),
-                   doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99),
+                   percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5],
+                   percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11],
                    min, max, avg, 0.0,
-                   totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows),
+                   totalRows, totalBlocks, totalLen/1024.0, compRatio,
                    pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs);
   varDataSetLen(result, sz);
   UNUSED(sz);

src/query/src/qExecutor.c

@@ -935,7 +935,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx,
       setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock);
     } else {
       SColIndex* pCol = &pOperator->pExpr[i].base.colInfo;
-      if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCol->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) ||
+      if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == TSDB_FUNC_BLKINFO) ||
           (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) {
         SColIndex*       pColIndex = &pOperator->pExpr[i].base.colInfo;
         SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex);
@@ -1718,7 +1718,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
       case OP_TimeWindow: {
         pRuntimeEnv->proot =
             createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput);
-        setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot);
+        int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType;
+        if (opType != OP_DummyInput && opType != OP_Join) {
+          setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot);
+        }
         break;
       }
       case OP_Groupby: {
@@ -3267,6 +3270,25 @@ STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool
   return pTableQueryInfo;
 }
 
+STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) {
+  STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo));
+
+  pTableQueryInfo->win = win;
+  pTableQueryInfo->lastKey = win.skey;
+
+  pTableQueryInfo->pTable = NULL;
+  pTableQueryInfo->cur.vgroupIndex = -1;
+
+  // set more initial size of interval/groupby query
+  int32_t initialSize = 16;
+  int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT);
+  if (code != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+
+  return pTableQueryInfo;
+}
+
 void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) {
   if (pTableQueryInfo == NULL) {
     return;
@@ -3771,7 +3793,7 @@ void queryCostStatis(SQInfo *pQInfo) {
 //
 //  int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock);
 //
-//  qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv),
+//  qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv),
 //         pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey);
 //}
 
@@ -4347,7 +4369,12 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) {
 
   STableBlockDist tableBlockDist = {0};
   tableBlockDist.numOfTables     = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables;
-  tableBlockDist.dataBlockInfos  = taosArrayInit(512, sizeof(SFileBlockInfo));
+
+  int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS;
+  tableBlockDist.dataBlockInfos  = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo));
+  taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps);
+  tableBlockDist.maxRows = INT_MIN;
+  tableBlockDist.minRows = INT_MAX;
 
   tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist);
   tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle);
@@ -4429,7 +4456,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRu
   SColumnInfoData infoData = {{0}};
   infoData.info.type = TSDB_DATA_TYPE_BINARY;
   infoData.info.bytes = 1024;
-  infoData.info.colId = TSDB_BLOCK_DIST_COLUMN_INDEX;
+  infoData.info.colId = 0;
   taosArrayPush(pInfo->block.pDataBlock, &infoData);
 
   SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
@@ -5958,10 +5985,7 @@ static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *p
   if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) {
     if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
       return TSDB_TBNAME_COLUMN_INDEX;
-    } else if (pExpr->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) {
-      return TSDB_BLOCK_DIST_COLUMN_INDEX;     
     }
-    
 
     while(j < pTableInfo->numOfTags) {
       if (pExpr->colInfo.colId == pTagCols[j].colId) {
@@ -6531,14 +6555,14 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
 
       type  = TSDB_DATA_TYPE_DOUBLE;
       bytes = tDataTypes[type].bytes;
+    } else if (pExprs[i].base.functionId == TSDB_FUNC_BLKINFO) {
+      SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN};
+      type = s.type;
+      bytes = s.bytes;
     } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) {  // parse the normal column
       SSchema* s = tGetTbnameColumnSchema();
       type = s->type;
       bytes = s->bytes;
-    } else if (pExprs[i].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) {
-      SSchema s = tGetBlockDistColumnSchema(); 
-      type = s.type;
-      bytes = s.bytes;
     } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) {
       // it is a user-defined constant value column
       assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ);
@@ -6551,7 +6575,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
     } else {
       int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols);
       if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) {
-        if (j < TSDB_BLOCK_DIST_COLUMN_INDEX || j >= pTableInfo->numOfTags) {
+        if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) {
           return TSDB_CODE_QRY_INVALID_MSG;
         }
       } else {
@@ -6787,9 +6811,6 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) {
       assert(f < pQueryAttr->numOfCols);
     } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) {
       // do nothing for user-defined constant value result columns
-    } else if (pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) {
-      pColIndex->colIndex = 0;// only one source column, so it must be 0;
-      assert(pQueryAttr->numOfOutput == 1);
     } else {
       int32_t f = 0;
       for (f = 0; f < pQueryAttr->numOfTags; ++f) {
@@ -6799,7 +6820,7 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) {
         }
       }
 
-      assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX || pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX);
+      assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX);
     }
   }
 }
@@ -6991,7 +7012,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
   colIdCheck(pQueryAttr, pQInfo->qId);
 
   // todo refactor
-  pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX);
+  pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.functionId == TSDB_FUNC_BLKINFO);
 
   qDebug("qmsg:%p QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->qId, pQInfo);
   return pQInfo;

src/query/src/qPlan.c

 #include "os.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "qPlan.h"
 #include "qExecutor.h"
 #include "qUtil.h"

src/client/src/tscSchemaUtil.c → src/query/src/qTableMeta.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
 
 #include "os.h"
 #include "taosmsg.h"
-#include "tschemautil.h"
+#include "qTableMeta.h"
 #include "ttokendef.h"
 #include "taosdef.h"
 #include "tutil.h"
-#include "tsclient.h"
 
 int32_t tscGetNumOfTags(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
-  
+
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
-  
+
   if (pTableMeta->tableType == TSDB_NORMAL_TABLE) {
     assert(tinfo.numOfTags == 0);
     return 0;
   }
-  
+
   if (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE) {
     return tinfo.numOfTags;
   }
-  
+
   assert(tinfo.numOfTags == 0);
   return 0;
 }
 
 int32_t tscGetNumOfColumns(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
-  
+
   // table created according to super table, use data from super table
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   return tinfo.numOfColumns;
@@ -54,10 +39,10 @@ SSchema *tscGetTableSchema(const STableMeta *pTableMeta) {
 
 SSchema* tscGetTableTagSchema(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE));
-  
+
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   assert(tinfo.numOfTags > 0);
-  
+
   return tscGetTableColumnSchema(pTableMeta, tinfo.numOfColumns);
 }
 
@@ -68,7 +53,7 @@ STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) {
 
 SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
   assert(pTableMeta != NULL);
-  
+
   SSchema* pSchema = (SSchema*) pTableMeta->schema;
   return &pSchema[colIndex];
 }
@@ -88,7 +73,7 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
 
 STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
   assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2 && pTableMetaMsg->numOfTags >= 0);
-  
+
   int32_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
   STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
 
@@ -97,11 +82,11 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
   pTableMeta->suid      = pTableMetaMsg->suid;
 
   pTableMeta->tableInfo = (STableComInfo) {
-    .numOfTags    = pTableMetaMsg->numOfTags,
-    .precision    = pTableMetaMsg->precision,
-    .numOfColumns = pTableMetaMsg->numOfColumns,
+      .numOfTags    = pTableMetaMsg->numOfTags,
+      .precision    = pTableMetaMsg->precision,
+      .numOfColumns = pTableMetaMsg->numOfColumns,
   };
-  
+
   pTableMeta->id.tid = pTableMetaMsg->tid;
   pTableMeta->id.uid = pTableMetaMsg->uid;
 
@@ -109,69 +94,14 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
   pTableMeta->tversion = pTableMetaMsg->tversion;
 
   tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN);
-  
+
   memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
-  
+
   int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns;
   for(int32_t i = 0; i < numOfTotalCols; ++i) {
     pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
   }
-  
-  return pTableMeta;
-}
-
-bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) {
-  assert(pExisted != NULL && src != NULL);
-  if (pExisted->numOfEps != src->numOfEps) {
-    return false;
-  }
-
-  for(int32_t i = 0; i < pExisted->numOfEps; ++i) {
-    if (pExisted->ep[i].port != src->epAddr[i].port) {
-      return false;
-    }
-
-    if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) {
-      return false;
-    }
-  }
 
-  return true;
-}
-
-SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) {
-  assert(pVgroupMsg != NULL);
-
-  SNewVgroupInfo info = {0};
-  info.numOfEps = pVgroupMsg->numOfEps;
-  info.vgId     = pVgroupMsg->vgId;
-  info.inUse    = 0;   // 0 is the default value of inUse in case of multiple replica
-
-  assert(info.numOfEps >= 1 && info.vgId >= 1);
-  for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) {
-    tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN);
-    info.ep[i].port = pVgroupMsg->epAddr[i].port;
-  }
-
-  return info;
-}
-
-// todo refactor
-UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) {
-  for (int32_t i = 0; i < num; ++i) {
-    while (*input != 0 && *input++ != delim) {
-    };
-  }
-  return input;
-}
-
-UNUSED_FUNC static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) {
-  size_t len = 0;
-  while (*src != delimiter && *src != 0) {
-    *dst++ = *src++;
-    len++;
-  }
-  
-  return len;
+  return pTableMeta;
 }
 

src/query/src/qUtil.c

@@ -581,6 +581,9 @@ void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) {
   tbufWriteUint32(bw, pDist->numOfTables);
   tbufWriteUint16(bw, pDist->numOfFiles);
   tbufWriteUint64(bw, pDist->totalSize);
+  tbufWriteUint64(bw, pDist->totalRows);
+  tbufWriteInt32(bw, pDist->maxRows);
+  tbufWriteInt32(bw, pDist->minRows);
   tbufWriteUint32(bw, pDist->numOfRowsInMemTable);
   tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos));
 
@@ -616,13 +619,16 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi
   pDist->numOfTables = tbufReadUint32(&br);
   pDist->numOfFiles  = tbufReadUint16(&br);
   pDist->totalSize   = tbufReadUint64(&br);
+  pDist->totalRows   = tbufReadUint64(&br);
+  pDist->maxRows     = tbufReadInt32(&br);
+  pDist->minRows     = tbufReadInt32(&br);
   pDist->numOfRowsInMemTable = tbufReadUint32(&br);
-  int64_t numOfBlocks = tbufReadUint64(&br);
+  int64_t numSteps = tbufReadUint64(&br);
 
   bool comp = tbufReadUint8(&br);
   uint32_t compLen = tbufReadUint32(&br);
 
-  size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo));
+  size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo));
 
   char* outputBuf = NULL;
   if (comp) {
@@ -633,12 +639,12 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi
 
     int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf,
                                             (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0);
-    assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo));
+    assert(orignalLen == numSteps *sizeof(SFileBlockInfo));
   } else {
     outputBuf = (char*) tbufReadBinary(&br, &originalLen);
   }
 
-  pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo));
+  pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo));
   if (comp) {
     tfree(outputBuf);
   }

src/tsdb/inc/tsdbCommit.h

@@ -33,6 +33,7 @@ void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
 int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
 void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
 void *tsdbCommitData(STsdbRepo *pRepo);
+int   tsdbApplyRtn(STsdbRepo *pRepo);
 
 static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
   if (fid >= pRtn->maxFid) {

src/tsdb/inc/tsdbint.h

@@ -86,6 +86,7 @@ struct STsdbRepo {
   SMemTable*      mem;
   SMemTable*      imem;
   STsdbFS*        fs;
+  SRtn            rtn;
   tsem_t          readyToCommit;
   pthread_mutex_t mutex;
   bool            repoLocked;

src/tsdb/src/tsdbCommit.c

@@ -86,7 +86,6 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError);
 static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo);
 static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
                                       TSKEY maxKey, int maxRows, int8_t update);
-static int  tsdbApplyRtn(STsdbRepo *pRepo);
 static int  tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
 
 void *tsdbCommitData(STsdbRepo *pRepo) {
@@ -1431,7 +1430,7 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
   return false;
 }
 
-static int tsdbApplyRtn(STsdbRepo *pRepo) {
+int tsdbApplyRtn(STsdbRepo *pRepo) {
   SRtn       rtn;
   SFSIter    fsiter;
   STsdbFS *  pfs = REPO_FS(pRepo);

src/tsdb/src/tsdbFS.c

@@ -33,7 +33,9 @@ static int  tsdbScanDataDir(STsdbRepo *pRepo);
 static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf);
 static int  tsdbRestoreCurrent(STsdbRepo *pRepo);
 static int  tsdbComparTFILE(const void *arg1, const void *arg2);
-static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo);
+static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired);
+static int  tsdbProcessExpiredFS(STsdbRepo *pRepo);
+static int  tsdbCreateMeta(STsdbRepo *pRepo);
 
 // ================== CURRENT file header info
 static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) {
@@ -212,6 +214,8 @@ STsdbFS *tsdbNewFS(STsdbCfg *pCfg) {
     return NULL;
   }
 
+  pfs->intxn = false;
+
   pfs->nstatus = tsdbNewFSStatus(maxFSet);
   if (pfs->nstatus == NULL) {
     tsdbFreeFS(pfs);
@@ -234,22 +238,84 @@ void *tsdbFreeFS(STsdbFS *pfs) {
   return NULL;
 }
 
+static int tsdbProcessExpiredFS(STsdbRepo *pRepo) {
+  tsdbStartFSTxn(pRepo, 0, 0);
+  if (tsdbCreateMeta(pRepo) < 0) {
+    tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (tsdbApplyRtn(pRepo) < 0) {
+    tsdbEndFSTxnWithError(REPO_FS(pRepo));
+    tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+  if (tsdbEndFSTxn(pRepo) < 0) {
+    tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+  return 0;
+}
+
+static int tsdbCreateMeta(STsdbRepo *pRepo) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  SMFile * pOMFile = pfs->cstatus->pmf;
+  SMFile   mf;
+  SDiskID  did;
+
+  if (pOMFile != NULL) {
+    // keep the old meta file
+    tsdbUpdateMFile(pfs, pOMFile);
+    return 0;
+  }
+
+  // Create a new meta file
+  did.level = TFS_PRIMARY_LEVEL;
+  did.id = TFS_PRIMARY_ID;
+  tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
+
+  if (tsdbCreateMFile(&mf, true) < 0) {
+    tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
+
+  if (tsdbUpdateMFileHeader(&mf) < 0) {
+    tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
+    tsdbApplyMFileChange(&mf, pOMFile);
+    return -1;
+  }
+
+  TSDB_FILE_FSYNC(&mf);
+  tsdbCloseMFile(&mf);
+  tsdbUpdateMFile(pfs, &mf);
+
+  return 0;
+}
+
 int tsdbOpenFS(STsdbRepo *pRepo) {
   STsdbFS *pfs = REPO_FS(pRepo);
   char     current[TSDB_FILENAME_LEN] = "\0";
+  int      nExpired = 0;
 
   ASSERT(pfs != NULL);
 
   tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
 
+  tsdbGetRtnSnap(pRepo, &pRepo->rtn);
   if (access(current, F_OK) == 0) {
     if (tsdbOpenFSFromCurrent(pRepo) < 0) {
       tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno));
       return -1;
     }
 
-    tsdbScanAndTryFixDFilesHeader(pRepo);
+    tsdbScanAndTryFixDFilesHeader(pRepo, &nExpired);
+    if (nExpired > 0) {
+      tsdbProcessExpiredFS(pRepo);
+    }
   } else {
+    // should skip expired fileset inside of the function
     if (tsdbRestoreCurrent(pRepo) < 0) {
       tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno));
       return -1;
@@ -1110,6 +1176,11 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) {
 
       ASSERT(tvid == REPO_ID(pRepo));
 
+      if (tfid < pRepo->rtn.minFid) {  // skip file expired
+        ++index;
+        continue;
+      }
+
       if (ftype == 0) {
         fset.fid = tfid;
       } else {
@@ -1206,7 +1277,7 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) {
   }
 }
 
-static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) {
+static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo, int32_t *nExpired) {
   STsdbFS *  pfs = REPO_FS(pRepo);
   SFSStatus *pStatus = pfs->cstatus;
   SDFInfo    info;
@@ -1214,7 +1285,9 @@ static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) {
   for (size_t i = 0; i < taosArrayGetSize(pStatus->df); i++) {
     SDFileSet fset;
     tsdbInitDFileSetEx(&fset, (SDFileSet *)taosArrayGet(pStatus->df, i));
-
+    if (fset.fid < pRepo->rtn.minFid) {
+      ++*nExpired;
+    }
     tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid);
 
     if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) {

src/tsdb/src/tsdbRead.c

@@ -2128,6 +2128,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle;
 
   pTableBlockInfo->totalSize = 0;
+  pTableBlockInfo->totalRows = 0;
   STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb);
 
   // find the start data block in file
@@ -2201,7 +2202,12 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist
         pTableBlockInfo->totalSize += pBlock[j].len;
 
         int32_t numOfRows = pBlock[j].numOfRows;
-        taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows);
+        pTableBlockInfo->totalRows += numOfRows;
+        if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows;
+        if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows;
+        int32_t  stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS;
+        SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex);
+        blockInfo->numBlocksOfStep++;
       }
     }
   }

src/tsdb/src/tsdbSync.c

@@ -424,24 +424,42 @@ static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) {
         }
 
         if (tsdbSendDecision(pSynch, false) < 0) {
-          tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+          tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
           return -1;
         }
       } else {
         // Need to copy from remote
-        tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid);
-
-        // Notify remote to send there file here
-        if (tsdbSendDecision(pSynch, true) < 0) {
-          tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
-          return -1;
+        int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn));
+        if (fidLevel < 0) {  // expired fileset
+          tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid);
+          if (tsdbSendDecision(pSynch, false) < 0) {
+            tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+            return -1;
+          }
+          // Move forward
+          if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+            tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+            return -1;
+          }
+          if (pLSet) {
+            pLSet = tsdbFSIterNext(&fsiter);
+          }
+          // Next loop
+          continue;
+        } else {
+          tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid);
+          // Notify remote to send there file here
+          if (tsdbSendDecision(pSynch, true) < 0) {
+            tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+            return -1;
+          }
         }
 
         // Create local files and copy from remote
         SDiskID   did;
         SDFileSet fset;
 
-        tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id));
+        tfsAllocDisk(fidLevel, &(did.level), &(did.id));
         if (did.level == TFS_UNDECIDED_LEVEL) {
           terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
           tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno));
@@ -548,6 +566,13 @@ static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) {
   STsdbRepo *pRepo = pSynch->pRepo;
   bool toSend = false;
 
+  // skip expired fileset
+  if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) {
+    tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid,
+             pSynch->rtn.minFid);
+    return 0;
+  }
+
   if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) {
     tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno));
     return -1;

src/util/inc/tarray.h

@@ -106,6 +106,14 @@ void* taosArrayGetLast(const SArray* pArray);
  */
 size_t taosArrayGetSize(const SArray* pArray);
 
+/**
+ * set the size of array
+ * @param pArray
+ * @param size size of the array
+ * @return
+ */
+void taosArraySetSize(SArray* pArray, size_t size);
+
 /**
  * insert data into array
  * @param pArray

src/util/src/tarray.c

@@ -115,6 +115,11 @@ void* taosArrayGetLast(const SArray* pArray) {
 
 size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; }
 
+void taosArraySetSize(SArray* pArray, size_t size) {
+  assert(size <= pArray->capacity);
+  pArray->size = size;
+}
+
 void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
   if (pArray == NULL || pData == NULL) {
     return NULL;

tests/examples/c/apitest.c

@@ -402,6 +402,471 @@ void verify_prepare(TAOS* taos) {
   taos_stmt_close(stmt);
 }
 
+
+
+
+
+void verify_prepare2(TAOS* taos) {
+  TAOS_RES* result = taos_query(taos, "drop database if exists test;");
+  taos_free_result(result);
+  usleep(100000);
+  result = taos_query(taos, "create database test;");
+
+  int code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  usleep(100000);
+  taos_select_db(taos, "test");
+
+  // create table
+  const char* sql = "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))";
+  result = taos_query(taos, sql);
+  code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  // insert 10 records
+  struct {
+      int64_t ts[10];
+      int8_t b[10];
+      int8_t v1[10];
+      int16_t v2[10];
+      int32_t v4[10];
+      int64_t v8[10];
+      float f4[10];
+      double f8[10];
+      char bin[10][40];
+      char blob[10][80];
+  } v;
+
+  int32_t *t8_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t16_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t32_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t64_len = malloc(sizeof(int32_t) * 10);
+  int32_t *float_len = malloc(sizeof(int32_t) * 10);
+  int32_t *double_len = malloc(sizeof(int32_t) * 10);
+  int32_t *bin_len = malloc(sizeof(int32_t) * 10);
+  int32_t *blob_len = malloc(sizeof(int32_t) * 10);
+
+  TAOS_STMT* stmt = taos_stmt_init(taos);
+  TAOS_MULTI_BIND params[10];
+  char is_null[10] = {0};
+  
+  params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
+  params[0].buffer_length = sizeof(v.ts[0]);
+  params[0].buffer = v.ts;
+  params[0].length = t64_len;
+  params[0].is_null = is_null;
+  params[0].num = 10;
+
+  params[1].buffer_type = TSDB_DATA_TYPE_BOOL;
+  params[1].buffer_length = sizeof(v.b[0]);
+  params[1].buffer = v.b;
+  params[1].length = t8_len;
+  params[1].is_null = is_null;
+  params[1].num = 10;
+
+  params[2].buffer_type = TSDB_DATA_TYPE_TINYINT;
+  params[2].buffer_length = sizeof(v.v1[0]);
+  params[2].buffer = v.v1;
+  params[2].length = t8_len;
+  params[2].is_null = is_null;
+  params[2].num = 10;
+
+  params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT;
+  params[3].buffer_length = sizeof(v.v2[0]);
+  params[3].buffer = v.v2;
+  params[3].length = t16_len;
+  params[3].is_null = is_null;
+  params[3].num = 10;
+
+  params[4].buffer_type = TSDB_DATA_TYPE_INT;
+  params[4].buffer_length = sizeof(v.v4[0]);
+  params[4].buffer = v.v4;
+  params[4].length = t32_len;
+  params[4].is_null = is_null;
+  params[4].num = 10;
+
+  params[5].buffer_type = TSDB_DATA_TYPE_BIGINT;
+  params[5].buffer_length = sizeof(v.v8[0]);
+  params[5].buffer = v.v8;
+  params[5].length = t64_len;
+  params[5].is_null = is_null;
+  params[5].num = 10;
+
+  params[6].buffer_type = TSDB_DATA_TYPE_FLOAT;
+  params[6].buffer_length = sizeof(v.f4[0]);
+  params[6].buffer = v.f4;
+  params[6].length = float_len;
+  params[6].is_null = is_null;
+  params[6].num = 10;
+
+  params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE;
+  params[7].buffer_length = sizeof(v.f8[0]);
+  params[7].buffer = v.f8;
+  params[7].length = double_len;
+  params[7].is_null = is_null;
+  params[7].num = 10;
+
+  params[8].buffer_type = TSDB_DATA_TYPE_BINARY;
+  params[8].buffer_length = sizeof(v.bin[0]);
+  params[8].buffer = v.bin;
+  params[8].length = bin_len;
+  params[8].is_null = is_null;
+  params[8].num = 10;
+
+  params[9].buffer_type = TSDB_DATA_TYPE_NCHAR;
+  params[9].buffer_length = sizeof(v.blob[0]);
+  params[9].buffer = v.blob;
+  params[9].length = blob_len;
+  params[9].is_null = is_null;
+  params[9].num = 10;
+
+
+  sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)";
+  code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code);
+  }
+
+  code = taos_stmt_set_tbname(stmt, "m1");
+  if (code != 0){
+    printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code);
+  }
+  
+  int64_t ts = 1591060628000;
+  for (int i = 0; i < 10; ++i) {
+    v.ts[i] = ts++;
+    is_null[i] = 0;
+
+    v.b[i] = (int8_t)i % 2;
+    v.v1[i] = (int8_t)i;
+    v.v2[i] = (int16_t)(i * 2);
+    v.v4[i] = (int32_t)(i * 4);
+    v.v8[i] = (int64_t)(i * 8);
+    v.f4[i] = (float)(i * 40);
+    v.f8[i] = (double)(i * 80);
+    for (int j = 0; j < sizeof(v.bin[0]) - 1; ++j) {
+      v.bin[i][j] = (char)(i + '0');
+    }    
+    strcpy(v.blob[i], "一二三四五六七八九十");
+
+    t8_len[i] = sizeof(int8_t);
+    t16_len[i] = sizeof(int16_t);
+    t32_len[i] = sizeof(int32_t);
+    t64_len[i] = sizeof(int64_t);
+    float_len[i] = sizeof(float);
+    double_len[i] = sizeof(double);
+    bin_len[i] = sizeof(v.bin[0]);
+    blob_len[i] = (int32_t)strlen(v.blob[i]);
+  }
+
+  taos_stmt_bind_param_batch(stmt, params);
+  taos_stmt_add_batch(stmt);
+  
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute insert statement.\033[0m\n");
+    return;
+  }
+  taos_stmt_close(stmt);
+
+
+
+  // query the records
+  stmt = taos_stmt_init(taos);
+  taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0);
+  TAOS_BIND qparams[2];
+
+  int8_t v1 = 5;
+  int16_t v2 = 15;
+  qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT;
+  qparams[0].buffer_length = sizeof(v1);
+  qparams[0].buffer = &v1;
+  qparams[0].length = &qparams[0].buffer_length;
+  qparams[0].is_null = NULL;
+
+  qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT;
+  qparams[1].buffer_length = sizeof(v2);
+  qparams[1].buffer = &v2;
+  qparams[1].length = &qparams[1].buffer_length;
+  qparams[1].is_null = NULL;
+
+  taos_stmt_bind_param(stmt, qparams);
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute select statement.\033[0m\n");
+    return;
+  }
+
+  result = taos_stmt_use_result(stmt);
+
+  TAOS_ROW    row;
+  int         rows = 0;
+  int         num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  char        temp[256];
+
+  // fetch the records row by row
+  while ((row = taos_fetch_row(result))) {
+    rows++;
+    taos_print_row(temp, row, fields, num_fields);
+    printf("%s\n", temp);
+  }
+
+  taos_free_result(result);
+  taos_stmt_close(stmt);
+}
+
+
+
+void verify_prepare3(TAOS* taos) {
+  TAOS_RES* result = taos_query(taos, "drop database if exists test;");
+  taos_free_result(result);
+  usleep(100000);
+  result = taos_query(taos, "create database test;");
+
+  int code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  usleep(100000);
+  taos_select_db(taos, "test");
+
+  // create table
+  const char* sql = "create stable st1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10)) tags (id1 int, id2 binary(40))";
+  result = taos_query(taos, sql);
+  code = taos_errno(result);
+  if (code != 0) {
+    printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result));
+    taos_free_result(result);
+    return;
+  }
+  taos_free_result(result);
+
+  TAOS_BIND tags[2];
+
+  int32_t id1 = 1;
+  char id2[40] = "abcdefghijklmnopqrstuvwxyz0123456789";
+  uintptr_t id2_len = strlen(id2);
+  
+  tags[0].buffer_type = TSDB_DATA_TYPE_INT;
+  tags[0].buffer_length = sizeof(int);
+  tags[0].buffer = &id1;
+  tags[0].length = NULL;
+  tags[0].is_null = NULL;
+
+  tags[1].buffer_type = TSDB_DATA_TYPE_BINARY;
+  tags[1].buffer_length = sizeof(id2);
+  tags[1].buffer = id2;
+  tags[1].length = &id2_len;
+  tags[1].is_null = NULL;
+
+
+  // insert 10 records
+  struct {
+      int64_t ts[10];
+      int8_t b[10];
+      int8_t v1[10];
+      int16_t v2[10];
+      int32_t v4[10];
+      int64_t v8[10];
+      float f4[10];
+      double f8[10];
+      char bin[10][40];
+      char blob[10][80];
+  } v;
+
+  int32_t *t8_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t16_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t32_len = malloc(sizeof(int32_t) * 10);
+  int32_t *t64_len = malloc(sizeof(int32_t) * 10);
+  int32_t *float_len = malloc(sizeof(int32_t) * 10);
+  int32_t *double_len = malloc(sizeof(int32_t) * 10);
+  int32_t *bin_len = malloc(sizeof(int32_t) * 10);
+  int32_t *blob_len = malloc(sizeof(int32_t) * 10);
+
+  TAOS_STMT* stmt = taos_stmt_init(taos);
+  TAOS_MULTI_BIND params[10];
+  char is_null[10] = {0};
+  
+  params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
+  params[0].buffer_length = sizeof(v.ts[0]);
+  params[0].buffer = v.ts;
+  params[0].length = t64_len;
+  params[0].is_null = is_null;
+  params[0].num = 10;
+
+  params[1].buffer_type = TSDB_DATA_TYPE_BOOL;
+  params[1].buffer_length = sizeof(v.b[0]);
+  params[1].buffer = v.b;
+  params[1].length = t8_len;
+  params[1].is_null = is_null;
+  params[1].num = 10;
+
+  params[2].buffer_type = TSDB_DATA_TYPE_TINYINT;
+  params[2].buffer_length = sizeof(v.v1[0]);
+  params[2].buffer = v.v1;
+  params[2].length = t8_len;
+  params[2].is_null = is_null;
+  params[2].num = 10;
+
+  params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT;
+  params[3].buffer_length = sizeof(v.v2[0]);
+  params[3].buffer = v.v2;
+  params[3].length = t16_len;
+  params[3].is_null = is_null;
+  params[3].num = 10;
+
+  params[4].buffer_type = TSDB_DATA_TYPE_INT;
+  params[4].buffer_length = sizeof(v.v4[0]);
+  params[4].buffer = v.v4;
+  params[4].length = t32_len;
+  params[4].is_null = is_null;
+  params[4].num = 10;
+
+  params[5].buffer_type = TSDB_DATA_TYPE_BIGINT;
+  params[5].buffer_length = sizeof(v.v8[0]);
+  params[5].buffer = v.v8;
+  params[5].length = t64_len;
+  params[5].is_null = is_null;
+  params[5].num = 10;
+
+  params[6].buffer_type = TSDB_DATA_TYPE_FLOAT;
+  params[6].buffer_length = sizeof(v.f4[0]);
+  params[6].buffer = v.f4;
+  params[6].length = float_len;
+  params[6].is_null = is_null;
+  params[6].num = 10;
+
+  params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE;
+  params[7].buffer_length = sizeof(v.f8[0]);
+  params[7].buffer = v.f8;
+  params[7].length = double_len;
+  params[7].is_null = is_null;
+  params[7].num = 10;
+
+  params[8].buffer_type = TSDB_DATA_TYPE_BINARY;
+  params[8].buffer_length = sizeof(v.bin[0]);
+  params[8].buffer = v.bin;
+  params[8].length = bin_len;
+  params[8].is_null = is_null;
+  params[8].num = 10;
+
+  params[9].buffer_type = TSDB_DATA_TYPE_NCHAR;
+  params[9].buffer_length = sizeof(v.blob[0]);
+  params[9].buffer = v.blob;
+  params[9].length = blob_len;
+  params[9].is_null = is_null;
+  params[9].num = 10;
+
+
+  sql = "insert into ? using st1 tags(?,?) values(?,?,?,?,?,?,?,?,?,?)";
+  code = taos_stmt_prepare(stmt, sql, 0);
+  if (code != 0){
+    printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code);
+  }
+
+  code = taos_stmt_set_tbname_tags(stmt, "m1", tags);
+  if (code != 0){
+    printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code);
+  }
+  
+  int64_t ts = 1591060628000;
+  for (int i = 0; i < 10; ++i) {
+    v.ts[i] = ts++;
+    is_null[i] = 0;
+  
+    v.b[i] = (int8_t)i % 2;
+    v.v1[i] = (int8_t)i;
+    v.v2[i] = (int16_t)(i * 2);
+    v.v4[i] = (int32_t)(i * 4);
+    v.v8[i] = (int64_t)(i * 8);
+    v.f4[i] = (float)(i * 40);
+    v.f8[i] = (double)(i * 80);
+    for (int j = 0; j < sizeof(v.bin[0]) - 1; ++j) {
+      v.bin[i][j] = (char)(i + '0');
+    }    
+    strcpy(v.blob[i], "一二三四五六七八九十");
+
+    t8_len[i] = sizeof(int8_t);
+    t16_len[i] = sizeof(int16_t);
+    t32_len[i] = sizeof(int32_t);
+    t64_len[i] = sizeof(int64_t);
+    float_len[i] = sizeof(float);
+    double_len[i] = sizeof(double);
+    bin_len[i] = sizeof(v.bin[0]);
+    blob_len[i] = (int32_t)strlen(v.blob[i]);
+  }
+
+
+  taos_stmt_bind_param_batch(stmt, params);
+  taos_stmt_add_batch(stmt);
+  
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute insert statement.\033[0m\n");
+    return;
+  }
+  taos_stmt_close(stmt);
+
+  // query the records
+  stmt = taos_stmt_init(taos);
+  taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0);
+
+  TAOS_BIND qparams[2];
+
+  int8_t v1 = 5;
+  int16_t v2 = 15;
+  qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT;
+  qparams[0].buffer_length = sizeof(v1);
+  qparams[0].buffer = &v1;
+  qparams[0].length = &qparams[0].buffer_length;
+  qparams[0].is_null = NULL;
+
+  qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT;
+  qparams[1].buffer_length = sizeof(v2);
+  qparams[1].buffer = &v2;
+  qparams[1].length = &qparams[1].buffer_length;
+  qparams[1].is_null = NULL;
+
+  taos_stmt_bind_param(stmt, qparams);
+  if (taos_stmt_execute(stmt) != 0) {
+    printf("\033[31mfailed to execute select statement.\033[0m\n");
+    return;
+  }
+
+  result = taos_stmt_use_result(stmt);
+
+  TAOS_ROW    row;
+  int         rows = 0;
+  int         num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  char        temp[256];
+
+  // fetch the records row by row
+  while ((row = taos_fetch_row(result))) {
+    rows++;
+    taos_print_row(temp, row, fields, num_fields);
+    printf("%s\n", temp);
+  }
+
+  taos_free_result(result);
+  taos_stmt_close(stmt);
+}
+
+
+
 void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows)
 {
   if (numOfRows > 0) {
@@ -493,6 +958,12 @@ int main(int argc, char *argv[]) {
   printf("************ verify prepare *************\n");
   verify_prepare(taos);
 
+  printf("************ verify prepare2 *************\n");
+  verify_prepare2(taos);
+
+  printf("************ verify prepare3 *************\n");
+  verify_prepare3(taos);
+
   printf("************ verify stream  *************\n");
   verify_stream(taos);
   printf("done\n");

tests/pytest/query/queryInsertValue.py

@@ -45,7 +45,7 @@ class TDTestCase:
         tdSql.query("select * from st")
         tdSql.checkRows(1)
 
-        tdSql.execute("alter table st add column length int")
+        tdSql.execute("alter table st add column len int")
         tdSql.execute("insert into t1 values(now, 1, 2)")
         tdSql.query("select last(*) from st")
         tdSql.checkData(0, 2, 2);

tests/script/api/batchprepare.c

@@ -1046,6 +1046,7 @@ int stmt_funcb_autoctb1(TAOS_STMT *stmt) {
   free(params);
   free(is_null);
   free(no_null);
+  free(tags);
 
   return 0;
 }
@@ -1258,6 +1259,7 @@ int stmt_funcb_autoctb2(TAOS_STMT *stmt) {
   free(params);
   free(is_null);
   free(no_null);
+  free(tags);
 
   return 0;
 }
@@ -1446,6 +1448,7 @@ int stmt_funcb_autoctb3(TAOS_STMT *stmt) {
   free(params);
   free(is_null);
   free(no_null);
+  free(tags);
 
   return 0;
 }
@@ -1635,6 +1638,7 @@ int stmt_funcb_autoctb_e1(TAOS_STMT *stmt) {
   free(params);
   free(is_null);
   free(no_null);
+  free(tags);
 
   return 0;
 }
@@ -1849,6 +1853,7 @@ int stmt_funcb_autoctb_e2(TAOS_STMT *stmt) {
   free(params);
   free(is_null);
   free(no_null);
+  free(tags);
 
   return 0;
 }

tests/script/fullGeneralSuite.sim

@@ -33,6 +33,7 @@ run general/compute/percentile.sim
 run general/compute/stddev.sim
 run general/compute/sum.sim
 run general/compute/top.sim
+run general/compute/block_dist.sim
 run general/db/alter_option.sim
 run general/db/alter_tables_d2.sim
 run general/db/alter_tables_v1.sim

tests/script/general/compute/block_dist.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/exec.sh -n dnode1 -s start
+sleep 2000
+sql connect
+
+$dbPrefix = m_di_db
+$tbPrefix = m_di_tb
+$mtPrefix = m_di_mt
+$ntPrefix = m_di_nt
+$tbNum = 1
+$rowNum = 2000
+
+print =============== step1
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+$nt = $ntPrefix . $i
+
+sql drop database $db -x step1
+step1:
+sql create database $db
+sql use $db
+sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int)
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using $mt tags( $i )
+  
+  $x = 0
+  while $x < $rowNum
+    $cc = $x * 60000
+    $ms = 1601481600000 + $cc
+    sql insert into $tb values ($ms , $x ) 
+    $x = $x + 1
+  endw 
+ 
+  $i = $i + 1
+endw 
+
+sql create table $nt (ts timestamp, tbcol int)
+$x = 0
+while $x < $rowNum
+    $cc = $x * 60000
+    $ms = 1601481600000 + $cc
+    sql insert into $nt values ($ms , $x ) 
+    $x = $x + 1  
+endw
+
+sleep 100
+
+print =============== step2
+$i = 0
+$tb = $tbPrefix . $i
+
+sql select _block_dist() from $tb
+
+if $rows != 1 then
+  print expect 1, actual:$rows
+  return -1
+endi
+
+print =============== step3
+$i = 0
+$mt = $mtPrefix . $i
+sql select _block_dist() from $mt
+
+if $rows != 1 then
+  print expect 1, actual:$rows
+  return -1
+endi
+
+print =============== step4
+$i = 0
+$nt = $ntPrefix . $i
+
+sql select _block_dist() from $nt
+
+if $rows != 1 then
+  print expect 1, actual:$rows
+  return -1
+endi
+
+print =============== clear
+sql drop database $db
+sql show databases
+if $rows != 0 then 
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/general/compute/testSuite.sim

@@ -14,3 +14,4 @@ run general/compute/percentile.sim
 run general/compute/stddev.sim
 run general/compute/sum.sim
 run general/compute/top.sim
+run general/compute/block_dist.sim

tests/script/general/parser/import_file.sim

@@ -18,6 +18,7 @@ system general/parser/gendata.sh
 sql create table tbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16),  imei BINARY(16),  mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2))
 print ====== create tables success, starting import data
 
+sql import into tbx file '~/data.sql'
 sql import into tbx file '~/data.sql'
 
 sql select count(*) from tbx

tests/script/regressionSuite.sim

@@ -32,6 +32,7 @@ run general/compute/percentile.sim
 run general/compute/stddev.sim
 run general/compute/sum.sim
 run general/compute/top.sim
+run general/compute/block_dist.sim
 run general/db/alter_option.sim
 run general/db/alter_tables_d2.sim
 run general/db/alter_tables_v1.sim