Merge branch '3.0' into feature/shm

.clang-format

@@ -5,7 +5,6 @@ AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: true
-AlignConsecutiveMacros: true
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
 AlignTrailingComments: true

2.0/src/query/inc/qExecutor.h

@@ -574,11 +574,11 @@ typedef struct SMultiwayMergeInfo {
 } SMultiwayMergeInfo;
 
 // todo support the disk-based sort
-typedef struct SOrderOperatorInfo {
+typedef struct SSortOperatorInfo {
   int32_t      colIndex;
   int32_t      order;
   SSDataBlock *pDataBlock;
-} SOrderOperatorInfo;
+} SSortOperatorInfo;
 
 void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
 
@@ -609,7 +609,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator
                                         int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
 
 SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
-SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
+SOperatorInfo* createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
 
 SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
 SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);

2.0/src/query/src/qExecutor.c

@@ -2301,7 +2301,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
       }
 
       case OP_Order: {
-        pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order);
+        pRuntimeEnv->proot = createSortOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order);
         break;
       }
 
@@ -5516,7 +5516,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) {
     return NULL;
   }
 
-  SOrderOperatorInfo* pInfo = pOperator->info;
+  SSortOperatorInfo* pInfo = pOperator->info;
 
   SSDataBlock* pBlock = NULL;
   while(1) {
@@ -5556,8 +5556,8 @@ static SSDataBlock* doSort(void* param, bool* newgroup) {
   return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL;
 }
 
-SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) {
-  SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo));
+SOperatorInfo *createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) {
+  SSortOperatorInfo* pInfo = calloc(1, sizeof(SSortOperatorInfo));
 
   {
       SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock));
@@ -6611,7 +6611,7 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
 }
 
 static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
-  SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param;
+  SSortOperatorInfo* pInfo = (SSortOperatorInfo*) param;
   pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
 }
 

example/src/tmq.c

@@ -80,8 +80,9 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  const char* sql = "select * from tu1";
-  pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  /*const char* sql = "select * from tu1";*/
+  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
     return -1;

example/src/tstream.c

@@ -20,7 +20,7 @@
 #include "taos.h"
 
 int32_t init_env() {
-  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
   if (pConn == NULL) {
     return -1;
   }
@@ -65,7 +65,7 @@ int32_t init_env() {
 int32_t create_stream() {
   printf("create stream\n");
   TAOS_RES* pRes;
-  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
   if (pConn == NULL) {
     return -1;
   }

include/client/consumer/consumer.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_CONSUMER_H_
-#define _TD_CONSUMER_H_
-
-#include "tlist.h"
-#include "tarray.h"
-#include "hash.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-  //consumer handle
-  struct tmq_consumer_t;
-  typedef struct tmq_consumer_t tmq_consumer_t;
-
-  //consumer config
-  struct tmq_consumer_config_t;
-  typedef struct tmq_consumer_config_t tmq_consumer_config_t;
-
-  //response err
-  struct tmq_resp_err_t;
-  typedef struct tmq_resp_err_t tmq_resp_err_t;
-
-  struct tmq_message_t;
-  typedef struct tmq_message_t tmq_message_t;
-
-  struct tmq_col_batch_t;
-  typedef struct tmq_col_batch_t tmq_col_batch_t;
-
-  //get content of message
-  tmq_col_batch_t* tmq_get_msg_col_by_idx(tmq_message_t*, int32_t col_id);
-  tmq_col_batch_t* tmq_get_msg_col_by_name(tmq_message_t*, const char*);
-
-  //consumer config
-  int32_t tmq_conf_set(tmq_consumer_config_t* , const char* config_key, const char* config_value, char* errstr, int32_t errstr_cap);
-
-  //consumer initialization
-  //resouces are supposed to be free by users by calling tmq_consumer_destroy
-  tmq_consumer_t* tmq_consumer_new(tmq_consumer_config_t* , char* errstr, int32_t errstr_cap);
-
-  //subscribe
-  tmq_resp_err_t tmq_subscribe(tmq_consumer_t*, const SList*);
-  tmq_resp_err_t tmq_unsubscribe(tmq_consumer_t*);
-
-  //consume
-  //resouces are supposed to be free by users by calling tmq_message_destroy
-  tmq_message_t* tmq_consume_poll(tmq_consumer_t*, int64_t blocking_time);
-
-  //destroy message and free memory
-  void tmq_message_destroy(tmq_message_t*);
-
-  //close consumer
-  int32_t tmq_consumer_close(tmq_consumer_t*);
-
-  //destroy consumer
-  void tmq_consumer_destroy(tmq_message_t*);
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_CONSUMER_H_*/

include/client/stream/stream.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_STREAM_H_
-#define _TD_STREAM_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_STREAM_H_*/
\ No newline at end of file

include/client/taos.h

@@ -26,9 +26,11 @@ extern "C" {
 typedef void   TAOS;
 typedef void   TAOS_STMT;
 typedef void   TAOS_RES;
+typedef void **TAOS_ROW;
+#if 0
 typedef void   TAOS_STREAM;
+#endif
 typedef void   TAOS_SUB;
-typedef void **TAOS_ROW;
 
 // Data type definition
 #define TSDB_DATA_TYPE_NULL       0   // 1 bytes
@@ -149,6 +151,7 @@ DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@@ -179,15 +182,20 @@ DLL_EXPORT int         taos_errno(TAOS_RES *tres);
 DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
 DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
 
+// Shuduo: temporary enable for app build
+#if 1
 typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp,
                                     void *param, int interval);
 DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
+#endif
 
+#if 0
 DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
                                          int64_t stime, void *param, void (*callback)(void *));
 DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
+#endif
 
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
@@ -215,8 +223,12 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 
-DLL_EXPORT tmq_t      *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
-DLL_EXPORT tmq_t      *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+// will be removed in 3.0
+DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+
+// will replace last one
+DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
@@ -255,20 +267,26 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
-DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
-DLL_EXPORT char    *tmq_get_topic_name(tmq_message_t *message);
-DLL_EXPORT void    *tmq_get_topic_schema(tmq_t *tmq, const char *topic);
-DLL_EXPORT void     tmq_message_destroy(tmq_message_t *tmq_message);
+DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
+DLL_EXPORT char       *tmq_get_topic_name(tmq_message_t *message);
+DLL_EXPORT int32_t     tmq_get_vgroup_id(tmq_message_t *message);
+DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
+DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
+DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
+DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
+DLL_EXPORT void        tmq_message_destroy(tmq_message_t *tmq_message);
 
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
+#if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
+#endif
 
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
 
-/* -------------------------------- OTHER -------------------------------- */
+/* ------------------------------ TMQ END -------------------------------- */
+#if 1 // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
-
-DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
+#endif
 
 #ifdef __cplusplus
 }

include/common/tdatablock.h

@@ -29,8 +29,9 @@ typedef struct SCorEpSet {
 } SCorEpSet;
 
 typedef struct SBlockOrderInfo {
+  bool             nullFirst;
   int32_t          order;
-  int32_t          colIndex;
+  int32_t          slotId;
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
@@ -176,7 +177,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 
 SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
 
-int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
+int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 
 int32_t      blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);

include/common/tdataformat.h

@@ -63,7 +63,7 @@ extern "C" {
 typedef struct {
   col_id_t colId;        // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
   int32_t  type : 8;     // column type
-  int32_t  bytes : 24;   // column bytes (restore to int32_t in case of misuse)
+  int32_t  bytes : 24;   // column bytes (0~16M)
   int32_t  sma : 8;      // block SMA: 0, no SMA, 1, sum/min/max, 2, ...
   int32_t  offset : 24;  // point offset in STpRow after the header part.
 } STColumn;
@@ -81,12 +81,12 @@ typedef struct {
 
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
-  int32_t version;    // version
-  int32_t numOfCols;  // Number of columns appended
-  int32_t tlen;   // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
-                  // (bytes))
-  uint16_t flen;  // First part length in a STpRow after the header part
-  uint16_t vlen;  // pure value part length, excluded the overhead (bytes only)
+  int32_t      numOfCols;  // Number of columns appended
+  schema_ver_t version;    // schema version
+  uint16_t     flen;       // First part length in a STpRow after the header part
+  int32_t      vlen;       // pure value part length, excluded the overhead (bytes only)
+  int32_t      tlen;       // maximum length of a STpRow without the header part
+                           // (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
   STColumn columns[];
 } STSchema;
 
@@ -120,13 +120,13 @@ static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) {
 
 // ----------------- SCHEMA BUILDER DEFINITION
 typedef struct {
-  int32_t   tCols;
-  int32_t   nCols;
-  int32_t   tlen;
-  uint16_t  flen;
-  uint16_t  vlen;
-  int32_t   version;
-  STColumn *columns;
+  int32_t      tCols;
+  int32_t      nCols;
+  schema_ver_t version;
+  uint16_t     flen;
+  int32_t      vlen;
+  int32_t      tlen;
+  STColumn    *columns;
 } STSchemaBuilder;
 
 #define TD_VTYPE_BITS  2  // val type
@@ -136,9 +136,9 @@ typedef struct {
 #define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
 #define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
 
-int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
+int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
-void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
+void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 

include/common/tmsg.h

@@ -801,7 +801,10 @@ typedef struct SVgroupInfo {
   uint32_t hashBegin;
   uint32_t hashEnd;
   SEpSet   epSet;
-  int32_t  numOfTable;  // unit is TSDB_TABLE_NUM_UNIT
+  union {
+    int32_t numOfTable;  // unit is TSDB_TABLE_NUM_UNIT
+    int32_t taskId;      // used in stream
+  };
 } SVgroupInfo;
 
 typedef struct {
@@ -2212,23 +2215,6 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
   return buf;
 }
 
-typedef struct {
-  int64_t uid;
-  int32_t numOfRows;
-  char*   colData;
-} SMqTbData;
-
-typedef struct {
-  char       topicName[TSDB_TOPIC_FNAME_LEN];
-  int64_t    committedOffset;
-  int64_t    reqOffset;
-  int64_t    rspOffset;
-  int32_t    skipLogNum;
-  int32_t    bodyLen;
-  int32_t    numOfTb;
-  SMqTbData* tbData;
-} SMqTopicData;
-
 typedef struct {
   int8_t  mqMsgType;
   int32_t code;
@@ -2256,8 +2242,11 @@ typedef struct {
 } SMqSubVgEp;
 
 typedef struct {
-  char    topic[TSDB_TOPIC_FNAME_LEN];
-  SArray* vgs;  // SArray<SMqSubVgEp>
+  char        topic[TSDB_TOPIC_FNAME_LEN];
+  int8_t      isSchemaAdaptive;
+  SArray*     vgs;  // SArray<SMqSubVgEp>
+  int32_t     numOfFields;
+  TAOS_FIELD* fields;
 } SMqSubTopicEp;
 
 typedef struct {
@@ -2278,32 +2267,6 @@ typedef struct {
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-typedef struct {
-  int32_t curBlock;
-  int32_t curRow;
-  void**  uData;
-} SMqRowIter;
-
-struct tmq_message_t {
-  SMqPollRsp msg;
-  void*      vg;
-  SMqRowIter iter;
-};
-
-#if 0
-struct tmq_message_t {
-  SMqRspHead head;
-  union {
-    SMqPollRsp       consumeRsp;
-    SMqCMGetSubEpRsp getEpRsp;
-  };
-  void*   extra;
-  int32_t curBlock;
-  int32_t curRow;
-  void**  uData;
-};
-#endif
-
 static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
@@ -2328,17 +2291,21 @@ static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
 static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
   int32_t tlen = 0;
   tlen += taosEncodeString(buf, pTopicEp->topic);
+  tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive);
   int32_t sz = taosArrayGetSize(pTopicEp->vgs);
   tlen += taosEncodeFixedI32(buf, sz);
   for (int32_t i = 0; i < sz; i++) {
     SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i);
     tlen += tEncodeSMqSubVgEp(buf, pVgEp);
   }
+  tlen += taosEncodeFixedI32(buf, pTopicEp->numOfFields);
+  // tlen += taosEncodeBinary(buf, pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
   return tlen;
 }
 
 static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) {
   buf = taosDecodeStringTo(buf, pTopicEp->topic);
+  buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive);
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
   pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp));
@@ -2350,6 +2317,8 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
     buf = tDecodeSMqSubVgEp(buf, &vgEp);
     taosArrayPush(pTopicEp->vgs, &vgEp);
   }
+  buf = taosDecodeFixedI32(buf, &pTopicEp->numOfFields);
+  // buf = taosDecodeBinary(buf, (void**)&pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
   return buf;
 }
 

include/common/ttokendef.h

@@ -139,46 +139,54 @@
 #define TK_TOPIC                          121
 #define TK_AS                             122
 #define TK_NK_BOOL                        123
-#define TK_NK_VARIABLE                    124
-#define TK_BETWEEN                        125
-#define TK_IS                             126
-#define TK_NULL                           127
-#define TK_NK_LT                          128
-#define TK_NK_GT                          129
-#define TK_NK_LE                          130
-#define TK_NK_GE                          131
-#define TK_NK_NE                          132
-#define TK_MATCH                          133
-#define TK_NMATCH                         134
-#define TK_IN                             135
-#define TK_JOIN                           136
-#define TK_INNER                          137
-#define TK_SELECT                         138
-#define TK_DISTINCT                       139
-#define TK_WHERE                          140
-#define TK_PARTITION                      141
-#define TK_BY                             142
-#define TK_SESSION                        143
-#define TK_STATE_WINDOW                   144
-#define TK_SLIDING                        145
-#define TK_FILL                           146
-#define TK_VALUE                          147
-#define TK_NONE                           148
-#define TK_PREV                           149
-#define TK_LINEAR                         150
-#define TK_NEXT                           151
-#define TK_GROUP                          152
-#define TK_HAVING                         153
-#define TK_ORDER                          154
-#define TK_SLIMIT                         155
-#define TK_SOFFSET                        156
-#define TK_LIMIT                          157
-#define TK_OFFSET                         158
-#define TK_ASC                            159
-#define TK_DESC                           160
-#define TK_NULLS                          161
-#define TK_FIRST                          162
-#define TK_LAST                           163
+#define TK_NULL                           124
+#define TK_NK_VARIABLE                    125
+#define TK_NK_UNDERLINE                   126
+#define TK_ROWTS                          127
+#define TK_TBNAME                         128
+#define TK_QSTARTTS                       129
+#define TK_QENDTS                         130
+#define TK_WSTARTTS                       131
+#define TK_WENDTS                         132
+#define TK_WDURATION                      133
+#define TK_BETWEEN                        134
+#define TK_IS                             135
+#define TK_NK_LT                          136
+#define TK_NK_GT                          137
+#define TK_NK_LE                          138
+#define TK_NK_GE                          139
+#define TK_NK_NE                          140
+#define TK_MATCH                          141
+#define TK_NMATCH                         142
+#define TK_IN                             143
+#define TK_JOIN                           144
+#define TK_INNER                          145
+#define TK_SELECT                         146
+#define TK_DISTINCT                       147
+#define TK_WHERE                          148
+#define TK_PARTITION                      149
+#define TK_BY                             150
+#define TK_SESSION                        151
+#define TK_STATE_WINDOW                   152
+#define TK_SLIDING                        153
+#define TK_FILL                           154
+#define TK_VALUE                          155
+#define TK_NONE                           156
+#define TK_PREV                           157
+#define TK_LINEAR                         158
+#define TK_NEXT                           159
+#define TK_GROUP                          160
+#define TK_HAVING                         161
+#define TK_ORDER                          162
+#define TK_SLIMIT                         163
+#define TK_SOFFSET                        164
+#define TK_LIMIT                          165
+#define TK_OFFSET                         166
+#define TK_ASC                            167
+#define TK_DESC                           168
+#define TK_NULLS                          169
+#define TK_FIRST                          170
+#define TK_LAST                           171
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/common/ttypes.h

@@ -30,6 +30,7 @@ typedef uint8_t  TDRowValT;
 typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 typedef int32_t  col_bytes_t;
+typedef uint16_t schema_ver_t;
 
 #pragma pack(push, 1)
 typedef struct {

include/libs/executor/executor.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "tcommon.h"
 #include "query.h"
+#include "tcommon.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -36,7 +36,7 @@ typedef struct SReadHandle {
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
 #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
-  
+
 /**
  * Create the exec task for streaming mode
  * @param pMsg
@@ -54,6 +54,16 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
  */
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
 
+/**
+ * Set multiple input data blocks for the stream scan.
+ * @param tinfo
+ * @param pBlocks
+ * @param numOfInputBlock
+ * @param type
+ * @return
+ */
+int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
+
 /**
  * Update the table id list, add or remove.
  *
@@ -86,16 +96,6 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
  */
 int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds);
 
-/**
- * Retrieve the produced results information, if current query is not paused or completed,
- * this function will be blocked to wait for the query execution completed or paused,
- * in which case enough results have been produced already.
- *
- * @param tinfo
- * @return
- */
-int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
-
 /**
  * kill the ongoing query and free the query handle and corresponding resources automatically
  * @param tinfo  qhandle
@@ -158,50 +158,6 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
  */
 int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
 
-//================================================================================================
-// query handle management
-/**
- * Query handle mgmt object
- * @param vgId
- * @return
- */
-void* qOpenTaskMgmt(int32_t vgId);
-
-/**
- * broadcast the close information and wait for all query stop.
- * @param pExecutor
- */
-void qTaskMgmtNotifyClosing(void* pExecutor);
-
-/**
- * Re-open the query handle management module when opening the vnode again.
- * @param pExecutor
- */
-void qQueryMgmtReOpen(void* pExecutor);
-
-/**
- * Close query mgmt and clean up resources.
- * @param pExecutor
- */
-void qCleanupTaskMgmt(void* pExecutor);
-
-/**
- * Add the query into the query mgmt object
- * @param pMgmt
- * @param qId
- * @param qInfo
- * @return
- */
-void** qRegisterTask(void* pMgmt, uint64_t qId, void* qInfo);
-
-/**
- * acquire the query handle according to the key from query mgmt object.
- * @param pMgmt
- * @param key
- * @return
- */
-void** qAcquireTask(void* pMgmt, uint64_t key);
-
 /**
  * release the query handle and decrease the reference count in cache
  * @param pMgmt
@@ -211,14 +167,6 @@ void** qAcquireTask(void* pMgmt, uint64_t key);
  */
 void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
 
-/**
- * De-register the query handle from the management module and free it immediately.
- * @param pMgmt
- * @param pQInfo
- * @return
- */
-void** qDeregisterQInfo(void* pMgmt, void* pQInfo);
-
 void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
 
 #ifdef __cplusplus

include/libs/function/functionMgt.h

@@ -96,7 +96,16 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_SERVER_SERSION,
   FUNCTION_TYPE_SERVER_STATUS,
   FUNCTION_TYPE_CURRENT_USER,
-  FUNCTION_TYPE_USER
+  FUNCTION_TYPE_USER,
+
+  // pseudo column function
+  FUNCTION_TYPE_ROWTS = 3500,
+  FUNCTION_TYPE_TBNAME,
+  FUNCTION_TYPE_QSTARTTS,
+  FUNCTION_TYPE_QENDTS,
+  FUNCTION_TYPE_WSTARTTS,
+  FUNCTION_TYPE_WENDTS,
+  FUNCTION_TYPE_WDURATION
 } EFunctionType;
 
 struct SqlFunctionCtx;
@@ -118,6 +127,8 @@ bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
+bool fmIsWindowPseudoColumnFunc(int32_t funcId);
+bool fmIsWindowClauseFunc(int32_t funcId);
 
 int32_t fmFuncScanType(int32_t funcId);
 

include/libs/nodes/nodes.h

@@ -121,6 +121,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_EXCHANGE,
   QUERY_NODE_LOGIC_PLAN_WINDOW,
   QUERY_NODE_LOGIC_PLAN_SORT,
+  QUERY_NODE_LOGIC_PLAN_PARTITION,
   QUERY_NODE_LOGIC_SUBPLAN,
   QUERY_NODE_LOGIC_PLAN,
 

include/libs/nodes/plannodes.h

@@ -66,7 +66,11 @@ typedef struct SAggLogicNode {
 typedef struct SProjectLogicNode {
   SLogicNode node;
   SNodeList* pProjections;
-  char stmtName[TSDB_TABLE_NAME_LEN]; 
+  char stmtName[TSDB_TABLE_NAME_LEN];
+  int64_t limit;
+  int64_t offset;
+  int64_t slimit;
+  int64_t soffset;
 } SProjectLogicNode;
 
 typedef struct SVnodeModifLogicNode {
@@ -106,6 +110,11 @@ typedef struct SSortLogicNode {
   SNodeList* pSortKeys;
 } SSortLogicNode;
 
+typedef struct SPartitionLogicNode {
+  SLogicNode node;
+  SNodeList* pPartitionKeys;
+} SPartitionLogicNode;
+
 typedef enum ESubplanType {
   SUBPLAN_TYPE_MERGE = 1,
   SUBPLAN_TYPE_PARTIAL,
@@ -150,7 +159,8 @@ typedef struct SDataBlockDescNode {
   ENodeType type;
   int16_t dataBlockId;
   SNodeList* pSlots;
-  int32_t resultRowSize;
+  int32_t totalRowSize;
+  int32_t outputRowSize;
   int16_t precision;
 } SDataBlockDescNode;
 
@@ -195,6 +205,10 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
+  int64_t limit;
+  int64_t offset;
+  int64_t slimit;
+  int64_t soffset;
 } SProjectPhysiNode;
 
 typedef struct SJoinPhysiNode {
@@ -283,10 +297,17 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
 } SSubplan;
 
+typedef enum EQueryMode {
+  QUERY_MODE_NORMAL = 1,
+  QUERY_MODE_EXPLAIN,
+  QUERY_MODE_EXPLAIN_AN
+} EQueryMode;
+
 typedef struct SQueryPlan {
   ENodeType type;
   uint64_t queryId;
   int32_t numOfSubplans;
+
   SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
 } SQueryPlan;
 

include/libs/nodes/querynodes.h

@@ -180,8 +180,8 @@ typedef struct SOrderByExprNode {
 
 typedef struct SLimitNode {
   ENodeType type; // QUERY_NODE_LIMIT
-  uint64_t limit;
-  uint64_t offset;
+  int64_t limit;
+  int64_t offset;
 } SLimitNode;
 
 typedef struct SStateWindowNode {

include/libs/stream/tstream.h

@@ -30,6 +30,11 @@ enum {
   STREAM_TASK_STATUS__STOP,
 };
 
+enum {
+  STREAM_CREATED_BY__USER = 1,
+  STREAM_CREATED_BY__SMA,
+};
+
 #if 0
 // pipe  -> fetch/pipe queue
 // merge -> merge      queue

source/client/src/clientMain.c

-#include "os.h"
-#include "tref.h"
-#include "trpc.h"
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "os.h"
 #include "query.h"
-#include "tmsg.h"
 #include "tglobal.h"
-#include "catalog.h"
+#include "tmsg.h"
+#include "tref.h"
+#include "trpc.h"
 #include "version.h"
 
 #define TSC_VAR_NOT_RELEASE 1
@@ -24,7 +39,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
     }
   }
 
-  int ret = taos_options_imp(option, (const char*)arg);
+  int ret = taos_options_imp(option, (const char *)arg);
   atomic_store_32(&lock, 0);
   return ret;
 }
@@ -69,13 +84,13 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
   return taos_connect_internal(ip, user, pass, NULL, db, port);
 }
 
-void taos_close(TAOS* taos) {
+void taos_close(TAOS *taos) {
   if (taos == NULL) {
     return;
   }
 
   STscObj *pTscObj = (STscObj *)taos;
-  tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
+  tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
 
   /*taosRemoveRef(clientConnRefPool, pTscObj->id);*/
 }
@@ -85,48 +100,46 @@ int taos_errno(TAOS_RES *tres) {
     return terrno;
   }
 
-  return ((SRequestObj*) tres)->code;
+  return ((SRequestObj *)tres)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
-  SRequestObj *pRequest = (SRequestObj *) res;
+  SRequestObj *pRequest = (SRequestObj *)res;
 
   if (pRequest == NULL) {
-    return (const char*) tstrerror(terrno);
+    return (const char *)tstrerror(terrno);
   }
 
   if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
     return pRequest->msgBuf;
   } else {
-    return (const char*)tstrerror(pRequest->code);
+    return (const char *)tstrerror(pRequest->code);
   }
 }
 
 void taos_free_result(TAOS_RES *res) {
-  SRequestObj* pRequest = (SRequestObj*) res;
+  SRequestObj *pRequest = (SRequestObj *)res;
   destroyRequest(pRequest);
 }
 
-int  taos_field_count(TAOS_RES *res) {
+int taos_field_count(TAOS_RES *res) {
   if (res == NULL) {
     return 0;
   }
 
-  SRequestObj* pRequest = (SRequestObj*) res;
-  SReqResultInfo* pResInfo = &pRequest->body.resInfo;
+  SRequestObj    *pRequest = (SRequestObj *)res;
+  SReqResultInfo *pResInfo = &pRequest->body.resInfo;
   return pResInfo->numOfCols;
 }
 
-int  taos_num_fields(TAOS_RES *res) {
-  return taos_field_count(res);
-}
+int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
 
 TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   if (taos_num_fields(res) == 0) {
     return NULL;
   }
 
-  SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo);
+  SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
   return pResInfo->fields;
 }
 
@@ -135,7 +148,7 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
     return NULL;
   }
 
-  return taos_query_l(taos, sql, (int32_t) strlen(sql));
+  return taos_query_l(taos, sql, (int32_t)strlen(sql));
 }
 
 TAOS_ROW taos_fetch_row(TAOS_RES *res) {
@@ -143,18 +156,16 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   }
 
-  SRequestObj *pRequest = (SRequestObj *) res;
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->code != TSDB_CODE_SUCCESS ||
-      taos_num_fields(res) == 0) {
+  SRequestObj *pRequest = (SRequestObj *)res;
+  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
+      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
     return NULL;
   }
 
   return doFetchRow(pRequest, true);
 }
 
-int  taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
+int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
   int32_t len = 0;
   for (int i = 0; i < num_fields; ++i) {
     if (i > 0) {
@@ -213,7 +224,7 @@ int  taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
 
       case TSDB_DATA_TYPE_BINARY:
       case TSDB_DATA_TYPE_NCHAR: {
-        int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE);
+        int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
         if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
           assert(charLen <= fields[i].bytes && charLen >= 0);
         } else {
@@ -238,31 +249,44 @@ int  taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
   return len;
 }
 
-int* taos_fetch_lengths(TAOS_RES *res) {
+int *taos_fetch_lengths(TAOS_RES *res) {
   if (res == NULL) {
     return NULL;
   }
 
-  return ((SRequestObj*) res)->body.resInfo.length;
+  return ((SRequestObj *)res)->body.resInfo.length;
 }
 
 // todo intergrate with tDataTypes
 const char *taos_data_type(int type) {
   switch (type) {
-    case TSDB_DATA_TYPE_NULL:            return "TSDB_DATA_TYPE_NULL";
-    case TSDB_DATA_TYPE_BOOL:            return "TSDB_DATA_TYPE_BOOL";
-    case TSDB_DATA_TYPE_TINYINT:         return "TSDB_DATA_TYPE_TINYINT";
-    case TSDB_DATA_TYPE_SMALLINT:        return "TSDB_DATA_TYPE_SMALLINT";
-    case TSDB_DATA_TYPE_INT:             return "TSDB_DATA_TYPE_INT";
-    case TSDB_DATA_TYPE_BIGINT:          return "TSDB_DATA_TYPE_BIGINT";
-    case TSDB_DATA_TYPE_FLOAT:           return "TSDB_DATA_TYPE_FLOAT";
-    case TSDB_DATA_TYPE_DOUBLE:          return "TSDB_DATA_TYPE_DOUBLE";
-    case TSDB_DATA_TYPE_VARCHAR:         return "TSDB_DATA_TYPE_VARCHAR";
-//    case TSDB_DATA_TYPE_BINARY:          return "TSDB_DATA_TYPE_VARCHAR";
-    case TSDB_DATA_TYPE_TIMESTAMP:       return "TSDB_DATA_TYPE_TIMESTAMP";
-    case TSDB_DATA_TYPE_NCHAR:           return "TSDB_DATA_TYPE_NCHAR";
-    case TSDB_DATA_TYPE_JSON:            return "TSDB_DATA_TYPE_JSON";
-    default: return "UNKNOWN";
+    case TSDB_DATA_TYPE_NULL:
+      return "TSDB_DATA_TYPE_NULL";
+    case TSDB_DATA_TYPE_BOOL:
+      return "TSDB_DATA_TYPE_BOOL";
+    case TSDB_DATA_TYPE_TINYINT:
+      return "TSDB_DATA_TYPE_TINYINT";
+    case TSDB_DATA_TYPE_SMALLINT:
+      return "TSDB_DATA_TYPE_SMALLINT";
+    case TSDB_DATA_TYPE_INT:
+      return "TSDB_DATA_TYPE_INT";
+    case TSDB_DATA_TYPE_BIGINT:
+      return "TSDB_DATA_TYPE_BIGINT";
+    case TSDB_DATA_TYPE_FLOAT:
+      return "TSDB_DATA_TYPE_FLOAT";
+    case TSDB_DATA_TYPE_DOUBLE:
+      return "TSDB_DATA_TYPE_DOUBLE";
+    case TSDB_DATA_TYPE_VARCHAR:
+      return "TSDB_DATA_TYPE_VARCHAR";
+      //    case TSDB_DATA_TYPE_BINARY:          return "TSDB_DATA_TYPE_VARCHAR";
+    case TSDB_DATA_TYPE_TIMESTAMP:
+      return "TSDB_DATA_TYPE_TIMESTAMP";
+    case TSDB_DATA_TYPE_NCHAR:
+      return "TSDB_DATA_TYPE_NCHAR";
+    case TSDB_DATA_TYPE_JSON:
+      return "TSDB_DATA_TYPE_JSON";
+    default:
+      return "UNKNOWN";
   }
 }
 
@@ -273,8 +297,8 @@ int taos_affected_rows(TAOS_RES *res) {
     return 0;
   }
 
-  SRequestObj* pRequest = (SRequestObj*) res;
-  SReqResultInfo* pResInfo = &pRequest->body.resInfo;
+  SRequestObj    *pRequest = (SRequestObj *)res;
+  SReqResultInfo *pResInfo = &pRequest->body.resInfo;
   return pResInfo->numOfRows;
 }
 
@@ -295,8 +319,8 @@ int taos_select_db(TAOS *taos, const char *db) {
   char sql[256] = {0};
   snprintf(sql, tListLen(sql), "use %s", db);
 
-  TAOS_RES* pRequest = taos_query(taos, sql);
-  int32_t code = taos_errno(pRequest);
+  TAOS_RES *pRequest = taos_query(taos, sql);
+  int32_t   code = taos_errno(pRequest);
 
   taos_free_result(pRequest);
   return code;
@@ -307,61 +331,57 @@ void taos_stop_query(TAOS_RES *res) {
     return;
   }
 
-  SRequestObj* pRequest = (SRequestObj*) res;
-  int32_t numOfFields = taos_num_fields(pRequest);
+  SRequestObj *pRequest = (SRequestObj *)res;
+  int32_t      numOfFields = taos_num_fields(pRequest);
 
   // It is not a query, no need to stop.
   if (numOfFields == 0) {
     return;
   }
 
-//  scheduleCancelJob(pRequest->body.pQueryJob);
+  //  scheduleCancelJob(pRequest->body.pQueryJob);
 }
 
 bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
-  SRequestObj* pRequestObj = res;
-  SReqResultInfo* pResultInfo = &pRequestObj->body.resInfo;
+  SRequestObj    *pRequestObj = res;
+  SReqResultInfo *pResultInfo = &pRequestObj->body.resInfo;
   if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
     return true;
   }
 
-  SResultColumn* pCol = &pRequestObj->body.resInfo.pCol[col];
+  SResultColumn *pCol = &pRequestObj->body.resInfo.pCol[col];
   return colDataIsNull_f(pCol->nullbitmap, row);
 }
 
-int  taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
+int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   if (res == NULL) {
     return 0;
   }
 
-  SRequestObj *pRequest = (SRequestObj *) res;
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
-      pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->code != TSDB_CODE_SUCCESS ||
-      taos_num_fields(res) == 0) {
+  SRequestObj *pRequest = (SRequestObj *)res;
+  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
+      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
     return 0;
   }
 
   doFetchRow(pRequest, false);
 
   // TODO refactor
-  SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
+  SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
   pResultInfo->current = pResultInfo->numOfRows;
   *rows = pResultInfo->row;
 
   return pResultInfo->numOfRows;
 }
 
-int  taos_validate_sql(TAOS *taos, const char *sql) {
-  return true;
-}
+int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
 
 const char *taos_get_server_info(TAOS *taos) {
   if (taos == NULL) {
     return NULL;
   }
 
-  STscObj* pTscObj = (STscObj*) taos;
+  STscObj *pTscObj = (STscObj *)taos;
   return pTscObj->ver;
 }
 
@@ -387,52 +407,62 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
     // TODO
 }
 
-TAOS_STMT* taos_stmt_init(TAOS* taos) {
-    // TODO
-    return NULL;
+TAOS_STMT *taos_stmt_init(TAOS *taos) {
+  // TODO
+  return NULL;
 }
 
-int taos_stmt_close(TAOS_STMT* stmt) {
-    // TODO
-    return -1;
+int taos_stmt_close(TAOS_STMT *stmt) {
+  // TODO
+  return -1;
 }
 
-int taos_stmt_execute(TAOS_STMT* stmt) {
-    // TODO
-    return -1;
+int taos_stmt_execute(TAOS_STMT *stmt) {
+  // TODO
+  return -1;
 }
 
 char *taos_stmt_errstr(TAOS_STMT *stmt) {
-    // TODO
-    return NULL;
+  // TODO
+  return NULL;
 }
 
-int taos_stmt_affected_rows(TAOS_STMT* stmt) {
-    // TODO
-    return -1;
+int taos_stmt_affected_rows(TAOS_STMT *stmt) {
+  // TODO
+  return -1;
 }
 
-TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
-    // TODO
-    return NULL;
+TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
+  // TODO
+  return NULL;
 }
 
-int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) {
-    // TODO
-    return -1;
+int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) {
+  // TODO
+  return -1;
 }
 
-int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
-    // TODO
-    return -1;
+int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
+  // TODO
+  return -1;
+}
+
+int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
+  // TODO
+  return -1;
+}
+
+int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
+  // TODO
+  return -1;
 }
 
-int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
+int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
     // TODO
     return -1;
 }
 
-int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
+int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
     // TODO
     return -1;
 }
@@ -442,6 +472,11 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) {
     return -1;
 }
 
+TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
+    // TODO
+    return NULL;
+}
+
 int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
     // TODO
     return -1;

source/client/src/tmq.c

@@ -26,6 +26,18 @@
 #include "tqueue.h"
 #include "tref.h"
 
+typedef struct {
+  int32_t curBlock;
+  int32_t curRow;
+  void**  uData;
+} SMqRowIter;
+
+struct tmq_message_t {
+  SMqPollRsp msg;
+  void*      vg;
+  SMqRowIter iter;
+};
+
 struct tmq_list_t {
   SArray container;
 };
@@ -99,13 +111,14 @@ typedef struct {
 
 typedef struct {
   // subscribe info
-  int32_t        sqlLen;
-  char*          sql;
-  char*          topicName;
-  int64_t        topicId;
-  int32_t        nextVgIdx;
-  SArray*        vgs;  // SArray<SMqClientVg>
-  SSchemaWrapper schema;
+  int32_t     sqlLen;
+  char*       sql;
+  char*       topicName;
+  int64_t     topicId;
+  SArray*     vgs;  // SArray<SMqClientVg>
+  int8_t      isSchemaAdaptive;
+  int32_t     numOfFields;
+  TAOS_FIELD* fields;
 } SMqClientTopic;
 
 typedef struct {
@@ -130,11 +143,11 @@ typedef struct {
 } SMqPollCbParam;
 
 typedef struct {
-  tmq_t* tmq;
-  /*SMqClientVg* pVg;*/
+  tmq_t*         tmq;
   int32_t        async;
   tsem_t         rspSem;
   tmq_resp_err_t rspErr;
+  /*SMqClientVg* pVg;*/
 } SMqCommitCbParam;
 
 tmq_conf_t* tmq_conf_new() {
@@ -471,7 +484,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
     tNameExtractFullName(&name, topicFname);
     tscDebug("subscribe topic: %s", topicFname);
     SMqClientTopic topic = {
-        .nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL};
+        .sql = NULL,
+        .sqlLen = 0,
+        .topicId = 0,
+        .topicName = topicFname,
+        .vgs = NULL,
+    };
     topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
     taosArrayPush(tmq->clientTopics, &topic);
     taosArrayPush(req.topicNames, &topicFname);
@@ -615,6 +633,7 @@ _return:
   return pRequest;
 }
 
+#if 0
 TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
@@ -700,6 +719,7 @@ _return:
 
   return pRequest;
 }
+#endif
 
 static char* formatTimestamp(char* buf, int64_t val, int precision) {
   time_t  tt;

source/client/test/tmqTest.cpp

@@ -35,7 +35,7 @@ int main(int argc, char** argv) {
 
 TEST(testCase, driverInit_Test) {
   // taosInitGlobalCfg();
-//  taos_init();
+  //  taos_init();
 }
 
 TEST(testCase, create_topic_ctb_Test) {
@@ -55,8 +55,9 @@ TEST(testCase, create_topic_ctb_Test) {
 
   taos_free_result(pRes);
 
-  char* sql = "select * from tu";
-  pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
+  // char* sql = "select * from tu";
+  // pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
+  pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from tu");
   taos_free_result(pRes);
   taos_close(pConn);
 }
@@ -69,7 +70,7 @@ TEST(testCase, create_topic_stb_Test) {
   if (taos_errno(pRes) != 0) {
     printf("error in use db, reason:%s\n", taos_errstr(pRes));
   }
-  //taos_free_result(pRes);
+  // taos_free_result(pRes);
 
   TAOS_FIELD* pFields = taos_fetch_fields(pRes);
   ASSERT_TRUE(pFields == nullptr);
@@ -79,8 +80,9 @@ TEST(testCase, create_topic_stb_Test) {
 
   taos_free_result(pRes);
 
-  char* sql = "select * from st1";
-  pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  // char* sql = "select * from st1";
+  // pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from st1");
   taos_free_result(pRes);
   taos_close(pConn);
 }

source/common/src/tdatablock.c

@@ -647,7 +647,6 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
 typedef struct SSDataBlockSortHelper {
   SArray*      orderInfo;  // SArray<SBlockOrderInfo>
   SSDataBlock* pDataBlock;
-  bool         nullFirst;
 } SSDataBlockSortHelper;
 
 int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
@@ -672,11 +671,11 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
       }
 
       if (rightNull) {
-        return pHelper->nullFirst ? 1 : -1;
+        return pOrder->nullFirst ? 1 : -1;
       }
 
       if (leftNull) {
-        return pHelper->nullFirst ? -1 : 1;
+        return pOrder->nullFirst ? -1 : 1;
       }
     }
 
@@ -907,7 +906,7 @@ static __compar_fn_t getComparFn(int32_t type, int32_t order) {
   }
 }
 
-int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) {
+int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
   ASSERT(pDataBlock != NULL && pOrderInfo != NULL);
   if (pDataBlock->info.rows <= 1) {
     return TSDB_CODE_SUCCESS;
@@ -922,7 +921,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
   for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) {
     SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
 
-    SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex);
+    SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
     if (pColInfoData->hasNull) {
       sortColumnHasNull = true;
     }
@@ -961,10 +960,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
 
   int64_t p0 = taosGetTimestampUs();
 
-  SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
+  SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
   for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) {
     struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i);
-    pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex);
+    pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
   }
 
   taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar);
@@ -1012,7 +1011,7 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex);
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->slotId);
     pInfo->pColData = pColInfo;
     sortValLengthPerRow += pColInfo->info.bytes;
   }
@@ -1106,7 +1105,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
   // Allocate the additional buffer.
   int64_t p0 = taosGetTimestampUs();
 
-  SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
+  SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
 
   uint32_t rows = pDataBlock->info.rows;
   SHelper* index = createTupleIndex_rv(rows, helper.orderInfo, pDataBlock);

source/common/src/tdataformat.c

@@ -123,7 +123,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
   return buf;
 }
 
-int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
+int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   if (pBuilder == NULL) return -1;
 
   pBuilder->tCols = 256;
@@ -140,7 +140,7 @@ void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) {
   }
 }
 
-void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) {
+void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->nCols = 0;
   pBuilder->tlen = 0;
   pBuilder->flen = 0;
@@ -168,6 +168,9 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
     colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
   }
 
+  // TODO: set sma value by user input
+  pCol->sma = 1;
+
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);

source/common/test/commonTests.cpp

@@ -167,10 +167,10 @@ TEST(testCase, Datablock_test) {
   printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
 
   SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
-  SBlockOrderInfo order = {.order = TSDB_ORDER_ASC, .colIndex = 0};
+  SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0};
   taosArrayPush(pOrderInfo, &order);
 
-  blockDataSort(b, pOrderInfo, true);
+  blockDataSort(b, pOrderInfo);
   blockDataDestroy(b);
 
   taosArrayDestroy(pOrderInfo);

source/dnode/mgmt/vnode/src/vmMsg.c

@@ -246,44 +246,45 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
 
 void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
   // Requests handled by VNODE
-  dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, vmProcessMergeMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
 
   dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID);
   dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID);

source/dnode/mnode/impl/inc/mndDef.h

@@ -731,10 +731,10 @@ typedef struct {
   int32_t  vgNum;
   SRWLatch lock;
   int8_t   status;
-  int8_t   sourceType;
-  int8_t   sinkType;
   // int32_t  sqlLen;
-  int32_t        sinkVgId;  // 0 for automatic
+  int8_t         createdBy;      // STREAM_CREATED_BY__USER or SMA
+  int32_t        fixedSinkVgId;  // 0 for shuffle
+  int64_t        smaId;          // 0 for unused
   char*          sql;
   char*          logicalPlan;
   char*          physicalPlan;

source/dnode/mnode/impl/inc/mndScheduler.h

@@ -27,7 +27,7 @@ void    mndCleanupScheduler(SMnode* pMnode);
 
 int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
 
-int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId);
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndStream.h

@@ -31,7 +31,7 @@ void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId);
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDef.c

@@ -26,8 +26,11 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
   if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
+  /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
   if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
   // TODO encode tasks
   if (pObj->tasks) {
@@ -69,8 +72,11 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
-  if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;
+  /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
   if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
   pObj->tasks = NULL;
   int32_t sz;

source/dnode/mnode/impl/src/mndScheduler.c

@@ -119,7 +119,92 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
   return pVgroup;
 }
 
-int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
+int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
+  SSdb*   pSdb = pMnode->pSdb;
+  void*   pIter = NULL;
+  SArray* tasks = taosArrayGetP(pStream->tasks, 0);
+
+  ASSERT(taosArrayGetSize(pStream->tasks) == 1);
+
+  while (1) {
+    SVgObj* pVgroup;
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
+    if (pIter == NULL) break;
+    if (pVgroup->dbUid != pStream->dbUid) {
+      sdbRelease(pSdb, pVgroup);
+      continue;
+    }
+    SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+    if (pTask == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    taosArrayPush(tasks, &pTask);
+
+    pTask->nodeId = pVgroup->vgId;
+    pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
+
+    // source
+    pTask->sourceType = TASK_SOURCE__MERGE;
+
+    // exec
+    pTask->execType = TASK_EXEC__NONE;
+
+    // sink
+    if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
+      pTask->sinkType = TASK_SINK__SMA;
+      pTask->smaSink.smaId = pStream->smaId;
+    } else {
+      pTask->sinkType = TASK_SINK__TABLE;
+    }
+
+    // dispatch
+    pTask->dispatchType = TASK_DISPATCH__NONE;
+
+    mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
+  }
+  return 0;
+}
+
+int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
+  ASSERT(pStream->fixedSinkVgId != 0);
+  SArray*      tasks = taosArrayGetP(pStream->tasks, 0);
+  SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+  if (pTask == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  taosArrayPush(tasks, &pTask);
+
+  pTask->nodeId = pStream->fixedSinkVgId;
+  SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
+  if (pVgroup == NULL) {
+    return -1;
+  }
+  pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
+  // source
+  pTask->sourceType = TASK_SOURCE__MERGE;
+
+  // exec
+  pTask->execType = TASK_EXEC__NONE;
+
+  // sink
+  if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
+    pTask->sinkType = TASK_SINK__SMA;
+    pTask->smaSink.smaId = pStream->smaId;
+  } else {
+    pTask->sinkType = TASK_SINK__TABLE;
+  }
+  //
+  // dispatch
+  pTask->dispatchType = TASK_DISPATCH__NONE;
+
+  mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
+
+  return 0;
+}
+
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   SSdb*       pSdb = pMnode->pSdb;
   SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
   if (pPlan == NULL) {
@@ -132,6 +217,19 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
   ASSERT(totLevel <= 2);
   pStream->tasks = taosArrayInit(totLevel, sizeof(void*));
 
+  bool hasExtraSink = false;
+  if (totLevel == 2) {
+    SArray* taskOneLevel = taosArrayInit(0, sizeof(void*));
+    taosArrayPush(pStream->tasks, &taskOneLevel);
+    // add extra sink
+    hasExtraSink = true;
+    if (pStream->fixedSinkVgId == 0) {
+      mndAddShuffledSinkToStream(pMnode, pTrans, pStream);
+    } else {
+      mndAddFixedSinkToStream(pMnode, pTrans, pStream);
+    }
+  }
+
   for (int32_t level = 0; level < totLevel; level++) {
     SArray*        taskOneLevel = taosArrayInit(0, sizeof(void*));
     SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level);
@@ -164,9 +262,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
           // only for inplace
           pTask->sinkType = TASK_SINK__SHOW;
           pTask->showSink.reserved = 0;
-          if (smaId != -1) {
-            pTask->sinkType = TASK_SINK__SMA;
-            pTask->smaSink.smaId = smaId;
+          if (!hasExtraSink) {
+#if 1
+            if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
+              pTask->sinkType = TASK_SINK__SMA;
+              pTask->smaSink.smaId = pStream->smaId;
+            } else {
+              pTask->sinkType = TASK_SINK__TABLE;
+            }
+#endif
           }
         } else {
           pTask->sinkType = TASK_SINK__NONE;
@@ -175,17 +279,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
         // dispatch part
         if (level == 0) {
           pTask->dispatchType = TASK_DISPATCH__NONE;
-          // if inplace sink, no dispatcher
-          // if fixed ep, add fixed ep dispatcher
-          // if shuffle, add shuffle dispatcher
         } else {
           // add fixed ep dispatcher
           int32_t lastLevel = level - 1;
           ASSERT(lastLevel == 0);
+          if (hasExtraSink) lastLevel++;
           SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel);
           // one merge only
           ASSERT(taosArrayGetSize(pArray) == 1);
-          SStreamTask* lastLevelTask = taosArrayGetP(pArray, lastLevel);
+          SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
           pTask->dispatchMsgType = TDMT_VND_TASK_MERGE_EXEC;
           pTask->dispatchType = TASK_DISPATCH__FIXED;
 
@@ -222,18 +324,55 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
       /*pTask->sinkType = TASK_SINK__NONE;*/
 
       // dispatch part
-      pTask->dispatchType = TASK_DISPATCH__NONE;
-#if 0
-      pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
-      pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
-      SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
-      ASSERT(pDb);
-      if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
-        sdbRelease(pSdb, pDb);
-        qDestroyQueryPlan(pPlan);
-        return -1;
+      ASSERT(hasExtraSink);
+      /*pTask->dispatchType = TASK_DISPATCH__NONE;*/
+#if 1
+
+      if (hasExtraSink) {
+        // add dispatcher
+        if (pStream->fixedSinkVgId == 0) {
+          pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
+
+          pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
+          SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
+          ASSERT(pDb);
+          if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
+            sdbRelease(pSdb, pDb);
+            qDestroyQueryPlan(pPlan);
+            return -1;
+          }
+          sdbRelease(pSdb, pDb);
+
+          // put taskId to useDbRsp
+          // TODO: optimize
+          SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+          int32_t sz = taosArrayGetSize(pVgs);
+          SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
+          int32_t sinkLvSize = taosArrayGetSize(sinkLv);
+          for (int32_t i = 0; i < sz; i++) {
+            SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
+            for (int32_t j = 0; j < sinkLvSize; j++) {
+              SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
+              /*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
+              if (pLastLevelTask->nodeId == pVgInfo->vgId) {
+                pVgInfo->taskId = pLastLevelTask->taskId;
+                /*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
+                break;
+              }
+            }
+          }
+        } else {
+          pTask->dispatchType = TASK_DISPATCH__FIXED;
+          pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
+          SArray* pArray = taosArrayGetP(pStream->tasks, 0);
+          // one sink only
+          ASSERT(taosArrayGetSize(pArray) == 1);
+          SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
+          pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
+          pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
+          pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
+        }
       }
-      sdbRelease(pSdb, pDb);
 #endif
 
       // exec part

source/dnode/mnode/impl/src/mndSma.c

@@ -415,6 +415,10 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   streamObj.dbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
+  streamObj.createdBy = STREAM_CREATED_BY__SMA;
+  // TODO
+  streamObj.fixedSinkVgId = 0;
+  streamObj.smaId = smaObj.uid;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
 
@@ -428,7 +432,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, smaObj.uid) != 0) goto _OVER;
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndStream.c

@@ -246,7 +246,7 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
   return code;
 }
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId) {
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
   SNode *pAst = NULL;
 
   if (nodesStringToNode(ast, &pAst) < 0) {
@@ -271,7 +271,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
     return -1;
   }
 
-  if (mndScheduleStream(pMnode, pTrans, pStream, smaId) < 0) {
+  if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
     mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
     return -1;
   }
@@ -300,6 +300,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.dbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
+  streamObj.createdBy = STREAM_CREATED_BY__USER;
+  // TODO
+  streamObj.fixedSinkVgId = 0;
+  streamObj.smaId = 0;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
 
@@ -310,7 +314,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   }
   mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
 
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, -1) != 0) {
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
     mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
     return -1;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -62,8 +62,6 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT
 
 static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
 
-static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
-
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
                      .keyType = SDB_KEY_BINARY,
@@ -98,14 +96,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
     return NULL;
   }
 
-#if 0
-  if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tDeleteSMqSubscribeObj(pSub);
-    taosMemoryFree(pSub);
-    return NULL;
-  }
-#endif
   // TODO: disable alter subscribed table
   return pSub;
 }
@@ -210,45 +200,6 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
   return 0;
 }
 
-#if 0
-static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) {
-  SMnode             *pMnode = pMsg->pNode;
-  uint8_t            *str = pMsg->rpcMsg.pCont;
-  SMqCMResetOffsetReq req;
-
-  SCoder decoder;
-  tCoderInit(&decoder, TD_LITTLE_ENDIAN, str, pMsg->rpcMsg.contLen, TD_DECODER);
-  tDecodeSMqCMResetOffsetReq(&decoder, &req);
-
-  SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-  if (pHash == NULL) {
-    return -1;
-  }
-
-  for (int32_t i = 0; i < req.num; i++) {
-    SMqOffset    *pOffset = &req.offsets[i];
-    SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t));
-    if (pVgOffset == NULL) {
-      pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets));
-      if (pVgOffset == NULL) {
-        return -1;
-      }
-      pVgOffset->offsets = taosArrayInit(0, sizeof(void *));
-      taosArrayPush(pVgOffset->offsets, &pOffset);
-    }
-    taosHashPut(pHash, &pOffset->vgId, sizeof(int32_t), &pVgOffset, sizeof(void *));
-  }
-
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
-  if (pTrans == NULL) {
-    mError("mq-reset-offset: failed since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-#endif
-
 static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   SMnode           *pMnode = pMsg->pNode;
   SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@@ -574,251 +525,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   return 0;
 }
 
-#if 0
-      for (int32_t j = 0; j < consumerNum; j++) {
-        bool            changed = false;
-        bool            unfished = false;
-
-        bool            canUseLeft = imbalanceSolved < imbalanceVg;
-        bool            mustUseLeft = canUseLeft && (imbalanceVg - imbalanceSolved >= consumerNum - j);
-        ASSERT(imbalanceVg - imbalanceSolved <= consumerNum - j);
-
-        int32_t maxVg = vgEachConsumer + canUseLeft;
-        int32_t minVg = vgEachConsumer + mustUseLeft;
-
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
-        int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
-        int32_t         vgThisConsumerAfterRb;
-        if (vgThisConsumerBeforeRb > maxVg) {
-          vgThisConsumerAfterRb = maxVg; 
-          imbalanceSolved++;
-          changed = true;
-        } else if (vgThisConsumerBeforeRb < minVg) {
-          vgThisConsumerAfterRb = minVg;
-          if (mustUseLeft) imbalanceSolved++;
-          changed = true;
-        } else {
-          vgThisConsumerAfterRb = vgThisConsumerBeforeRb;
-        }
-
-        if (vgThisConsumerBeforeRb > vgThisConsumerAfterRb) {
-          while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
-            // put into unassigned
-            SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
-            ASSERT(pConsumerEp != NULL);
-            ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
-            taosArrayPush(unassignedVgStash, pConsumerEp);
-          }
-
-        } else if (vgThisConsumerBeforeRb < vgThisConsumerAfterRb) {
-          // assign from unassigned
-          while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
-            // if no unassgined, save j
-            if (taosArrayGetSize(unassignedVgStash) == 0) {
-              taosArrayPush(unassignedConsumerIdx, &j);
-              unfished = true;
-              break;
-            }
-            // assign vg to consumer
-            SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
-            ASSERT(pConsumerEp != NULL);
-            pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
-            pConsumerEp->consumerId = pSubConsumer->consumerId;
-            taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
-            // build msg and persist into trans
-          }
-        }
-
-        if (changed && !unfished) {
-          SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
-          pRebConsumer->epoch++;
-          if (vgThisConsumerAfterRb != 0) {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
-          } else {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
-          }
-          SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
-          sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-          mndTransAppendRedolog(pTrans, pConsumerRaw);
-          mndReleaseConsumer(pMnode, pRebConsumer);
-          // TODO: save history
-        }
-      }
-
-      for (int32_t j = 0; j < taosArrayGetSize(unassignedConsumerIdx); j++) {
-        bool            canUseLeft = imbalanceSolved < imbalanceVg;
-        int32_t         consumerIdx = *(int32_t *)taosArrayGet(unassignedConsumerIdx, j);
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, consumerIdx);
-        if (canUseLeft) imbalanceSolved++;
-        // must use
-        int32_t vgThisConsumerAfterRb = taosArrayGetSize(pSubConsumer->vgInfo) + canUseLeft;
-        while (taosArrayGetSize(pSubConsumer->vgInfo) < vgEachConsumer + canUseLeft) {
-          // assign vg to consumer
-          SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
-          ASSERT(pConsumerEp != NULL);
-          pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
-          pConsumerEp->consumerId = pSubConsumer->consumerId;
-          taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
-          // build msg and persist into trans
-        }
-        SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
-        pRebConsumer->epoch++;
-        atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
-        SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
-        sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-        mndTransAppendRedolog(pTrans, pConsumerRaw);
-        mndReleaseConsumer(pMnode, pRebConsumer);
-        // TODO: save history
-      }
-#endif
-
-#if 0
-    //update consumer status for the subscribption
-    for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
-      SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
-      int64_t        consumerId = pCEp->consumerId;
-      if (pCEp->status != -1) {
-        int32_t consumerHbStatus = atomic_fetch_add_32(&pCEp->consumerHbStatus, 1);
-        if (consumerHbStatus < MND_SUBSCRIBE_REBALANCE_CNT) {
-          continue;
-        }
-        // put consumer into lostConsumer
-        SMqConsumerEp* lostConsumer = taosArrayPush(pSub->lostConsumer, pCEp);
-        lostConsumer->qmsg = NULL;
-        // put vg into unassigned
-        taosArrayPush(pSub->unassignedVg, pCEp);
-        // remove from assigned
-        // TODO: swap with last one, reduce size and reset i
-        taosArrayRemove(pSub->assigned, i);
-        // remove from available consumer
-        for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
-          if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) {
-            taosArrayRemove(pSub->availConsumer, j);
-            break;
-          }
-          // TODO: acquire consumer, set status to unavail
-        }
-#if 0
-        SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId);
-        pConsumer->epoch++;
-        printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size);
-        SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer);
-        sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-        sdbWriteNotFree(pMnode->pSdb, pRaw);
-        mndReleaseConsumer(pMnode, pConsumer);
-#endif
-      }
-    }
-    // no available consumer, skip rebalance
-    if (taosArrayGetSize(pSub->availConsumer) == 0) {
-      continue;
-    }
-    taosArrayGet(pSub->availConsumer, 0);
-    // rebalance condition1 : have unassigned vg
-    // assign vg to a consumer, trying to find the least assigned one
-    if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) {
-      char *topic = NULL;
-      char *cgroup = NULL;
-      mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
-
-      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-      STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
-      for (int32_t i = 0; i < sz; i++) {
-        int64_t        consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx);
-        pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer);
-
-        SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg);
-        pCEp->oldConsumerId = pCEp->consumerId;
-        pCEp->consumerId = consumerId;
-        taosArrayPush(pSub->assigned, pCEp);
-
-        SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
-        pConsumer->epoch++;
-        SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);
-        sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-        sdbWrite(pMnode->pSdb, pConsumerRaw);
-        mndReleaseConsumer(pMnode, pConsumer);
-
-        void* msg;
-        int32_t msgLen;
-        mndBuildRebalanceMsg(&msg, &msgLen, pTopic, pCEp, cgroup, topic);
-
-        // persist msg
-        STransAction action = {0};
-        action.epSet = pCEp->epSet;
-        action.pCont = msg;
-        action.contLen = sizeof(SMsgHead) + msgLen;
-        action.msgType = TDMT_VND_MQ_SET_CONN;
-        mndTransAppendRedoAction(pTrans, &action);
-
-        // persist data
-        SSdbRaw *pRaw = mndSubActionEncode(pSub);
-        sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-        mndTransAppendRedolog(pTrans, pRaw);
-      }
-
-      if (mndTransPrepare(pMnode, pTrans) != 0) {
-        mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-      }
-      mndReleaseTopic(pMnode, pTopic);
-      mndTransDrop(pTrans);
-      taosMemoryFreeClear(topic);
-      taosMemoryFreeClear(cgroup);
-    }
-    // rebalance condition2 : imbalance assignment
-  }
-  return 0;
-}
-#endif
-
-#if 0
-static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
-  SSdb      *pSdb = pMnode->pSdb;
-  SVgObj    *pVgroup = NULL;
-  SQueryPlan *pPlan = qStringToQueryPlan(pTopic->physicalPlan);
-  SArray    *pArray = NULL;
-  SNodeListNode    *inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
-  SSubplan  *plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
-  SArray    *unassignedVg = pSub->unassignedVg;
-
-  void *pIter = NULL;
-  while (1) {
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
-    if (pIter == NULL) break;
-    if (pVgroup->dbUid != pTopic->dbUid) {
-      sdbRelease(pSdb, pVgroup);
-      continue;
-    }
-
-    pSub->vgNum++;
-    plan->execNode.nodeId = pVgroup->vgId;
-    plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
-
-    if (schedulerConvertDagToTaskList(pPlan, &pArray) < 0) {
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
-      mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
-      return -1;
-    }
-
-    SMqConsumerEp consumerEp = {0};
-    consumerEp.status = 0;
-    consumerEp.consumerId = -1;
-    STaskInfo *pTaskInfo = taosArrayGet(pArray, 0);
-    consumerEp.epSet = pTaskInfo->addr.epset;
-    consumerEp.vgId = pTaskInfo->addr.nodeId;
-
-    ASSERT(consumerEp.vgId == pVgroup->vgId);
-    consumerEp.qmsg = strdup(pTaskInfo->msg->msg);
-    taosArrayPush(unassignedVg, &consumerEp);
-    // TODO: free taskInfo
-    taosArrayDestroy(pArray);
-  }
-
-  /*qDestroyQueryDag(pDag);*/
-  return 0;
-}
-#endif
-
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
   ASSERT(pConsumerEp->oldConsumerId == -1);
@@ -1140,7 +846,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
     }
   }
 
-  if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree);
+  if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);
 
   // persist consumerObj
   SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);

source/dnode/mnode/impl/src/mndTopic.c

@@ -65,9 +65,11 @@ void mndCleanupTopic(SMnode *pMnode) {}
 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t  logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
-  int32_t  physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
-  int32_t  size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE;
+  int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
+  int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
+  int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
+  int32_t size =
+      sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + swLen + MND_TOPIC_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
   if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
 
@@ -86,8 +88,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
 
-  int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
-  void   *swBuf = taosMemoryMalloc(swLen);
+  void *swBuf = taosMemoryMalloc(swLen);
   if (swBuf == NULL) {
     goto TOPIC_ENCODE_OVER;
   }

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -67,12 +67,13 @@ typedef struct {
   uint8_t  last : 1;
   uint8_t  blkVer : 7;
   uint8_t  numOfSubBlocks;
-  int16_t  numOfCols;    // not including timestamp column
+  col_id_t numOfCols;    // not including timestamp column
   uint32_t len;          // data block length
-  uint32_t keyLen : 24;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  uint32_t keyLen : 20;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  uint32_t algorithm : 4;
   uint32_t reserve : 8;
-  int32_t  algorithm : 8;
-  int32_t  numOfRows : 24;
+  col_id_t numOfBSma;
+  uint16_t numOfRows;
   int64_t  offset;
   uint64_t aggrStat : 1;
   uint64_t aggrOffset : 63;
@@ -80,7 +81,7 @@ typedef struct {
   TSKEY    keyLast;
 } SBlockV0;
 
-#define SBlock SBlockV0  // latest SBlock definition
+#define SBlock          SBlockV0  // latest SBlock definition
 
 #endif
 

source/dnode/vnode/src/tq/tq.c

@@ -356,6 +356,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
+    ASSERT(0);
     return -1;
   }
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
@@ -508,7 +509,9 @@ int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTaskExecReq req;
   tDecodeSStreamTaskExecReq(msg, &req);
 
-  int32_t      taskId = req.taskId;
+  int32_t taskId = req.taskId;
+  ASSERT(taskId);
+
   SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
   ASSERT(pTask);
 

source/dnode/vnode/src/tq/tqRead.c

@@ -114,10 +114,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
 
   int32_t numOfRows = pHandle->pBlock->numOfRows;
-  int32_t numOfCols = pHandle->pSchema->numOfCols;
+  /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
-  // TODO: stable case
   if (colNumNeed > pSchemaWrapper->nCols) {
     colNumNeed = pSchemaWrapper->nCols;
   }
@@ -138,56 +137,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       colNeed++;
     } else {
       SColumnInfoData colInfo = {0};
-      int             sz = numOfRows * pColSchema->bytes;
+      /*int             sz = numOfRows * pColSchema->bytes;*/
       colInfo.info.bytes = pColSchema->bytes;
       colInfo.info.colId = pColSchema->colId;
       colInfo.info.type = pColSchema->type;
 
-      colInfo.pData = taosMemoryCalloc(1, sz);
-      if (colInfo.pData == NULL) {
-        // TODO free
-        taosArrayDestroy(pArray);
+      if (blockDataEnsureColumnCapacity(&colInfo, numOfRows) < 0) {
+        taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
         return NULL;
       }
-
-      blockDataEnsureColumnCapacity(&colInfo, numOfRows);
       taosArrayPush(pArray, &colInfo);
       colMeta++;
       colNeed++;
     }
   }
 
-  int j = 0;
-  for (int32_t i = 0; i < colNumNeed; i++) {
-    col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i);
-    while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
-      j++;
-    }
-    if (j >= pSchemaWrapper->nCols) {
-      continue;
-    }
-    SSchema*        pColSchema = &pSchemaWrapper->pSchema[j];
-    SColumnInfoData colInfo = {0};
-    int             sz = numOfRows * pColSchema->bytes;
-    colInfo.info.bytes = pColSchema->bytes;
-    colInfo.info.colId = colId;
-    colInfo.info.type = pColSchema->type;
-
-    colInfo.pData = taosMemoryCalloc(1, sz);
-    if (colInfo.pData == NULL) {
-      // TODO free
-      taosArrayDestroy(pArray);
-      return NULL;
-    }
-
-    blockDataEnsureColumnCapacity(&colInfo, numOfRows);
-    taosArrayPush(pArray, &colInfo);
-  }
-
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
-  // int32_t kvIdx = 0;
   int32_t curRow = 0;
   tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
@@ -200,25 +167,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes);
-    }
-#if 0
-    for (int32_t i = 0; i < colNumNeed; i++) {
-      SColumnInfoData* pColData = taosArrayGet(pArray, i);
-      STColumn*        pCol = schemaColAt(pTschema, i);
-      // TODO
-      if(pCol->colId != pColData->info.colId) {
-        continue;
-      }
-      // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
-      SCellVal sVal = {0};
-      if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
-        // TODO: reach end
-        break;
-      }
-      memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
+      // TODO handle null
+      colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL);
     }
-#endif
     curRow++;
   }
   return pArray;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -43,20 +43,20 @@ typedef struct {
 
 #define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5)
 
-#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh))
-#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh)))
-#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet))
-#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable)
-#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD)
-#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA)
-#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST)
-#define TSDB_COMMIT_SMAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD)
-#define TSDB_COMMIT_SMAL_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL)
-#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh))
-#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh))
-#define TSDB_COMMIT_EXBUF(ch) TSDB_READ_EXBUF(&((ch)->readh))
+#define TSDB_COMMIT_REPO(ch)         TSDB_READ_REPO(&(ch->readh))
+#define TSDB_COMMIT_REPO_ID(ch)      REPO_ID(TSDB_READ_REPO(&(ch->readh)))
+#define TSDB_COMMIT_WRITE_FSET(ch)   (&((ch)->wSet))
+#define TSDB_COMMIT_TABLE(ch)        ((ch)->pTable)
+#define TSDB_COMMIT_HEAD_FILE(ch)    TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD)
+#define TSDB_COMMIT_DATA_FILE(ch)    TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA)
+#define TSDB_COMMIT_LAST_FILE(ch)    TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST)
+#define TSDB_COMMIT_SMAD_FILE(ch)    TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAD)
+#define TSDB_COMMIT_SMAL_FILE(ch)    TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_SMAL)
+#define TSDB_COMMIT_BUF(ch)          TSDB_READ_BUF(&((ch)->readh))
+#define TSDB_COMMIT_COMP_BUF(ch)     TSDB_READ_COMP_BUF(&((ch)->readh))
+#define TSDB_COMMIT_EXBUF(ch)        TSDB_READ_EXBUF(&((ch)->readh))
 #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock)
-#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
+#define TSDB_COMMIT_TXN_VERSION(ch)  FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
 
 static void tsdbStartCommit(STsdb *pRepo);
 static void tsdbEndCommit(STsdb *pTsdb, int eno);
@@ -1204,9 +1204,10 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
 
 int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
                        SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
-  STsdbCfg *    pCfg = REPO_CFG(pRepo);
-  SBlockData *  pBlockData = NULL;
+  STsdbCfg     *pCfg = REPO_CFG(pRepo);
+  SBlockData   *pBlockData = NULL;
   SAggrBlkData *pAggrBlkData = NULL;
+  STSchema     *pSchema = pTable->pSchema;
   int64_t       offset = 0, offsetAggr = 0;
   int           rowsToWrite = pDataCols->numOfRows;
 
@@ -1225,10 +1226,12 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   pAggrBlkData = (SAggrBlkData *)(*ppExBuf);
 
   // Get # of cols not all NULL(not including key column)
-  int nColsNotAllNull = 0;
+  col_id_t nColsNotAllNull = 0;
+  col_id_t nColsOfBlockSma = 0;
   for (int ncol = 1; ncol < pDataCols->numOfCols; ++ncol) {  // ncol from 1, we skip the timestamp column
-    SDataCol *   pDataCol = pDataCols->cols + ncol;
-    SBlockCol *  pBlockCol = pBlockData->cols + nColsNotAllNull;
+    STColumn    *pColumn = pSchema->columns + ncol;
+    SDataCol    *pDataCol = pDataCols->cols + ncol;
+    SBlockCol   *pBlockCol = pBlockData->cols + nColsNotAllNull;
     SAggrBlkCol *pAggrBlkCol = (SAggrBlkCol *)pAggrBlkData + nColsNotAllNull;
 
     if (isAllRowsNull(pDataCol)) {  // all data to commit are NULL, just ignore it
@@ -1260,7 +1263,12 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     } else {
       TD_SET_COL_ROWS_MISC(pBlockCol);
     }
-    nColsNotAllNull++;
+
+    ++nColsNotAllNull;
+
+    if (pColumn->sma) {
+      ++nColsOfBlockSma;
+    }
   }
 
   ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
@@ -1357,9 +1365,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     return -1;
   }
 
-  uint32_t aggrStatus = nColsNotAllNull > 0 ? 1 : 0;
+  uint32_t aggrStatus = nColsOfBlockSma > 0 ? 1 : 0;
   if (aggrStatus > 0) {
-
     taosCalcChecksumAppend(0, (uint8_t *)pAggrBlkData, tsizeAggr);
     tsdbUpdateDFileMagic(pDFileAggr, POINTER_SHIFT(pAggrBlkData, tsizeAggr - sizeof(TSCKSUM)));
 
@@ -1378,6 +1385,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   pBlock->keyLen = keyLen;
   pBlock->numOfSubBlocks = isSuper ? 1 : 0;
   pBlock->numOfCols = nColsNotAllNull;
+  pBlock->numOfBSma = nColsOfBlockSma;
   pBlock->keyFirst = dataColsKeyFirst(pDataCols);
   pBlock->keyLast = dataColsKeyLast(pDataCols);
   pBlock->aggrStat = aggrStatus;

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -321,7 +321,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
     return -1;
   }
 
-  size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
+  size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfBSma, (uint32_t)pBlock->blkVer);
   if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1;
 
   int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -189,12 +189,16 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       }
     } break;
     case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
-#if 0
+#if 1
+      
       SSmaCfg vCreateSmaReq = {0};
       if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
+        vWarn("vgId%d: TDMT_VND_CREATE_SMA received but deserialize failed since %s", pVnode->config.vgId, terrstr(terrno));
         return -1;
       }
+      vWarn("vgId%d: TDMT_VND_CREATE_SMA received for %s:%" PRIi64, pVnode->config.vgId, vCreateSmaReq.tSma.indexName,
+            vCreateSmaReq.tSma.indexUid);
 
       // record current timezone of server side
       tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN);

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -330,7 +330,6 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
 
   // step 2: insert data
-  STSmaDataWrapper *pSmaData = NULL;
   STsdb            *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb));
   STsdbCfg         *pCfg = &pTsdb->config;
 
@@ -416,6 +415,8 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   col_id_t numOfCols = 4096;
   ASSERT_GT(numOfCols, 0);
 
+#if 0
+  STSmaDataWrapper *pSmaData = NULL;
   pSmaData = (STSmaDataWrapper *)buf;
   printf(">> allocate [%d] time to %d and addr is %p\n", ++allocCnt, bufSize, pSmaData);
   pSmaData->skey = skey1;
@@ -459,9 +460,13 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   pSmaData->dataLen = (len - sizeof(STSmaDataWrapper));
 
   ASSERT_GE(bufSize, pSmaData->dataLen);
-
   // execute
   ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
+#endif
+
+  SSDataBlock *pSmaData = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
+  
+
 
   // step 3: query
   uint32_t checkDataCnt = 0;

source/libs/executor/inc/executorimpl.h

@@ -430,9 +430,10 @@ typedef struct STagScanInfo {
 } STagScanInfo;
 
 typedef struct SStreamBlockScanInfo {
+  SArray*      pBlockLists;   // multiple SSDatablock.
   SSDataBlock* pRes;          // result SSDataBlock
   int32_t      blockType;     // current block type
-  bool         blockValid;    // Is current data has returned?
+  int32_t      validBlockIndex;    // Is current data has returned?
   SColumnInfo* pCols;         // the output column info
   uint64_t     numOfRows;     // total scanned rows
   uint64_t     numOfExec;     // execution times
@@ -572,11 +573,13 @@ typedef struct SGroupbyOperatorInfo {
 typedef struct SSessionAggOperatorInfo {
   SOptrBasicInfo binfo;
   SAggSupporter  aggSup;
+  SGroupResInfo  groupResInfo;
   STimeWindow    curWindow;  // current time window
   TSKEY          prevTs;     // previous timestamp
   int32_t        numOfRows;  // number of rows
   int32_t        start;      // start row index
   bool           reptScan;   // next round scan
+  int64_t        gap;        // session window gap
 } SSessionAggOperatorInfo;
 
 typedef struct SStateWindowOperatorInfo {
@@ -593,8 +596,7 @@ typedef struct SSortedMergeOperatorInfo {
   SOptrBasicInfo     binfo;
   bool               hasVarCol;
   
-  SArray            *orderInfo;   // SArray<SBlockOrderInfo>
-  bool               nullFirst;
+  SArray*            pSortInfo;
   int32_t            numOfSources;
 
   SSortHandle       *pSortHandle;
@@ -613,12 +615,10 @@ typedef struct SSortedMergeOperatorInfo {
   SAggSupporter      aggSup;
 } SSortedMergeOperatorInfo;
 
-typedef struct SOrderOperatorInfo {
+typedef struct SSortOperatorInfo {
   uint32_t           sortBufSize;  // max buffer size for in-memory sort
   SSDataBlock       *pDataBlock;
-  bool               hasVarCol;    // has variable length column, such as binary/varchar/nchar
-  SArray            *orderInfo;
-  bool               nullFirst;
+  SArray*            pSortInfo;
   SSortHandle       *pSortHandle;
   int32_t            bufPageSize;
   int32_t            numOfRowsInRes;
@@ -629,7 +629,7 @@ typedef struct SOrderOperatorInfo {
   uint64_t           totalSize;     // total load bytes from remote
   uint64_t           totalRows;     // total number of rows
   uint64_t           totalElapsed;  // total elapsed time
-} SOrderOperatorInfo;
+} SSortOperatorInfo;
 
 typedef struct SDistinctDataInfo {
   int32_t index;
@@ -655,15 +655,15 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
                                            SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
+SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName,
                                               SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId);
 SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
                                           const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
                                        SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock,

source/libs/executor/inc/tsort.h

@@ -63,7 +63,7 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void*
  * @param type
  * @return
  */
-SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr);
+SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr);
 
 /**
  *

source/libs/executor/src/executil.c

@@ -19,8 +19,6 @@
 
 #include "executil.h"
 #include "executorimpl.h"
-//#include "queryLog.h"
-#include "tbuffer.h"
 #include "tcompression.h"
 #include "tlosertree.h"
 

source/libs/executor/src/executor.c

@@ -16,9 +16,10 @@
 #include "executor.h"
 #include "executorimpl.h"
 #include "planner.h"
+#include "tdatablock.h"
 #include "vnode.h"
 
-static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t type, char* id) {
+static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
   ASSERT(pOperator != NULL);
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     if (pOperator->numOfDownstream == 0) {
@@ -31,7 +32,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
       return TSDB_CODE_QRY_APP_ERROR;
     }
     pOperator->status = OP_NOT_OPENED;
-    return doSetStreamBlock(pOperator->pDownstream[0], input, type, id);
+    return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id);
   } else {
     SStreamBlockScanInfo* pInfo = pOperator->info;
 
@@ -48,15 +49,16 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
         return TSDB_CODE_QRY_APP_ERROR;
       }
     } else {
-      ASSERT(!pInfo->blockValid);
+      for (int32_t i = 0; i < numOfBlocks; ++i) {
+        SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
 
-      SSDataBlock* pDataBlock = input;
-      pInfo->pRes->info = pDataBlock->info;
-      taosArrayClear(pInfo->pRes->pDataBlock);
-      taosArrayAddAll(pInfo->pRes->pDataBlock, pDataBlock->pDataBlock);
+        SSDataBlock* p = createOneDataBlock(pDataBlock);
+        p->info = pDataBlock->info;
 
-      // set current block valid.
-      pInfo->blockValid = true;
+        taosArrayClear(p->pDataBlock);
+        taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
+        taosArrayPush(pInfo->pBlockLists, &p);
+      }
     }
 
     return TSDB_CODE_SUCCESS;
@@ -64,17 +66,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
 }
 
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) {
+  return qSetMultiStreamInput(tinfo, input, 1, type);
+}
+
+int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
   if (tinfo == NULL) {
     return TSDB_CODE_QRY_APP_ERROR;
   }
 
-  if (input == NULL) {
+  if (pBlocks == NULL || numOfBlocks == 0) {
     return TSDB_CODE_SUCCESS;
   }
 
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
 
-  int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*)input, type, GET_TASKID(pTaskInfo));
+  int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo));
   } else {

source/libs/executor/src/tsort.c

@@ -23,20 +23,19 @@
 #include "tsort.h"
 #include "tutil.h"
 
-typedef struct STupleHandle {
+struct STupleHandle {
   SSDataBlock* pBlock;
   int32_t      rowIndex;
-} STupleHandle;
+};
 
-typedef struct SSortHandle {
+struct SSortHandle {
   int32_t           type;
 
   int32_t           pageSize;
   int32_t           numOfPages;
   SDiskbasedBuf    *pBuf;
 
-  SArray           *pOrderInfo;
-  bool              nullFirst;
+  SArray           *pSortInfo;
   SArray           *pOrderedSource;
 
   _sort_fetch_block_fn_t  fetchfp;
@@ -60,7 +59,7 @@ typedef struct SSortHandle {
   bool              inMemSort;
   bool              needAdjust;
   STupleHandle      tupleHandle;
-} SSortHandle;
+};
 
 static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param);
 
@@ -90,18 +89,18 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) {
  * @param type
  * @return
  */
-SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) {
+SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr) {
   SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle));
 
   pSortHandle->type       = type;
   pSortHandle->pageSize   = pageSize;
   pSortHandle->numOfPages = numOfPages;
-  pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
-  pSortHandle->pOrderInfo = pOrderInfo;
-  pSortHandle->nullFirst  = nullFirst;
-  pSortHandle->cmpParam.orderInfo = pOrderInfo;
+  pSortHandle->pSortInfo  = pSortInfo;
+  pSortHandle->pDataBlock = createOneDataBlock(pBlock);
+
+  pSortHandle->pOrderedSource     = taosArrayInit(4, POINTER_BYTES);
+  pSortHandle->cmpParam.orderInfo = pSortInfo;
 
-  pSortHandle->pDataBlock = createDataBlock_rv(pSchema, numOfCols);
   tsortSetComparFp(pSortHandle, msortComparFn);
 
   if (idstr != NULL) {
@@ -364,14 +363,14 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
   for(int32_t i = 0; i < pInfo->size; ++i) {
     SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
 
-    SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex);
+    SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
 
     bool leftNull  = false;
     if (pLeftColInfoData->hasNull) {
       leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg);
     }
 
-    SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex);
+    SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
     bool rightNull = false;
     if (pRightColInfoData->hasNull) {
       rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg);
@@ -415,6 +414,9 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
 
 static int32_t doInternalMergeSort(SSortHandle* pHandle) {
   size_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource);
+  if (numOfSources == 0) {
+    return 0;
+  }
 
   // Calculate the I/O counts to complete the data sort.
   double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages));
@@ -542,7 +544,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
       if (size > sortBufSize) {
         // Perform the in-memory sort and then flush data in the buffer into disk.
         int64_t p = taosGetTimestampUs();
-        blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst);
+        blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
 
         int64_t el = taosGetTimestampUs() - p;
         pHandle->sortElapsed += el;
@@ -555,7 +557,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
       size_t size = blockDataGetSize(pHandle->pDataBlock);
 
       // Perform the in-memory sort and then flush data in the buffer into disk.
-      blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst);
+      blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
 
       // All sorted data can fit in memory, external memory sort is not needed. Return to directly
       if (size <= sortBufSize) {
@@ -603,6 +605,10 @@ int32_t tsortOpen(SSortHandle* pHandle) {
     ASSERT(numOfSources <= getNumOfInMemBufPages(pHandle->pBuf));
   }
 
+  if (numOfSources == 0) {
+    return 0;
+  }
+
   code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, 0, numOfSources - 1, pHandle);
   if (code != TSDB_CODE_SUCCESS) {
     return code;

source/libs/executor/test/executorTests.cpp

@@ -966,7 +966,7 @@ TEST(testCase, inMem_sort_Test) {
   exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
   taosArrayPush(pExprInfo, &exp1);
 
-  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
+  SOperatorInfo* pOperator = createSortOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
 
   bool newgroup = false;
   SSDataBlock* pRes = pOperator->getNextFn(pOperator, &newgroup);
@@ -1035,7 +1035,7 @@ TEST(testCase, external_sort_Test) {
   exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
 //  taosArrayPush(pExprInfo, &exp1);
 
-  SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
+  SOperatorInfo* pOperator = createSortOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
 
   bool newgroup = false;
   SSDataBlock* pRes = NULL;

source/libs/executor/test/sortTests.cpp

@@ -98,14 +98,14 @@ int32_t docomp(const void* p1, const void* p2, void* param) {
   for(int32_t i = 0; i < pInfo->size; ++i) {
     SBlockOrderInfo* pOrder = (SBlockOrderInfo*)TARRAY_GET_ELEM(pInfo, i);
 
-    SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex);
+    SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
 
     bool leftNull  = false;
     if (pLeftColInfoData->hasNull) {
       leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg);
     }
 
-    SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex);
+    SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
     bool rightNull = false;
     if (pRightColInfoData->hasNull) {
       rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg);

source/libs/function/inc/builtins.h

@@ -33,6 +33,8 @@ extern "C" {
 #define FUNC_MGT_DATETIME_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
 #define FUNC_MGT_TIMELINE_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
 #define FUNC_MGT_TIMEORDER_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
+#define FUNC_MGT_PSEUDO_COLUMN_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
+#define FUNC_MGT_WINDOW_PC_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/src/builtins.c

@@ -291,6 +291,76 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = NULL,
     .sprocessFunc = NULL,
     .finalizeFunc = NULL
+  },
+  {
+    .name = "_rowts",
+    .type = FUNCTION_TYPE_ROWTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "tbname",
+    .type = FUNCTION_TYPE_TBNAME,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_qstartts",
+    .type = FUNCTION_TYPE_QSTARTTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_qendts",
+    .type = FUNCTION_TYPE_QENDTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wstartts",
+    .type = FUNCTION_TYPE_QSTARTTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wendts",
+    .type = FUNCTION_TYPE_QENDTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wduration",
+    .type = FUNCTION_TYPE_WDURATION,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
   }
 };
 
@@ -329,6 +399,13 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       break;
     }
     case FUNCTION_TYPE_CONCAT:
+    case FUNCTION_TYPE_ROWTS:
+    case FUNCTION_TYPE_TBNAME:
+    case FUNCTION_TYPE_QSTARTTS:
+    case FUNCTION_TYPE_QENDTS:
+    case FUNCTION_TYPE_WSTARTTS:
+    case FUNCTION_TYPE_WENDTS:
+    case FUNCTION_TYPE_WDURATION:
       // todo
       break;
 

source/libs/function/src/functionMgt.c

@@ -44,6 +44,13 @@ static void doInitFunctionHashTable() {
   }
 }
 
+static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
+  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+    return false;
+  }
+  return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
+}
+
 int32_t fmFuncMgtInit() {
   taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable);
   return initFunctionCode;
@@ -89,10 +96,19 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
 }
 
 bool fmIsAggFunc(int32_t funcId) {
-  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
-    return false;
-  }
-  return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, FUNC_MGT_AGG_FUNC);
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC);
+}
+
+bool fmIsScalarFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC);
+}
+
+bool fmIsWindowPseudoColumnFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC);
+}
+
+bool fmIsWindowClauseFunc(int32_t funcId) {
+  return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId);
 }
 
 void fmFuncMgtDestroy() {

source/libs/index/inc/index_fst.h

@@ -21,9 +21,9 @@ extern "C" {
 #endif
 
 #include "indexInt.h"
-#include "index_fst_node.h"
 #include "index_fst_automation.h"
 #include "index_fst_counting_writer.h"
+#include "index_fst_node.h"
 #include "index_fst_registry.h"
 #include "index_fst_util.h"
 
@@ -257,9 +257,9 @@ typedef struct FstMeta {
 } FstMeta;
 
 typedef struct Fst {
-  FstMeta*        meta;
-  FstSlice*       data;  //
-  FstNode*        root;  //
+  FstMeta*      meta;
+  FstSlice*     data;  //
+  FstNode*      root;  //
   TdThreadMutex mtx;
 } Fst;
 
@@ -325,10 +325,10 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
 FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
 
 void fstStreamBuilderDestroy(FstStreamBuilder* b);
-// set up bound range
-// refator, simple code by marco
 
-FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
+// set up bound range
+// refator later:  to simple code by marco
+void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
 
 #ifdef __cplusplus
 }

source/libs/index/src/index_fst.c

@@ -1184,7 +1184,7 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
   sws->aut = automation;
   sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t));
 
-  sws->emptyOutput.null = false;
+  sws->emptyOutput.null = true;
   sws->emptyOutput.out = 0;
 
   sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
@@ -1239,8 +1239,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
   for (uint32_t i = 0; i < len; i++) {
     uint8_t  b = data[i];
     uint64_t res = 0;
-    bool     null = fstNodeFindInput(node, b, &res);
-    if (null == false) {
+    bool     find = fstNodeFindInput(node, b, &res);
+    if (find == true) {
       FstTransition trn;
       fstNodeGetTransitionAt(node, res, &trn);
       void* preState = autState;
@@ -1293,7 +1293,6 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
 
   return false;
 }
-
 StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
   AutomationCtx* aut = sws->aut;
   FstOutput      output = sws->emptyOutput;
@@ -1356,6 +1355,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
+      taosArrayDestroy(nodes);
       return NULL;
     }
     if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
@@ -1364,15 +1364,12 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
       taosArrayDestroy(nodes);
+      nodes = NULL;
       return result;
     }
     taosMemoryFreeClear(buf);
     fstSliceDestroy(&slice);
-  }
-  for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
-    FstNode** node = (FstNode**)taosArrayGet(nodes, i);
-    fstNodeDestroy(*node);
-  }
+  };
   taosArrayDestroy(nodes);
   return NULL;
 }
@@ -1425,9 +1422,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
   taosMemoryFreeClear(b->max);
   taosMemoryFree(b);
 }
-FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
+void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
   if (b == NULL) {
-    return NULL;
+    return;
   }
   if (type == GE) {
     b->min->type = Included;
@@ -1446,5 +1443,4 @@ FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, Rang
     fstSliceDestroy(&(b->max->data));
     b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
   }
-  return b;
 }

source/libs/index/src/index_fst_automation.c

@@ -85,10 +85,20 @@ static void* prefixStart(AutomationCtx* ctx) {
 };
 static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
-  return ssv->val == strlen(ctx->data);
+  if (ssv == NULL) {
+    return false;
+  }
+  if (ssv->type == FST_INT) {
+    return ssv->val == strlen(ctx->data);
+  } else {
+    return false;
+  }
 }
 static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
+  if (ssv == NULL) {
+    return false;
+  }
   return ssv->val >= 0;
 }
 static bool  prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
@@ -154,15 +164,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
     // add more search type
   }
 
-  char* dst = NULL;
-  if (data != NULL) {
-    char*  src = (char*)data;
-    size_t len = strlen(src);
-    dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1);
-    memcpy(dst, src, len);
-  }
-
-  ctx->data = dst;
+  ctx->data = (data != NULL ? strdup((char*)data) : NULL);
   ctx->type = atype;
   ctx->stdata = (void*)sv;
   return ctx;

source/libs/index/test/fstTest.cc

@@ -96,11 +96,41 @@ class FstReadMemory {
       char*       ch = (char*)fstSliceData(s, &sz);
       std::string key(ch, sz);
       printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
+      result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    for (size_t i = 0; i < result.size(); i++) {
+    streamWithStateDestroy(st);
+    fstStreamBuilderDestroy(sb);
+    return true;
+  }
+  bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
+                   RangeType highType, std::vector<uint64_t>& result) {
+    FstStreamBuilder* sb = fstSearch(_fst, ctx);
+
+    FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
+    FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());
+
+    // range [low, high);
+    fstStreamBuilderSetRange(sb, &l, lowType);
+    fstStreamBuilderSetRange(sb, &h, highType);
+
+    fstSliceDestroy(&l);
+    fstSliceDestroy(&h);
+
+    StreamWithState*       st = streamBuilderIntoStream(sb);
+    StreamWithStateResult* rt = NULL;
+    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+      // result.push_back((uint64_t)(rt->out.out));
+      FstSlice*   s = &rt->data;
+      int32_t     sz = 0;
+      char*       ch = (char*)fstSliceData(s, &sz);
+      std::string key(ch, sz);
+      printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
+      result.push_back(rt->out.out);
+      swsResultDestroy(rt);
     }
-    std::cout << std::endl;
+    streamWithStateDestroy(st);
+    fstStreamBuilderDestroy(sb);
     return true;
   }
   bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
@@ -233,7 +263,7 @@ void checkFstLongTerm() {
   // taosMemoryFree(ctx);
   // delete m;
 }
-void checkFstCheckIterator() {
+void checkFstCheckIterator1() {
   FstWriter* fw = new FstWriter;
   int64_t    s = taosGetTimestampUs();
   int        count = 2;
@@ -243,8 +273,7 @@ void checkFstCheckIterator() {
   std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
 
   fw->Put("Hello world", 1);
-  fw->Put("hello world", 2);
-  fw->Put("hello worle", 3);
+  fw->Put("Hello worle", 2);
   fw->Put("hello worlf", 4);
   delete fw;
 
@@ -258,15 +287,227 @@ void checkFstCheckIterator() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"H", AUTOMATION_PREFIX);
+  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
   for (int i = 0; i < result.size(); i++) {
     // assert(result[i] == i);  // check result
   }
+  automCtxDestroy(ctx);
+
+  delete m;
+}
+void checkFstCheckIterator2() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("a", 1);
+  fw->Put("b", 2);
+  fw->Put("c", 4);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+
+  // prefix search
+  std::vector<uint64_t> result;
+
+  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  m->Search(ctx, result);
+  std::cout << "size: " << result.size() << std::endl;
+  // assert(result.size() == count);
+  for (int i = 0; i < result.size(); i++) {
+    // assert(result[i] == i);  // check result
+  }
+  automCtxDestroy(ctx);
+
+  delete m;
+}
+void checkFstCheckIteratorPrefix() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("Hello world", 1);
+  fw->Put("Hello worle", 2);
+  fw->Put("hello worlf", 4);
+  fw->Put("ja", 4);
+  fw->Put("jb", 4);
+  fw->Put("jc", 4);
+  fw->Put("jddddddddd", 4);
+  fw->Put("jefffffff", 4);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 1);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 2);
+    automCtxDestroy(ctx);
+  }
+  {
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 1);
+    automCtxDestroy(ctx);
+  }
+  delete m;
+}
+void checkFstCheckIteratorRange1() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("a", 1);
+  fw->Put("b", 2);
+  fw->Put("c", 3);
+  fw->Put("d", 4);
+  fw->Put("e", 5);
+  fw->Put("f", 5);
+  fw->Put("G", 5);
+  delete fw;
 
-  taosMemoryFree(ctx);
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "e", LT, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "e", LT, result);
+    assert(result.size() == 2);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "e", LE, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "e", LE, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  delete m;
+}
+void checkFstCheckIteratorRange2() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("ab", 1);
+  fw->Put("b", 2);
+  fw->Put("cdd", 3);
+  fw->Put("cde", 3);
+  fw->Put("ddd", 4);
+  fw->Put("ed", 5);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "ed", LT, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "ed", LE, result);
+    assert(result.size() == 5);
+    automCtxDestroy(ctx);
+    // taosMemoryFree(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "ed", LE, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "ed", LT, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
   delete m;
 }
 
@@ -332,7 +573,11 @@ int main(int argc, char* argv[]) {
   // path suid colName ver
   // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
   //}
-  checkFstCheckIterator();
+  checkFstCheckIterator1();
+  checkFstCheckIterator2();
+  checkFstCheckIteratorPrefix();
+  checkFstCheckIteratorRange1();
+  checkFstCheckIteratorRange2();
   // checkFstLongTerm();
   // checkFstPrefixSearch();
 

source/libs/index/test/fstUT.cc

@@ -13,9 +13,9 @@
 #include "index_fst_util.h"
 #include "index_tfile.h"
 #include "tglobal.h"
+#include "tlog.h"
 #include "tskiplist.h"
 #include "tutil.h"
-#include "tlog.h"
 
 static std::string dir = "/tmp/index";
 

source/libs/nodes/src/nodesCloneFuncs.c

@@ -260,9 +260,9 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
 }
 
 static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pProjections);
-  COPY_CHAR_ARRAY_FIELD(stmtName);
   return (SNode*)pDst;
 }
 
@@ -307,10 +307,8 @@ static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
 }
 
 static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
-  COPY_SCALAR_FIELD(dataBlockId);
+  COPY_ALL_SCALAR_FIELDS;
   CLONE_NODE_LIST_FIELD(pSlots);
-  COPY_SCALAR_FIELD(resultRowSize);
-  COPY_SCALAR_FIELD(precision);
   return (SNode*)pDst;
 }
 

source/libs/nodes/src/nodesCodeFuncs.c

@@ -475,6 +475,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectLogicPlanProjections = "Projections";
+static const char* jkProjectLogicPlanLimit = "Limit";
+static const char* jkProjectLogicPlanOffset = "Offset";
+static const char* jkProjectLogicPlanSlimit = "SLimit";
+static const char* jkProjectLogicPlanSoffset = "SOffset";
 
 static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
@@ -483,6 +487,18 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanLimit, pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanOffset, pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset);
+  }
 
   return code;
 }
@@ -494,6 +510,18 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanLimit, &pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanOffset, &pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset);
+  }
 
   return code;
 }
@@ -802,6 +830,10 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectPhysiPlanProjections = "Projections";
+static const char* jkProjectPhysiPlanLimit = "Limit";
+static const char* jkProjectPhysiPlanOffset = "Offset";
+static const char* jkProjectPhysiPlanSlimit = "SLimit";
+static const char* jkProjectPhysiPlanSoffset = "SOffset";
 
 static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@@ -810,6 +842,18 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanLimit, pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanOffset, pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSlimit, pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSoffset, pNode->soffset);
+  }
 
   return code;
 }
@@ -821,6 +865,18 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanLimit, &pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanOffset, &pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSlimit, &pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSoffset, &pNode->soffset);
+  }
 
   return code;
 }
@@ -2057,19 +2113,26 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
 
 static const char* jkDataBlockDescDataBlockId = "DataBlockId";
 static const char* jkDataBlockDescSlots = "Slots";
-static const char* jkDataBlockResultRowSize = "ResultRowSize";
+static const char* jkDataBlockTotalRowSize = "TotalRowSize";
+static const char* jkDataBlockOutputRowSize = "OutputRowSize";
+static const char* jkDataBlockPrecision = "Precision";
 
 static int32_t dataBlockDescNodeToJson(const void* pObj, SJson* pJson) {
   const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj;
 
   int32_t code = tjsonAddIntegerToObject(pJson, jkDataBlockDescDataBlockId, pNode->dataBlockId);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkDataBlockResultRowSize, pNode->resultRowSize);
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockTotalRowSize, pNode->totalRowSize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockOutputRowSize, pNode->outputRowSize);
   }
-
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkDataBlockDescSlots, pNode->pSlots);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockPrecision, pNode->precision);
+  }
 
   return code;
 }
@@ -2079,12 +2142,17 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) {
 
   int32_t code = tjsonGetSmallIntValue(pJson, jkDataBlockDescDataBlockId, &pNode->dataBlockId);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkDataBlockResultRowSize, &pNode->resultRowSize);
+    code = tjsonGetIntValue(pJson, jkDataBlockTotalRowSize, &pNode->totalRowSize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDataBlockOutputRowSize, &pNode->outputRowSize);
   }
-
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkDataBlockDescSlots, &pNode->pSlots);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetSmallIntValue(pJson, jkDataBlockPrecision, &pNode->precision);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -582,11 +582,11 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
   nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt);
   taosHashCleanup(cxt.pColIdHash);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
-    nodesDestroyList(cxt.pCols);
+    nodesClearList(cxt.pCols);
     return cxt.errCode;
   }
   if (0 == LIST_LENGTH(cxt.pCols)) {
-    nodesDestroyList(cxt.pCols);
+    nodesClearList(cxt.pCols);
     cxt.pCols = NULL;
   }
   *pCols = cxt.pCols;

source/libs/parser/inc/sql.y

@@ -15,6 +15,7 @@
 #include <assert.h>
 #include <stdbool.h>
 
+#include "functionMgt.h"
 #include "nodes.h"
 #include "parToken.h"
 #include "ttokendef.h"
@@ -348,6 +349,7 @@ literal(A) ::= NK_STRING(B).
 literal(A) ::= NK_BOOL(B).                                                        { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); }
 literal(A) ::= TIMESTAMP(B) NK_STRING(C).                                         { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); }
 literal(A) ::= duration_literal(B).                                               { A = B; }
+literal(A) ::= NULL(B).                                                           { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL)); }
 
 duration_literal(A) ::= NK_VARIABLE(B).                                           { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
 
@@ -371,6 +373,7 @@ signed_literal(A) ::= NK_STRING(B).
 signed_literal(A) ::= NK_BOOL(B).                                                 { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); }
 signed_literal(A) ::= TIMESTAMP NK_STRING(B).                                     { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
 signed_literal(A) ::= duration_literal(B).                                        { A = releaseRawExprNode(pCxt, B); }
+signed_literal(A) ::= NULL.                                                       { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); }
 
 %type literal_list                                                                { SNodeList* }
 %destructor literal_list                                                          { nodesDestroyList($$); }
@@ -417,7 +420,7 @@ topic_name(A) ::= NK_ID(B).
 /************************************************ expression **********************************************************/
 expression(A) ::= literal(B).                                                     { A = B; }
 //expression(A) ::= NK_QUESTION(B).                                                 { A = B; }
-//expression(A) ::= pseudo_column(B).                                               { A = B; }
+expression(A) ::= pseudo_column(B).                                               { A = B; }
 expression(A) ::= column_reference(B).                                            { A = B; }
 expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).             { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
 expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D).                     { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); }
@@ -467,7 +470,38 @@ expression_list(A) ::= expression_list(B) NK_COMMA expression(C).
 column_reference(A) ::= column_name(B).                                           { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
 column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
 
-//pseudo_column(A) ::= NK_NOW.                                                      { A = createFunctionNode(pCxt, NULL, NULL); }
+//pseudo_column(A) ::= NK_NOW.                                                    { A = createFunctionNode(pCxt, NULL, NULL); }
+pseudo_column(A) ::=  NK_UNDERLINE(B) ROWTS(C).                                   {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  TBNAME(B).                                                  { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
+pseudo_column(A) ::=  NK_UNDERLINE(B) QSTARTTS(C).                                {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) QENDTS(C).                                  {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WSTARTTS(C).                                {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WENDTS(C).                                  {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WDURATION(C).                               {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
 
 /************************************************ predicate ***********************************************************/
 predicate(A) ::= expression(B) compare_op(C) expression(D).                       {

source/libs/parser/src/parAstCreater.c

@@ -577,11 +577,13 @@ SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pC
 SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) {
   SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   CHECK_OUT_OF_MEM(val);
-  val->literal = strndup(pLiteral->z, pLiteral->n);
-  if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
-    trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
+  if (NULL != pLiteral) {
+    val->literal = strndup(pLiteral->z, pLiteral->n);
+    if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
+      trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
+    }
+    CHECK_OUT_OF_MEM(val->literal);
   }
-  CHECK_OUT_OF_MEM(val->literal);
   val->node.resType.type = dataType;
   val->node.resType.bytes = IS_VAR_DATA_TYPE(dataType) ? strlen(val->literal) : tDataTypes[dataType].bytes;
   if (TSDB_DATA_TYPE_TIMESTAMP == dataType) {
@@ -717,8 +719,10 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft
 SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
   SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
   CHECK_OUT_OF_MEM(limitNode);
-  // limitNode->limit = limit;
-  // limitNode->offset = offset;
+  limitNode->limit = strtol(pLimit->z, NULL, 10);
+  if (NULL != pOffset) {
+    limitNode->offset = strtol(pOffset->z, NULL, 10);
+  }
   return (SNode*)limitNode;
 }
 

source/libs/parser/src/parTokenizer.c

@@ -99,17 +99,21 @@ static SKeyword keywordTable[] = {
     {"ON",            TK_ON},
     {"OR",            TK_OR},
     {"ORDER",         TK_ORDER},
+    {"PARTITION",     TK_PARTITION},
     {"PASS",          TK_PASS},
     {"PORT",          TK_PORT},
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
+    {"QENDTS",        TK_QENDTS},
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
+    {"QSTARTTS",      TK_QSTARTTS},
     {"QUORUM",        TK_QUORUM},
     {"REPLICA",       TK_REPLICA},
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
+    {"ROWTS",         TK_ROWTS},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
     {"SHOW",          TK_SHOW},
@@ -127,6 +131,7 @@ static SKeyword keywordTable[] = {
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
     {"TAGS",          TK_TAGS},
+    {"TBNAME",        TK_TBNAME},
     {"TIMESTAMP",     TK_TIMESTAMP},
     {"TINYINT",       TK_TINYINT},
     {"TOPIC",         TK_TOPIC},
@@ -141,7 +146,10 @@ static SKeyword keywordTable[] = {
     {"VARCHAR",       TK_VARCHAR},
     {"VGROUPS",       TK_VGROUPS},
     {"WAL",           TK_WAL},
+    {"WDURATION",     TK_WDURATION},
+    {"WENDTS",        TK_WENDTS},
     {"WHERE",         TK_WHERE},
+    {"WSTARTTS",      TK_WSTARTTS},
     // {"ID",           TK_ID},
     // {"STRING",       TK_STRING},
     // {"EQ",           TK_EQ},
@@ -233,7 +241,6 @@ static SKeyword keywordTable[] = {
     // {"TRIGGER",      TK_TRIGGER},
     // {"VIEW",         TK_VIEW},
     // {"SEMI",         TK_SEMI},
-    // {"TBNAME",       TK_TBNAME},
     // {"VNODES",       TK_VNODES},
 //    {"PARTITIONS",   TK_PARTITIONS},
     // {"TOPICS",       TK_TOPICS},
@@ -427,6 +434,10 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
       *tokenId = TK_NK_QUESTION;
       return 1;
     }
+    case '_': {
+      *tokenId = TK_NK_UNDERLINE;
+      return 1;
+    }
     case '`':
     case '\'':
     case '"': {

source/libs/parser/src/parTranslater.c

@@ -235,6 +235,8 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   nodesListAppend(pExpr->pAssociationList, (SNode*)pCol);
   if (NULL != pTable) {
     strcpy(pCol->tableAlias, pTable->tableAlias);
+  } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
+    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
   }
   strcpy(pCol->colName, pExpr->aliasName);
   pCol->node.resType = pExpr->resType;
@@ -617,7 +619,6 @@ static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) {
 }
 
 static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
-  // todo release
   if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
     return TSDB_CODE_SUCCESS;
   }
@@ -631,11 +632,6 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    // todo remove
-    //if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
-    //  taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1);
-    //}
-
     code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
   }
   taosArrayDestroy(vgroupList);
@@ -1488,12 +1484,6 @@ static int32_t nodeTypeToShowType(ENodeType nt) {
 
 static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
   SShowReq showReq = { .type = nodeTypeToShowType(nodeType(pStmt)) };
-  // if ('\0' != pStmt->dbName[0]) {
-  //   SName name = {0};
-  //   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
-  //   char dbFname[TSDB_DB_FNAME_LEN] = {0};
-  //   tNameGetFullDbName(&name, showReq.db);
-  // }
 
   pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
   if (NULL == pCxt->pCmdMsg) {

source/libs/parser/test/parserAstTest.cpp

@@ -206,6 +206,13 @@ TEST_F(ParserTest, selectExpression) {
   ASSERT_TRUE(run());
 }
 
+TEST_F(ParserTest, selectPseudoColumn) {
+  setDatabase("root", "test");
+
+  bind("SELECT _wstartts, _wendts, count(*) FROM t1 interval(10s)");
+  ASSERT_TRUE(run());
+}
+
 TEST_F(ParserTest, selectClause) {
   setDatabase("root", "test");
 
@@ -480,7 +487,10 @@ TEST_F(ParserTest, createTable) {
   bind("create table "
        "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') "
        "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') "
-       "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc')"
+       "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "
+       "if not exists test.t4 using test.st1 (tag1, tag2) tags(3, null) "
+       "if not exists test.t5 using test.st1 (tag1, tag2) tags(null, 'abc') "
+       "if not exists test.t6 using test.st1 (tag1, tag2) tags(null, null)"
       );
   ASSERT_TRUE(run());
 

source/libs/planner/src/planLogicCreater.c

@@ -142,6 +142,46 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STab
   return SCAN_TYPE_TAG;
 }
 
+static SNodeptr createPrimaryKeyCol(uint64_t tableId) {
+  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
+  if (NULL == pCol) {
+    return NULL;
+  }
+  pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
+  pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+  pCol->tableId = tableId;
+  pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
+  pCol->colType = COLUMN_TYPE_COLUMN;
+  strcpy(pCol->colName, "#primarykey");
+  return pCol;
+}
+
+static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) {
+  if (NULL == *pCols) {
+    *pCols = nodesMakeList();
+    if (NULL == *pCols) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  bool found = false;
+  SNode* pCol = NULL;
+  FOREACH(pCol, *pCols) {
+    if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCol)->colId) {
+      found = true;
+      break;
+    }
+  }
+
+  if (!found) {
+    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(*pCols, createPrimaryKeyCol(tableId))) {
+      nodesDestroyList(*pCols);
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
   SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
   if (NULL == pScan) {
@@ -161,7 +201,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   // set columns to scan
   SNodeList* pCols = NULL;
   int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
-  if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addPrimaryKeyCol(pScan->pMeta->uid, &pCols);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
     pScan->pScanCols = nodesCloneList(pCols);
     if (NULL == pScan) {
       code = TSDB_CODE_OUT_OF_MEMORY;
@@ -171,13 +215,15 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta);
 
   // set output
-  if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
+  if (TSDB_CODE_SUCCESS == code) {
     pScan->node.pTargets = nodesCloneList(pCols);
     if (NULL == pScan) {
       code = TSDB_CODE_OUT_OF_MEMORY;
     }
   }
 
+  nodesClearList(pCols);
+
   if (TSDB_CODE_SUCCESS == code) {
     *pLogicNode = (SLogicNode*)pScan;
   } else {
@@ -411,7 +457,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
 }
 
 static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) {
-  int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs);
+  int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs);
 
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
@@ -548,6 +594,22 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  if (NULL != pSelect->pLimit) {
+    pProject->limit = ((SLimitNode*)pSelect->pLimit)->limit;
+    pProject->offset = ((SLimitNode*)pSelect->pLimit)->offset;
+  } else {
+    pProject->limit = -1;
+    pProject->offset = -1;
+  }
+
+  if (NULL != pSelect->pSlimit) {
+    pProject->slimit = ((SLimitNode*)pSelect->pSlimit)->limit;
+    pProject->soffset = ((SLimitNode*)pSelect->pSlimit)->offset;
+  } else {
+    pProject->slimit = -1;
+    pProject->soffset = -1;
+  }
+
   int32_t code = TSDB_CODE_SUCCESS;
 
   pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
@@ -569,15 +631,42 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
   return code;
 }
 
+static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
+  if (NULL == pSelect->pPartitionByList) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SPartitionLogicNode* pPartition = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PARTITION);
+  if (NULL == pPartition) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
+  if (!pSelect->isDistinct) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
   SLogicNode* pRoot = NULL;
   int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createChildLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot);
   }

source/libs/planner/src/planPhysiCreater.c

@@ -36,10 +36,6 @@ typedef struct SPhysiPlanContext {
 } SPhysiPlanContext;
 
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
-  if (QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode)) {
-    return getSlotKey(((SOrderByExprNode*)pNode)->pExpr, pStmtName, pKey);
-  }
-
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SColumnNode* pCol = (SColumnNode*)pNode;
     if (NULL != pStmtName) {
@@ -135,7 +131,8 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
       code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
     }
     if (TSDB_CODE_SUCCESS == code) {
-      pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->outputRowSize += ((SExprNode*)pNode)->resType.bytes;
       ++slotId;
     } else {
       break;
@@ -184,15 +181,19 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
   int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0;
   SNode* pNode = NULL;
   FOREACH(pNode, pList) {
+    SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode;
     char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0};
-    int32_t len = getSlotKey(pNode, pStmtName, name);
+    int32_t len = getSlotKey(pExpr, pStmtName, name);
     SSlotIndex* pIndex = taosHashGet(pHash, name, len);
     if (NULL == pIndex) {
-      code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output));
+      code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output));
       if (TSDB_CODE_SUCCESS == code) {
         code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
       }
-      pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->totalRowSize += ((SExprNode*)pExpr)->resType.bytes;
+      if (output) {
+        pDataBlockDesc->outputRowSize += ((SExprNode*)pExpr)->resType.bytes;
+      }
       slotId = nextSlotId;
       ++nextSlotId;
     } else {
@@ -314,20 +315,6 @@ static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pL
   return TSDB_CODE_SUCCESS;
 }
 
-static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) {
-  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
-  if (NULL == pCol) {
-    return NULL;
-  }
-  pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
-  pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
-  pCol->tableId = tableId;
-  pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
-  pCol->colType = COLUMN_TYPE_COLUMN;
-  strcpy(pCol->colName, "#primarykey");
-  return pCol;
-}
-
 static int32_t colIdCompare(const void* pLeft, const void* pRight) {
   SColumnNode* pLeftCol = *(SColumnNode**)pLeft;
   SColumnNode* pRightCol = *(SColumnNode**)pRight;
@@ -356,35 +343,10 @@ static int32_t sortScanCols(SNodeList* pScanCols) {
 }
 
 static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) {
-  if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode)
-      || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) {
-    pScanPhysiNode->pScanCols = nodesMakeList();
-    if (NULL == pScanPhysiNode->pScanCols) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    SNode* pNode;
-    FOREACH(pNode, pScanCols) {
-      if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) {
-        SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0);
-        strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias);
-        strcpy(pCol->colName, ((SColumnNode*)pNode)->colName);
-        continue;
-      }
-      if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-    }
-  } else {
-    pScanPhysiNode->pScanCols = nodesCloneList(pScanCols);
-    if (NULL == pScanPhysiNode->pScanCols) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
+  pScanPhysiNode->pScanCols = nodesCloneList(pScanCols);
+  if (NULL == pScanPhysiNode->pScanCols) {
+    return TSDB_CODE_OUT_OF_MEMORY;
   }
-
   return sortScanCols(pScanPhysiNode->pScanCols);
 }
 
@@ -600,7 +562,7 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
       return collectAndRewrite(pContext, pNode);
     }
     case QUERY_NODE_FUNCTION: {
-      if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) {
+      if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) {
         return collectAndRewrite(pContext, pNode);
       }
     }
@@ -711,6 +673,11 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pProject->limit = pProjectLogicNode->limit;
+  pProject->offset = pProjectLogicNode->offset;
+  pProject->slimit = pProjectLogicNode->slimit;
+  pProject->soffset = pProjectLogicNode->soffset;
+
   int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections);
   if (TSDB_CODE_SUCCESS == code) {
     code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc);

source/libs/planner/test/plannerTest.cpp

@@ -192,6 +192,9 @@ TEST_F(PlannerTest, interval) {
 
   bind("SELECT count(*) FROM t1 interval(10s)");
   ASSERT_TRUE(run());
+
+  bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
+  ASSERT_TRUE(run());
 }
 
 TEST_F(PlannerTest, sessionWindow) {
@@ -204,7 +207,7 @@ TEST_F(PlannerTest, sessionWindow) {
 TEST_F(PlannerTest, orderBy) {
   setDatabase("root", "test");
 
-  bind("SELECT * FROM t1 order by c1");
+  bind("SELECT c1 FROM t1 order by c1");
   ASSERT_TRUE(run());
 
   bind("SELECT c1 FROM t1 order by c2");
@@ -214,6 +217,32 @@ TEST_F(PlannerTest, orderBy) {
   ASSERT_TRUE(run());
 }
 
+TEST_F(PlannerTest, limit) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM t1 limit 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 limit 5 offset 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 limit 2, 5");
+  ASSERT_TRUE(run());
+}
+
+TEST_F(PlannerTest, slimit) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM t1 partition by c1 slimit 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 partition by c1 slimit 5 soffset 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 partition by c1 slimit 2, 5");
+  ASSERT_TRUE(run());
+}
+
 TEST_F(PlannerTest, showTables) {
   setDatabase("root", "test");
 

source/libs/stream/src/tstream.c

@@ -31,12 +31,13 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
   if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
     ((SMsgHead*)buf)->vgId = 0;
     req.taskId = pTask->inplaceDispatcher.taskId;
+
   } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
     ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId);
     *ppEpSet = &pTask->fixedEpDispatcher.epSet;
     req.taskId = pTask->fixedEpDispatcher.taskId;
+
   } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-    int32_t nodeId = 0;
     // TODO fix tbname issue
     char ctbName[TSDB_TABLE_FNAME_LEN + 22];
     // all groupId must be the same in an array
@@ -52,10 +53,12 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
     // TODO: optimize search process
     SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
     int32_t sz = taosArrayGetSize(vgInfo);
+    int32_t nodeId = 0;
     for (int32_t i = 0; i < sz; i++) {
       SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
       if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
         nodeId = pVgInfo->vgId;
+        req.taskId = pVgInfo->taskId;
         *ppEpSet = &pVgInfo->epSet;
         break;
       }
@@ -71,6 +74,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
   pMsg->contLen = tlen;
   pMsg->code = 0;
   pMsg->msgType = pTask->dispatchMsgType;
+  pMsg->noResp = 1;
 
   return 0;
 }
@@ -80,7 +84,7 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb
   while (1) {
     pIter = taosHashIterate(data, pIter);
     if (pIter == NULL) return 0;
-    SArray* pData = (SArray*)pIter;
+    SArray* pData = *(SArray**)pIter;
     SRpcMsg dispatchMsg = {0};
     SEpSet* pEpSet;
     if (streamBuildDispatchMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
@@ -98,7 +102,6 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK && pTask->sourceType != TASK_SOURCE__SCAN) return 0;
 
   // exec
-  // TODO: for shuffle dispatcher, merge data by groupId
   if (pTask->execType != TASK_EXEC__NONE) {
     ASSERT(workId < pTask->exec.numOfRunners);
     void* exec = pTask->exec.runners[workId].executor;
@@ -121,22 +124,23 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
       }
     } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
       const SArray* blocks = (const SArray*)input;
-      int32_t       sz = taosArrayGetSize(blocks);
-      for (int32_t i = 0; i < sz; i++) {
-        SSDataBlock* pBlock = taosArrayGet(blocks, i);
-        qSetStreamInput(exec, pBlock, inputType);
-        while (1) {
-          SSDataBlock* output;
-          uint64_t     ts;
-          if (qExecTask(exec, &output, &ts) < 0) {
-            ASSERT(false);
-          }
-          if (output == NULL) {
-            break;
-          }
-          taosArrayPush(pRes, output);
+      /*int32_t       sz = taosArrayGetSize(blocks);*/
+      /*for (int32_t i = 0; i < sz; i++) {*/
+      /*SSDataBlock* pBlock = taosArrayGet(blocks, i);*/
+      /*qSetStreamInput(exec, pBlock, inputType);*/
+      qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
+      while (1) {
+        SSDataBlock* output;
+        uint64_t     ts;
+        if (qExecTask(exec, &output, &ts) < 0) {
+          ASSERT(false);
+        }
+        if (output == NULL) {
+          break;
         }
+        taosArrayPush(pRes, output);
       }
+      /*}*/
     } else {
       ASSERT(0);
     }

source/libs/transport/src/transSrv.c

@@ -214,9 +214,11 @@ static void uvHandleReq(SSrvConn* pConn) {
     // no ref here
   }
 
-  if (pHead->noResp == 0) {
-    transMsg.handle = pConn;
-  }
+  // if pHead->noResp = 1,
+  // 1. server application should not send resp on handle
+  // 2. once send out data, cli conn released to conn pool immediately
+  // 3. not mixed with persist
+  transMsg.handle = pConn;
 
   STrans* pTransInst = pConn->pTransInst;
   (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);

tests/script/jenkins/basic.txt

@@ -21,6 +21,10 @@
 ./test.sh -f tsim/insert/backquote.sim
 ./test.sh -f tsim/insert/null.sim
 
+# ---- parser
+#./test.sh -f tsim/parser/groupby-basic.sim
+#./test.sh -f tsim/parser/fourArithmetic-basic.sim
+
 # ---- query
 ./test.sh -f tsim/query/interval.sim
 ./test.sh -f tsim/query/interval-offset.sim
@@ -33,5 +37,6 @@
 
 # ---- tmq
 ./test.sh -f tsim/tmq/basic.sim
+#./test.sh -f tsim/tmq/basic1.sim
 
 #======================b1-end===============

tests/script/sh/massiveTable/compileVersion.sh

@@ -75,10 +75,12 @@ rm -f /usr/bin/taos
 rm -f /usr/bin/taosd
 rm -f /usr/bin/create_table
 rm -f /usr/bin/tmq_demo
+rm -f /usr/bin/tmq_sim
 
 ln -s $taos_dir/taos /usr/bin/taos
 ln -s $taosd_dir/taosd /usr/bin/taosd
 ln -s $exec_process_dir/create_table /usr/bin/create_table
 ln -s $exec_process_dir/tmq_demo /usr/bin/tmq_demo
+ln -s $exec_process_dir/tmq_sim /usr/bin/tmq_sim
 
 

tests/script/tsim/parser/fourArithmetic-basic.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$dbNamme = d0
+print =============== create database
+sql create database $dbNamme vgroups 1
+sql show databases
+print $data00 $data01 $data02
+if $rows != 2 then 
+  return -1
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct0 using stb tags(1000)
+#sql create table ct1 using stb tags(2000)
+#sql create table ct3 using stb tags(3000)
+
+sql show tables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== insert data
+
+$tbPrefix = ct
+$tbNum = 1
+$rowNum = 10
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c2 = $x + 10
+    $c3 = $x * 10
+    $c4 = $x - 10
+
+    sql insert into $tb values ($tstart , $x , $c2 ,  $c3 , $c4 )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
+print ===> rows: $rows 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 10 then
+ return -1
+endi
+
+if $data01 != 10.000000000 then
+ return -1
+endi
+if $data02 != -nan then
+ return -1
+endi
+if $data03 != -10.000000000 then
+ return -1
+endi
+
+if $data91 != 10.000000000 then
+ return -1
+endi
+if $data92 != 10.000000000 then
+ return -1
+endi
+if $data93 != 8.000000000 then
+ return -1
+endi
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/interval-offset.sim

@@ -41,6 +41,38 @@ sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6 )
 sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7 )
 sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8 )
 
+print =============== insert data into child table ct2 (d)
+sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
+sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
+sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
+sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
+sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
+sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
+sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
+
+print =============== insert data into child table ct3 (n)
+sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL )
+sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 )
+sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 )
+sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 )
+sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 )
+sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 )
+sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 )
+sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 )
+sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 )
+
+print =============== insert data into child table ct4 (y)
+sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
+sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
+sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
+sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
+sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
+sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
+sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
+sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
+sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
+
+print ================ start query ======================
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
 print ===> rows: $rows
@@ -102,15 +134,6 @@ if $data80 != 1 then
   return -1
 endi 
 
-print =============== insert data into child table ct2 (d)
-sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
-sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
-sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
-sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
-sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
-sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
-sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
-
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
 print ===> rows: $rows
@@ -140,45 +163,19 @@ print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
 print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 
 print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 
 print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 
-#if $rows != 8 then
-#  return -1
-#endi 
-#if $data00 != 1 then
-#  return -1
-#endi  
-#if $data10 != 2 then
-#  return -1
-#endi   
-#if $data20 != 2 then
-#  return -1
-#endi   
-#if $data30 != 2 then
-#  return -1
-#endi   
-#if $data40 != 2 then
-#  return -1
-#endi   
-#if $data50 != 2 then
-#  return -1
-#endi   
-#if $data60 != 2 then
-#  return -1
-#endi 
-#if $data70 != 1 then
-#  return -1
-#endi
-
-print =============== insert data into child table ct3 (n)
-sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL );
-sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 );
-sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 );
-sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 );
-sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 );
-sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 );
-sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 );
-sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 );
-sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 );
-
+if $rows != 8 then
+  return -1
+endi 
+if $data00 != 1 then
+  return -1
+endi  
+if $data10 != 2 then
+  return -1
+endi   
+if $data70 != 1 then
+  return -1
+endi
+ 
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> rows: $rows
@@ -236,18 +233,6 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
 #  return -1
 #endi 
 
-
-print =============== insert data into child table ct4 (y)
-sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
-sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
-sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
-sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
-sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
-sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
-sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
-sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
-sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
-
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
 print ===> rows: $rows
@@ -305,6 +290,27 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
 #  return -1
 #endi 
 
+#=================================================
+print =============== stop and restart taosd
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
 
 
 

tests/script/tsim/testCaseSuite.sim

@@ -2,6 +2,8 @@
 run tsim/user/basic1.sim
 
 run tsim/db/basic1.sim
+run tsim/db/basic2.sim
+run tsim/db/basic3.sim
 run tsim/db/basic6.sim
 run tsim/db/basic7.sim
 run tsim/db/error1.sim
@@ -13,12 +15,15 @@ run tsim/insert/basic1.sim
 run tsim/insert/backquote.sim
 run tsim/insert/null.sim
 
+#run tsim/parser/groupby-basic.sim
+#run tsim/parser/fourArithmetic-basic.sim
+
 run tsim/query/interval.sim
-#run tsim/query/interval-offset.sim  # TD-14266
+run tsim/query/interval-offset.sim
 
 run tsim/show/basic.sim
 
 run tsim/table/basic1.sim
 
 run tsim/tmq/basic.sim
-
+#run tsim/tmq/basic1.sim

tests/script/tsim/tmq/basic1.sim

+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$dbNamme = d0
+print =============== create database , vgroup 1
+sql create database $dbNamme vgroups 1
+sql show databases
+print $data00 $data01 $data02
+if $rows != 2 then 
+  return -1
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct0 using stb tags(1000)
+sql create table ct1 using stb tags(2000)
+#sql create table ct3 using stb tags(3000)
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb
+print =============== will support: * from stb; function from stb/ctb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+#sql create topic topic_stb_function as select ts, abs(c1), sina(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+#sql create topic topic_ctb_function as select ts, abs(c1), sina(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+#sql create topic topic_ntb_function as select ts, abs(c1), sina(c2) from ntb
+
+sql show tables
+if $rows != 3 then 
+  return -1
+endi
+
+print =============== insert data
+
+$tbPrefix = ct
+$tbNum = 2
+$rowNum = 10
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , $c , $x , $binary )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$totalMsgCnt = $rowNum * $tbNum
+print inserted totalMsgCnt: $totalMsgCnt
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20}@ then
+  return -1
+endi
+
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#print cmd result----> $system_content
+#if $system_content != @{consume success: 20}@ then
+#  return -1
+#endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 10}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 10}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20}@ then
+  return -1
+endi
+
+print =============== create database , vgroup 4
+$dbNamme = d1
+sql create database $dbNamme vgroups 4
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/test/c/CMakeLists.txt

 add_executable(create_table create_table.c)
 add_executable(tmq_demo tmqDemo.c)
+add_executable(tmq_sim tmqSim.c)
 target_link_libraries(
     create_table
     PUBLIC taos
@@ -14,3 +15,10 @@ target_link_libraries(
     PUBLIC common
     PUBLIC os
 )
+target_link_libraries(
+    tmq_sim
+    PUBLIC taos
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)