Merge remote-tracking branch 'origin/3.0' into feature/scheduler

contrib/CMakeLists.txt

@@ -83,11 +83,6 @@ if(${BUILD_WITH_NURAFT})
     cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_NURAFT})
 
-# iconv
-if(${BUILD_WITH_ICONV})
-    cat("${CMAKE_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-endif(${BUILD_WITH_ICONV})
-
 # download dependencies
 configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -213,10 +208,9 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
-    if (${TD_WINDOWS})
-        file(READ "libuv/include/uv.h" CONTENTS)
-        string(REGEX REPLACE "/([\r]*)\nstruct uv_tcp_s {" "/\\1\ntypedef BOOL (PASCAL *LPFN_CONNECTEX) (SOCKET s, const struct sockaddr* name, int namelen, PVOID lpSendBuffer, DWORD dwSendDataLength,LPDWORD lpdwBytesSent, LPOVERLAPPED lpOverlapped);\\1\nstruct uv_tcp_s {" CONTENTS_NEW "${CONTENTS}")
-        file(WRITE "libuv/include/uv.h" "${CONTENTS_NEW}")
+    if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+        MESSAGE("Windows need set no-sign-compare")
+        add_compile_options(-Wno-sign-compare)
     endif ()
     add_subdirectory(libuv)
 endif(${BUILD_WITH_UV})
@@ -249,15 +243,7 @@ if(${BUILD_WITH_SQLITE})
 endif(${BUILD_WITH_SQLITE})
 
 # pthread
-if(${BUILD_PTHREAD})
-    add_definitions(-DPTW32_STATIC_LIB)
-    add_subdirectory(pthread)
-endif(${BUILD_PTHREAD})
-
-# iconv
-if(${BUILD_WITH_ICONV})
-    add_subdirectory(iconv)
-endif(${BUILD_WITH_ICONV})
+
 
 # ================================================================================================
 # Build test

example/CMakeLists.txt

-aux_source_directory(src TMQ_DEMO_SRC)
+add_executable(tmq "")
+add_executable(tstream "")
 
-add_executable(tmq ${TMQ_DEMO_SRC})
-target_link_libraries(
-    tmq taos
+target_sources(tmq
+    PRIVATE
+    "src/tmq.c"
 )
-target_include_directories(
-    tmq
+
+target_sources(tstream
+    PRIVATE
+    "src/tstream.c"
+)
+target_link_libraries(tmq
+    taos
+)
+
+target_link_libraries(tstream
+    taos
+)
+
+target_include_directories(tmq
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_include_directories(tstream
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
 SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
+SET_TARGET_PROPERTIES(tstream PROPERTIES OUTPUT_NAME tstream)

example/src/tstream.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include "taos.h"
+
+int32_t init_env() {
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  if (pConn == NULL) {
+    return -1;
+  }
+
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  return 0;
+}
+
+int32_t create_stream() {
+  printf("create stream\n");
+  TAOS_RES* pRes;
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  if (pConn == NULL) {
+    return -1;
+  }
+
+  pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
+  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
+  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  taos_close(pConn);
+  return 0;
+}
+
+int main(int argc, char* argv[]) {
+  int code;
+  if (argc > 1) {
+    printf("env init\n");
+    code = init_env();
+  }
+  create_stream();
+#if 0
+  tmq_t*      tmq = build_consumer();
+  tmq_list_t* topic_list = build_topic_list();
+  /*perf_loop(tmq, topic_list);*/
+  /*basic_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
+#endif
+}

include/client/taos.h

@@ -214,7 +214,6 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 
-DLL_EXPORT TAOS_RES   *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
 DLL_EXPORT tmq_t      *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 DLL_EXPORT void        tmq_message_destroy(tmq_message_t *tmq_message);
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
@@ -258,7 +257,12 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
 DLL_EXPORT char    *tmq_get_topic_name(tmq_message_t *message);
 
-/* ---------------------- OTHER ---------------------------- */
+/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
+DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
+
+DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
+
+/* -------------------------------- OTHER -------------------------------- */
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 
 DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);

include/common/taosdef.h

@@ -63,9 +63,12 @@ typedef enum {
 } ETsdbStatisStatus;
 
 typedef enum {
-  TSDB_SMA_STAT_OK = 0,       // ready to provide service
-  TSDB_SMA_STAT_EXPIRED = 1,  // not ready or expired
-} ETsdbSmaStat;
+  TSDB_SMA_STAT_UNKNOWN = -1,  // unknown
+  TSDB_SMA_STAT_OK = 0,        // ready to provide service
+  TSDB_SMA_STAT_EXPIRED = 1,   // not ready or expired
+  TSDB_SMA_STAT_DROPPED = 2,   // sma dropped
+} ETsdbSmaStat;                // bit operation
+
 
 typedef enum {
   TSDB_SMA_TYPE_BLOCK = 0,       // Block-wise SMA
@@ -75,6 +78,8 @@ typedef enum {
 
 extern char *qtypeStr[];
 
+#define TSDB_PORT_DNODEDNODE 5
+#define TSDB_PORT_SYNC      10
 #define TSDB_PORT_HTTP      11
 
 #ifdef __cplusplus

include/common/tcommon.h

@@ -91,6 +91,9 @@ void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
+int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
+void*   tDecodeDataBlocks(const void* buf, SArray* blocks);
+
 static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
   // WARNING: do not use info.numOfCols,
   // sometimes info.numOfCols != array size

include/common/tdatablock.h

@@ -120,7 +120,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
 SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
 
 size_t blockDataGetSize(const SSDataBlock* pBlock);
-size_t blockDataGetRowSize(const SSDataBlock* pBlock);
+size_t blockDataGetRowSize(SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
 size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 
@@ -131,7 +131,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 
 int32_t      blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
 int32_t      blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
-void         blockDataClearup(SSDataBlock* pDataBlock);
+void         blockDataCleanup(SSDataBlock* pDataBlock);
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
 size_t       blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 void*        blockDataDestroy(SSDataBlock* pBlock);

include/common/tmsg.h

@@ -23,8 +23,8 @@
 #include "tencode.h"
 #include "thash.h"
 #include "tlist.h"
-#include "trow.h"
 #include "tname.h"
+#include "trow.h"
 #include "tuuid.h"
 
 #ifdef __cplusplus
@@ -109,6 +109,7 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_STREAMTABLES,
   TSDB_MGMT_TABLE_TP,
   TSDB_MGMT_TABLE_FUNC,
+  TSDB_MGMT_TABLE_INDEX,
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
@@ -268,11 +269,15 @@ typedef struct SSchema {
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
+  float   xFilesFactor;
+  int32_t aggregationMethod;
+  int32_t delay;
   int32_t numOfColumns;
   int32_t numOfTags;
+  int32_t commentLen;
   SArray* pColumns;
   SArray* pTags;
-  char    comment[TSDB_STB_COMMENT_LEN];
+  char*   comment;
 } SMCreateStbReq;
 
 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@@ -462,7 +467,8 @@ typedef struct {
   int32_t tz;  // query client timezone
   char    intervalUnit;
   char    slidingUnit;
-  char    offsetUnit;
+  char    offsetUnit;   // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+  int8_t  precision;
   int64_t interval;
   int64_t sliding;
   int64_t offset;
@@ -472,10 +478,9 @@ typedef struct {
   int32_t code;
 } SQueryTableRsp;
 
-int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp);
-
-int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp);
+int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
 
+int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
@@ -888,14 +893,14 @@ typedef struct {
 } SRetrieveTableRsp;
 
 typedef struct {
-  int64_t  handle;
-  int64_t  useconds;
-  int8_t   completed;  // all results are returned to client
-  int8_t   precision;
-  int8_t   compressed;
-  int32_t  compLen;
-  int32_t  numOfRows;
-  char     data[];
+  int64_t handle;
+  int64_t useconds;
+  int8_t  completed;  // all results are returned to client
+  int8_t  precision;
+  int8_t  compressed;
+  int32_t compLen;
+  int32_t numOfRows;
+  char    data[];
 } SRetrieveMetaTableRsp;
 
 typedef struct {
@@ -1362,28 +1367,48 @@ typedef struct {
   int64_t  tuid;
 } SDDropTopicReq;
 
+typedef struct {
+  float    xFilesFactor;
+  int8_t   delayUnit;
+  int8_t   nFuncIds;
+  int32_t* pFuncIds;
+  int64_t  delay;
+} SRSmaParam;
+
 typedef struct SVCreateTbReq {
   int64_t  ver;  // use a general definition
   char*    dbFName;
   char*    name;
   uint32_t ttl;
   uint32_t keep;
-  uint8_t  type;
   union {
+    uint8_t info;
     struct {
-      tb_uid_t suid;
-      uint32_t nCols;
-      SSchema* pSchema;
-      uint32_t nTagCols;
-      SSchema* pTagSchema;
+      uint8_t rollup : 1;  // 1 means rollup sma
+      uint8_t type : 7;
+    };
+  };
+  union {
+    struct {
+      tb_uid_t    suid;
+      uint32_t    nCols;
+      SSchema*    pSchema;
+      uint32_t    nTagCols;
+      SSchema*    pTagSchema;
+      col_id_t    nBSmaCols;
+      col_id_t*   pBSmaCols;
+      SRSmaParam* pRSmaParam;
     } stbCfg;
     struct {
       tb_uid_t suid;
       SKVRow   pTag;
     } ctbCfg;
     struct {
-      uint32_t nCols;
-      SSchema* pSchema;
+      uint32_t    nCols;
+      SSchema*    pSchema;
+      col_id_t    nBSmaCols;
+      col_id_t*   pBSmaCols;
+      SRSmaParam* pRSmaParam;
     } ntbCfg;
   };
 } SVCreateTbReq, SVUpdateTbReq;
@@ -1404,12 +1429,11 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
 void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
 
 typedef struct {
-  SArray* rspList; // SArray<SVCreateTbRsp>
+  SArray* rspList;  // SArray<SVCreateTbRsp>
 } SVCreateTbBatchRsp;
 
-int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp);
-int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp);
-
+int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
+int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
 
 typedef struct {
   int64_t  ver;
@@ -1919,12 +1943,47 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
   }
   return buf;
 }
+
+typedef struct {
+  char    name[TSDB_TABLE_FNAME_LEN];
+  char    stb[TSDB_TABLE_FNAME_LEN];
+  int8_t  igExists;
+  int8_t  intervalUnit;
+  int8_t  slidingUnit;
+  int8_t  timezone;
+  int32_t dstVgId;  // for stream
+  int64_t interval;
+  int64_t offset;
+  int64_t sliding;
+  int32_t exprLen;        // strlen + 1
+  int32_t tagsFilterLen;  // strlen + 1
+  int32_t sqlLen;         // strlen + 1
+  int32_t astLen;         // strlen + 1
+  char*   expr;  
+  char*   tagsFilter;
+  char*   sql;
+  char*   ast;
+} SMCreateSmaReq;
+
+int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
+int32_t tDeserializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
+void    tFreeSMCreateSmaReq(SMCreateSmaReq* pReq);
+
+typedef struct {
+  char   name[TSDB_TABLE_FNAME_LEN];
+  int8_t igNotExists;
+} SMDropSmaReq;
+
+int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
+int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
+
 typedef struct {
   int8_t   version;       // for compatibility(default 0)
   int8_t   intervalUnit;  // MACRO: TIME_UNIT_XXX
   int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
+  int8_t   timezoneInt;      // sma data expired if timezone changes.
   char     indexName[TSDB_INDEX_NAME_LEN];
-  char     timezone[TD_TIMEZONE_LEN];  // sma data expired if timezone changes.
+  char     timezone[TD_TIMEZONE_LEN];  
   int32_t  exprLen;
   int32_t  tagsFilterLen;
   int64_t  indexUid;
@@ -2031,8 +2090,8 @@ static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
   tlen += taosEncodeFixedI8(buf, pSma->version);
   tlen += taosEncodeFixedI8(buf, pSma->intervalUnit);
   tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
+  tlen += taosEncodeFixedI8(buf, pSma->timezoneInt);
   tlen += taosEncodeString(buf, pSma->indexName);
-  tlen += taosEncodeString(buf, pSma->timezone);
   tlen += taosEncodeFixedI32(buf, pSma->exprLen);
   tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen);
   tlen += taosEncodeFixedI64(buf, pSma->indexUid);
@@ -2066,8 +2125,8 @@ static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
   buf = taosDecodeFixedI8(buf, &pSma->version);
   buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
   buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
+  buf = taosDecodeFixedI8(buf, &pSma->timezoneInt);
   buf = taosDecodeStringTo(buf, pSma->indexName);
-  buf = taosDecodeStringTo(buf, pSma->timezone);
   buf = taosDecodeFixedI32(buf, &pSma->exprLen);
   buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen);
   buf = taosDecodeFixedI64(buf, &pSma->indexUid);
@@ -2291,22 +2350,30 @@ enum {
   STREAM_TASK_STATUS__STOP,
 };
 
+enum {
+  STREAM_NEXT_OP_DST__VND = 1,
+  STREAM_NEXT_OP_DST__SND,
+};
+
 typedef struct {
   void* inputHandle;
-  void* executor[4];
-} SStreamTaskParRunner;
+  void* executor;
+} SStreamRunner;
 
 typedef struct {
   int64_t streamId;
   int32_t taskId;
   int32_t level;
   int8_t  status;
-  int8_t  pipeEnd;
-  int8_t  parallel;
+  int8_t  pipeSource;
+  int8_t  pipeSink;
+  int8_t  numOfRunners;
+  int8_t  parallelizable;
+  int8_t  nextOpDst;  // vnode or snode
   SEpSet  NextOpEp;
   char*   qmsg;
   // not applied to encoder and decoder
-  SStreamTaskParRunner runner;
+  SStreamRunner runner[8];
   // void*                executor;
   // void*   stateStore;
   //  storage handle
@@ -2318,6 +2385,8 @@ static FORCE_INLINE SStreamTask* streamTaskNew(int64_t streamId, int32_t level)
     return NULL;
   }
   pTask->taskId = tGenIdPI32();
+  pTask->streamId = streamId;
+  pTask->level = level;
   pTask->status = STREAM_TASK_STATUS__RUNNING;
   pTask->qmsg = NULL;
   return pTask;
@@ -2338,13 +2407,20 @@ typedef struct {
 
 typedef struct {
   SStreamExecMsgHead head;
-  // TODO: other info needed by task
+  SArray*            data;  // SArray<SSDataBlock>
 } SStreamTaskExecReq;
 
 typedef struct {
   int32_t reserved;
 } SStreamTaskExecRsp;
 
+typedef struct {
+  SMsgHead head;
+  int64_t  streamId;
+  int64_t  version;
+  SArray*  res;  // SArray<SSDataBlock>
+} SStreamSmaSinkReq;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -127,6 +127,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL)
@@ -191,6 +193,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqCVConsumeReq, SMqCVConsumeRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TASK_EXEC, "vnode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
 
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)

include/common/trow.h

@@ -166,6 +166,7 @@ typedef struct {
 #define TD_ROW_HEAD_LEN  (sizeof(STSRow))
 #define TD_ROW_NCOLS_LEN (sizeof(col_id_t))
 
+#define TD_ROW_INFO(r)     ((r)->info)
 #define TD_ROW_TYPE(r)     ((r)->type)
 #define TD_ROW_DELETE(r)   ((r)->del)
 #define TD_ROW_ENDIAN(r)   ((r)->endian)
@@ -180,6 +181,7 @@ typedef struct {
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
 #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
 
+#define TD_ROW_SET_INFO(r, i)  (TD_ROW_INFO(r) = (i))
 #define TD_ROW_SET_TYPE(r, t)  (TD_ROW_TYPE(r) = (t))
 #define TD_ROW_SET_DELETE(r)   (TD_ROW_DELETE(r) = 1)
 #define TD_ROW_SET_SVER(r, v)  (TD_ROW_SVER(r) = (v))
@@ -473,6 +475,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
     return terrno;
   }
 
+  TD_ROW_SET_INFO(pBuilder->pBuf, 0);
   TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType);
 
   uint32_t len = 0;
@@ -968,10 +971,14 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in
 #endif
     return TSDB_CODE_SUCCESS;
   }
-  if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
+
+  if (TD_COL_ROWS_NORM(pCol)) {
+    pVal->valType = TD_VTYPE_NORM;
+  } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
     return terrno;
   }
-  if (TD_COL_ROWS_NORM(pCol) || tdValTypeIsNorm(pVal->valType)) {
+
+  if (tdValTypeIsNorm(pVal->valType)) {
     if (IS_VAR_DATA_TYPE(pCol->type)) {
       pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
     } else {

include/common/ttypes.h

@@ -46,7 +46,7 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))

include/dnode/mnode/sdb/sdb.h

@@ -119,10 +119,11 @@ typedef enum {
   SDB_CONSUMER = 13,
   SDB_TOPIC = 14,
   SDB_VGROUP = 15,
-  SDB_STB = 16,
-  SDB_DB = 17,
-  SDB_FUNC = 18,
-  SDB_MAX = 19
+  SDB_SMA = 16,
+  SDB_STB = 17,
+  SDB_DB = 18,
+  SDB_FUNC = 19,
+  SDB_MAX = 20
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/libs/executor/executor.h

@@ -35,15 +35,15 @@ typedef struct SReadHandle {
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
-#define STREAM_DATA_TYPE_SSDAT_BLOCK  0x2
-
- /**
-  * Create the exec task for streaming mode
-  * @param pMsg
-  * @param streamReadHandle
-  * @return
-  */
-qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle);
+#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
+  
+/**
+ * Create the exec task for streaming mode
+ * @param pMsg
+ * @param streamReadHandle
+ * @return
+ */
+qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
 
 /**
  * Set the input data block for the stream scan.
@@ -64,16 +64,17 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
  */
 int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd);
 
- /**
-  * Create the exec task object according to task json
-  * @param readHandle
-  * @param vgId
-  * @param pTaskInfoMsg
-  * @param pTaskInfo
-  * @param qId
-  * @return
-  */
-int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
+/**
+ * Create the exec task object according to task json
+ * @param readHandle
+ * @param vgId
+ * @param pTaskInfoMsg
+ * @param pTaskInfo
+ * @param qId
+ * @return
+ */
+int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
+                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
 
 /**
  * The main task execution function, including query on both table and multiple tables,
@@ -83,7 +84,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
  * @param handle
  * @return
  */
-int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds);
+int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds);
 
 /**
  * Retrieve the produced results information, if current query is not paused or completed,
@@ -146,7 +147,8 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
  * @param numOfIndex
  * @return
  */
-//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex);
+// int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex*
+// groupByIndex, int32_t numOfIndex);
 
 /**
  * Update the table id list of a given query.
@@ -169,19 +171,19 @@ void* qOpenTaskMgmt(int32_t vgId);
  * broadcast the close information and wait for all query stop.
  * @param pExecutor
  */
-void  qTaskMgmtNotifyClosing(void* pExecutor);
+void qTaskMgmtNotifyClosing(void* pExecutor);
 
 /**
  * Re-open the query handle management module when opening the vnode again.
  * @param pExecutor
  */
-void  qQueryMgmtReOpen(void *pExecutor);
+void qQueryMgmtReOpen(void* pExecutor);
 
 /**
  * Close query mgmt and clean up resources.
  * @param pExecutor
  */
-void  qCleanupTaskMgmt(void* pExecutor);
+void qCleanupTaskMgmt(void* pExecutor);
 
 /**
  * Add the query into the query mgmt object
@@ -190,7 +192,7 @@ void  qCleanupTaskMgmt(void* pExecutor);
  * @param qInfo
  * @return
  */
-void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo);
+void** qRegisterTask(void* pMgmt, uint64_t qId, void* qInfo);
 
 /**
  * acquire the query handle according to the key from query mgmt object.

include/libs/sync/sync.h

@@ -35,6 +35,7 @@ typedef enum {
   TAOS_SYNC_STATE_FOLLOWER = 100,
   TAOS_SYNC_STATE_CANDIDATE = 101,
   TAOS_SYNC_STATE_LEADER = 102,
+  TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
 typedef struct SSyncBuffer {
@@ -68,17 +69,20 @@ typedef struct SSnapshot {
 typedef struct SSyncFSM {
   void* data;
 
-  // when value in pBuf finish a raft flow, FpCommitCb is called, code indicates the result
+  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
   // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                     ESyncState state);
 
-  // when value in pBuf has been written into local log store, FpPreCommitCb is called, code indicates the result
+  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
   // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                        ESyncState state);
 
   // when log entry is updated by a new one, FpRollBackCb is called
   // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pBuf, SyncIndex index, bool isWeak, int32_t code);
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
+                       ESyncState state);
 
   // user should implement this function, use "data" to take snapshot into "snapshot"
   int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
@@ -157,10 +161,14 @@ void    syncCleanUp();
 int64_t    syncStart(const SSyncInfo* pSyncInfo);
 void       syncStop(int64_t rid);
 int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);        // use this function
-int32_t    syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);  // just for compatibility
+int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 ESyncState syncGetMyRole(int64_t rid);
-void       syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
+
+// propose with sequence number, to implement linearizable semantics
+int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
 extern int32_t sDebugFlag;
 

include/os/osSocket.h

@@ -51,10 +51,31 @@
 extern "C" {
 #endif
 
-#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 
+#if defined(WINDOWS)
   #define htobe64 htonll
 #endif
 
+#if defined(WINDOWS)
+  #define TAOS_EPOLL_WAIT_TIME 100
+  typedef SOCKET eventfd_t;
+  #define eventfd(a, b) -1
+  typedef SOCKET EpollFd;
+  #define EpollClose(pollFd) epoll_close(pollFd)
+  #ifndef EPOLLWAKEUP
+    #define EPOLLWAKEUP (1u << 29)
+  #endif
+#elif defined(_TD_DARWIN_64)
+  #define TAOS_EPOLL_WAIT_TIME 500
+  typedef int32_t SOCKET;
+  typedef SOCKET EpollFd;
+  #define EpollClose(pollFd) epoll_close(pollFd)
+#else
+  #define TAOS_EPOLL_WAIT_TIME 500
+  typedef int32_t SOCKET;
+  typedef SOCKET EpollFd;
+  #define EpollClose(pollFd) taosCloseSocket(pollFd)
+#endif
+
 #if defined(_TD_DARWIN_64)
 //  #define htobe64 htonll
 
@@ -64,12 +85,12 @@ extern "C" {
 #	define htole16(x) OSSwapHostToLittleInt16(x)
 #	define be16toh(x) OSSwapBigToHostInt16(x)
 #	define le16toh(x) OSSwapLittleToHostInt16(x)
- 
+
 #	define htobe32(x) OSSwapHostToBigInt32(x)
 #	define htole32(x) OSSwapHostToLittleInt32(x)
 #	define be32toh(x) OSSwapBigToHostInt32(x)
 #	define le32toh(x) OSSwapLittleToHostInt32(x)
- 
+
 #	define htobe64(x) OSSwapHostToBigInt64(x)
 #	define htole64(x) OSSwapHostToLittleInt64(x)
 #	define be64toh(x) OSSwapBigToHostInt64(x)
@@ -83,6 +104,17 @@ extern "C" {
 
 #define TAOS_EPOLL_WAIT_TIME 500
 
+typedef int32_t SocketFd;
+typedef SocketFd  EpollFd;
+
+typedef struct TdSocket {
+#if SOCKET_WITH_LOCK
+  TdThreadRwlock rwlock;
+#endif
+  int      refId;
+  SocketFd fd;
+} *TdSocketPtr, TdSocket;
+
 typedef struct TdSocketServer *TdSocketServerPtr;
 typedef struct TdSocket *TdSocketPtr;
 typedef struct TdEpoll *TdEpollPtr;
@@ -91,6 +123,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags,
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
 int32_t taosShutDownSocketRD(TdSocketPtr pSocket);

include/util/taoserror.h

@@ -274,18 +274,23 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_STREAM_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03F1)
 #define TSDB_CODE_MND_INVALID_STREAM_OPTION     TAOS_DEF_ERROR_CODE(0, 0x03F2)
 
+// mnode-sma
+#define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0400)
+#define TSDB_CODE_MND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0401)
+#define TSDB_CODE_MND_INVALID_SMA_OPTION        TAOS_DEF_ERROR_CODE(0, 0x0402)
+
 // dnode
-#define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0400)
-#define TSDB_CODE_DND_OFFLINE                   TAOS_DEF_ERROR_CODE(0, 0x0401)
-#define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0402)
-#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x0403)
-#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x0404)
-#define TSDB_CODE_NODE_PARSE_FILE_ERROR         TAOS_DEF_ERROR_CODE(0, 0x0405)
-#define TSDB_CODE_NODE_INVALID_OPTION           TAOS_DEF_ERROR_CODE(0, 0x0406)
-#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x0410)
-#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x0411)
-#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x0412)
-#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x0413)
+#define TSDB_CODE_DND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x04A0)
+#define TSDB_CODE_DND_OFFLINE                   TAOS_DEF_ERROR_CODE(0, 0x04A1)
+#define TSDB_CODE_DND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x04A2)
+#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x04A3)
+#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x04A4)
+#define TSDB_CODE_NODE_PARSE_FILE_ERROR         TAOS_DEF_ERROR_CODE(0, 0x04A5)
+#define TSDB_CODE_NODE_INVALID_OPTION           TAOS_DEF_ERROR_CODE(0, 0x04A6)
+#define TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED    TAOS_DEF_ERROR_CODE(0, 0x04A7)
+#define TSDB_CODE_DND_VNODE_NOT_DEPLOYED        TAOS_DEF_ERROR_CODE(0, 0x04A8)
+#define TSDB_CODE_DND_VNODE_INVALID_OPTION      TAOS_DEF_ERROR_CODE(0, 0x04A9)
+#define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES     TAOS_DEF_ERROR_CODE(0, 0x04AA)
 
 // vnode
 #define TSDB_CODE_VND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0500)
@@ -309,7 +314,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_IS_SYNCING                TAOS_DEF_ERROR_CODE(0, 0x0513)
 #define TSDB_CODE_VND_INVALID_TSDB_STATE        TAOS_DEF_ERROR_CODE(0, 0x0514)
 #define TSDB_CODE_VND_TB_NOT_EXIST              TAOS_DEF_ERROR_CODE(0, 0x0515)
-#define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0516)
+#define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
+#define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
@@ -336,8 +342,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_IVLD_TAG_VAL              TAOS_DEF_ERROR_CODE(0, 0x0615)
 #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0616)
 #define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x0617)
-#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x0618)
-#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x0619)
+#define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x0618)
+#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x0619)
+#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x0620)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)

source/client/inc/clientInt.h

@@ -141,18 +141,27 @@ typedef struct STscObj {
   SAppInstInfo*   pAppInfo;
 } STscObj;
 
+typedef struct SResultColumn {
+  union {
+    char*       nullbitmap;  // bitmap, one bit for each item in the list
+    int32_t*    offset;
+  };
+  char*         pData;
+} SResultColumn;
+
 typedef struct SReqResultInfo {
-  const char* pRspMsg;
-  const char* pData;
-  TAOS_FIELD* fields;
-  uint32_t    numOfCols;
-  int32_t*    length;
-  TAOS_ROW    row;
-  char**      pCol;
-  uint32_t    numOfRows;
-  uint64_t    totalRows;
-  uint32_t    current;
-  bool        completed;
+  const char*    pRspMsg;
+  const char*    pData;
+  TAOS_FIELD*    fields;
+  uint32_t       numOfCols;
+  int32_t*       length;
+  TAOS_ROW       row;
+  SResultColumn* pCol;
+  uint32_t       numOfRows;
+  uint64_t       totalRows;
+  uint32_t       current;
+  bool           completed;
+  int32_t        payloadLen;
 } SReqResultInfo;
 
 typedef struct SShowReqInfo {
@@ -227,7 +236,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
 
 void* doFetchRow(SRequestObj* pRequest);
 
-void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
+int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 

source/client/src/clientImpl.c

@@ -12,7 +12,7 @@
 static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
 static void          destroySendMsgInfo(SMsgSendInfo* pMsgBody);
-static void          setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
+static int32_t       setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
 
 static bool stringLengthCheck(const char* str, size_t maxsize) {
   if (str == NULL) {
@@ -558,13 +558,16 @@ void* doFetchRow(SRequestObj* pRequest) {
       }
 
       SReqResultInfo* pResInfo = &pRequest->body.resInfo;
-      int32_t         code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
-      if (code != TSDB_CODE_SUCCESS) {
-        pRequest->code = code;
+      pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData);
+      if (pRequest->code != TSDB_CODE_SUCCESS) {
+        return NULL;
+      }
+
+      pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
+      if (pRequest->code != TSDB_CODE_SUCCESS) {
         return NULL;
       }
 
-      setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData);
       tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
                pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId);
 
@@ -631,10 +634,23 @@ void* doFetchRow(SRequestObj* pRequest) {
 _return:
 
   for (int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
-    pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current;
+    SResultColumn* pCol = &pResultInfo->pCol[i];
+
     if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
-      pResultInfo->length[i] = varDataLen(pResultInfo->row[i]);
-      pResultInfo->row[i] = varDataVal(pResultInfo->row[i]);
+      if (pCol->offset[pResultInfo->current] != -1) {
+        char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData;
+
+        pResultInfo->length[i] = varDataLen(pStart);
+        pResultInfo->row[i] = varDataVal(pStart);
+      } else {
+        pResultInfo->row[i] = NULL;
+      }
+    } else {
+      if (!colDataIsNull_f(pCol->nullbitmap, pResultInfo->current)) {
+        pResultInfo->row[i] = pResultInfo->pCol[i].pData + pResultInfo->fields[i].bytes * pResultInfo->current;
+      } else {
+        pResultInfo->row[i] = NULL;
+      }
     }
   }
 
@@ -642,30 +658,52 @@ _return:
   return pResultInfo->row;
 }
 
-static void doPrepareResPtr(SReqResultInfo* pResInfo) {
+static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
   if (pResInfo->row == NULL) {
-    pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES);
-    pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES);
+    pResInfo->row    = calloc(pResInfo->numOfCols, POINTER_BYTES);
+    pResInfo->pCol   = calloc(pResInfo->numOfCols, sizeof(SResultColumn));
     pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t));
   }
+
+  if (pResInfo->row == NULL || pResInfo->pCol == NULL || pResInfo->length == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  } else {
+    return TSDB_CODE_SUCCESS;
+  }
 }
 
-void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
+int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
   assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
   if (numOfRows == 0) {
-    return;
+    return TSDB_CODE_SUCCESS;
   }
 
-  // todo check for the failure of malloc
-  doPrepareResPtr(pResultInfo);
+  int32_t code = doPrepareResPtr(pResultInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
 
-  int32_t offset = 0;
+  int32_t* colLength = (int32_t*)pResultInfo->pData;
+  char*    pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
   for (int32_t i = 0; i < numOfCols; ++i) {
+    colLength[i] = htonl(colLength[i]);
+
+    if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
+      pResultInfo->pCol[i].offset = (int32_t*)pStart;
+      pStart += numOfRows * sizeof(int32_t);
+    } else {
+      pResultInfo->pCol[i].nullbitmap = pStart;
+      pStart += BitmapLen(pResultInfo->numOfRows);
+    }
+
+    pResultInfo->pCol[i].pData = pStart;
     pResultInfo->length[i] = pResultInfo->fields[i].bytes;
-    pResultInfo->row[i] = (char*)(pResultInfo->pData + offset * pResultInfo->numOfRows);
-    pResultInfo->pCol[i] = pResultInfo->row[i];
-    offset += pResultInfo->fields[i].bytes;
+    pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+
+    pStart += colLength[i];
   }
+
+  return TSDB_CODE_SUCCESS;
 }
 
 char* getDbOfConnection(STscObj* pObj) {
@@ -687,15 +725,17 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
   taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
-void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
   assert(pResultInfo != NULL && pRsp != NULL);
 
-  pResultInfo->pRspMsg = (const char*)pRsp;
-  pResultInfo->pData = (void*)pRsp->data;
-  pResultInfo->numOfRows = htonl(pRsp->numOfRows);
-  pResultInfo->current = 0;
-  pResultInfo->completed = (pRsp->completed == 1);
+  pResultInfo->pRspMsg    = (const char*)pRsp;
+  pResultInfo->pData      = (void*)pRsp->data;
+  pResultInfo->numOfRows  = htonl(pRsp->numOfRows);
+  pResultInfo->current    = 0;
+  pResultInfo->completed  = (pRsp->completed == 1);
+  pResultInfo->payloadLen = htonl(pRsp->compLen);
 
+  // TODO handle the compressed case
   pResultInfo->totalRows += pResultInfo->numOfRows;
-  setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
+  return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows);
 }

source/client/src/tmq.c

@@ -471,8 +471,8 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
   }
   sqlLen = strlen(sql);
 
-  if (strlen(streamName) >= TSDB_TABLE_NAME_LEN) {
-    tscError("stream name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
+  if (strlen(tbName) >= TSDB_TABLE_NAME_LEN) {
+    tscError("output tb name too long, max length:%d", TSDB_TABLE_NAME_LEN - 1);
     terrno = TSDB_CODE_TSC_INVALID_INPUT;
     goto _return;
   }

source/client/test/clientTests.cpp

@@ -285,17 +285,17 @@ TEST(testCase, create_stable_Test) {
   ASSERT_EQ(numOfFields, 0);
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
-  }
-
-  pRes = taos_query(pConn, "use abc1");
-  taos_free_result(pRes);
-  pRes = taos_query(pConn, "drop stable `123_$^)`");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
-  }
+//  pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
+//  }
+//
+//  pRes = taos_query(pConn, "use abc1");
+//  taos_free_result(pRes);
+//  pRes = taos_query(pConn, "drop stable `123_$^)`");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
+//  }
 
   taos_close(pConn);
 }
@@ -335,7 +335,7 @@ TEST(testCase, create_ctable_Test) {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table tu using sts tags('2021-10-10 1:1:1');");
+  pRes = taos_query(pConn, "create table tu using st1 tags('2021-10-10 1:1:1');");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
   }
@@ -485,7 +485,9 @@ TEST(testCase, show_table_Test) {
 
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "show abc1.tables");
+  taos_query(pConn, "use abc1");
+
+  pRes = taos_query(pConn, "show tables");
   if (taos_errno(pRes) != 0) {
     printf("failed to show tables, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);
@@ -658,13 +660,7 @@ TEST(testCase, agg_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table tx using st1 tags(111111111111111)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create table, reason:%s\n", taos_errstr(pRes));
-  }
-  taos_free_result(pRes);
-
-  pRes = taos_query(pConn, "select count(*) from t_x_19");
+  pRes = taos_query(pConn, "select count(*), sum(k),min(k),max(k) from tu");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/src/tdatablock.c

@@ -497,11 +497,11 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
 
 /**
  *
- * +------------------+---------------+--------------------+
- * |the number of rows| column length |     column #1      |
- * |    (4 bytes)     |  (4 bytes)    |--------------------+
- * |                  |               | null bitmap| values|
- * +------------------+---------------+--------------------+
+ * +------------------+---------------------------------------------+
+ * |the number of rows|                    column #1                |
+ * |    (4 bytes)     |------------+-----------------------+--------+
+ * |                  | null bitmap| column length(4bytes) | values |
+ * +------------------+------------+-----------------------+--------+
  * @param buf
  * @param pBlock
  * @return
@@ -582,9 +582,10 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
   return TSDB_CODE_SUCCESS;
 }
 
-size_t blockDataGetRowSize(const SSDataBlock* pBlock) {
+size_t blockDataGetRowSize(SSDataBlock* pBlock) {
   ASSERT(pBlock != NULL);
-  size_t rowSize = 0;
+  if (pBlock->info.rowSize == 0) {
+    size_t rowSize = 0;
 
   size_t numOfCols = pBlock->info.numOfCols;
   for (int32_t i = 0; i < numOfCols; ++i) {
@@ -592,7 +593,10 @@ size_t blockDataGetRowSize(const SSDataBlock* pBlock) {
     rowSize += pColInfo->info.bytes;
   }
 
-  return rowSize;
+    pBlock->info.rowSize = rowSize;
+  }
+
+  return pBlock->info.rowSize;
 }
 
 /**
@@ -633,7 +637,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
     if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
       rowSize += sizeof(int32_t);
     } else {
-      rowSize += 1 / 8.0;
+      rowSize += 1/8.0;  // one bit for each record
     }
   }
 
@@ -1139,7 +1143,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
   return 0;
 }
 
-void blockDataClearup(SSDataBlock* pDataBlock) {
+void blockDataCleanup(SSDataBlock* pDataBlock) {
   pDataBlock->info.rows = 0;
 
   if (pDataBlock->info.hasVarCol) {
@@ -1300,3 +1304,26 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
   }
   return (void*)buf;
 }
+
+int32_t tEncodeDataBlocks(void** buf, const SArray* blocks) {
+  int32_t tlen = 0;
+  int32_t sz = taosArrayGetSize(blocks);
+  tlen += taosEncodeFixedI32(buf, sz);
+
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pBlock = taosArrayGet(blocks, i);
+    tlen += tEncodeDataBlock(buf, pBlock);
+  }
+
+  return tlen;
+}
+
+void* tDecodeDataBlocks(const void* buf, SArray* blocks) {
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock pBlock = {0};
+    buf = tDecodeDataBlock(buf, &pBlock);
+  }
+  return (void*)buf;
+}

source/common/src/tdataformat.c

@@ -421,6 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
   return NULL;
 }
 
+#if 0
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
   SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
   if (pRet == NULL) return NULL;
@@ -431,6 +432,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
   for (int i = 0; i < pDataCols->numOfCols; i++) {
     pRet->cols[i].type = pDataCols->cols[i].type;
+    pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
     pRet->cols[i].colId = pDataCols->cols[i].colId;
     pRet->cols[i].bytes = pDataCols->cols[i].bytes;
     pRet->cols[i].offset = pDataCols->cols[i].offset;
@@ -453,6 +455,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 
   return pRet;
 }
+#endif
 
 void tdResetDataCols(SDataCols *pCols) {
   if (pCols != NULL) {

source/common/src/tglobal.c

@@ -132,6 +132,9 @@ bool tsdbForceKeepFile = false;
 int32_t  tsDiskCfgNum = 0;
 SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0};
 
+// stream scheduler
+bool tsStreamSchedV = true;
+
 /*
  * minimum scale for whole system, millisecond by default
  * for TSDB_TIME_PRECISION_MILLI: 86400000L
@@ -585,4 +588,4 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
     taosResetLog();
     cfgDumpCfg(tsCfg, 1, false);
   }
-}
+}
\ No newline at end of file

source/common/src/tmsg.c

@@ -290,7 +290,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU32(buf, pReq->ttl);
   tlen += taosEncodeFixedU32(buf, pReq->keep);
-  tlen += taosEncodeFixedU8(buf, pReq->type);
+  tlen += taosEncodeFixedU8(buf, pReq->info);
 
   switch (pReq->type) {
     case TD_SUPER_TABLE:
@@ -309,6 +309,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
       }
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
+        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
+      }
+      if(pReq->rollup && pReq->stbCfg.pRSmaParam) {
+        SRSmaParam *param = pReq->stbCfg.pRSmaParam;
+        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
+        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeFixedI8(buf, param->nFuncIds);
+        for (int8_t i = 0; i < param->nFuncIds; ++i) {
+          tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
+        }
+        tlen += taosEncodeFixedI64(buf, param->delay);
+      }
       break;
     case TD_CHILD_TABLE:
       tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid);
@@ -322,6 +336,20 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
       }
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
+        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]);
+      }
+      if(pReq->rollup && pReq->ntbCfg.pRSmaParam) {
+        SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
+        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
+        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeFixedI8(buf, param->nFuncIds);
+        for (int8_t i = 0; i < param->nFuncIds; ++i) {
+          tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
+        }
+        tlen += taosEncodeFixedI64(buf, param->delay);
+      }
       break;
     default:
       ASSERT(0);
@@ -335,7 +363,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   buf = taosDecodeString(buf, &(pReq->name));
   buf = taosDecodeFixedU32(buf, &(pReq->ttl));
   buf = taosDecodeFixedU32(buf, &(pReq->keep));
-  buf = taosDecodeFixedU8(buf, &(pReq->type));
+  buf = taosDecodeFixedU8(buf, &(pReq->info));
 
   switch (pReq->type) {
     case TD_SUPER_TABLE:
@@ -356,6 +384,32 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
+      if (pReq->stbCfg.nBSmaCols > 0) {
+        pReq->stbCfg.pBSmaCols = (col_id_t *)malloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
+        for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
+          buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
+        }
+      } else {
+        pReq->stbCfg.pBSmaCols = NULL;
+      }
+      if (pReq->rollup) {
+        pReq->stbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam));
+        SRSmaParam *param = pReq->stbCfg.pRSmaParam;
+        buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
+        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeFixedI8(buf, &param->nFuncIds);
+        if (param->nFuncIds > 0) {
+          for (int8_t i = 0; i < param->nFuncIds; ++i) {
+            buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
+          }
+        } else {
+          param->pFuncIds = NULL;
+        }
+        buf = taosDecodeFixedI64(buf, &param->delay);
+      } else {
+        pReq->stbCfg.pRSmaParam = NULL;
+      }
       break;
     case TD_CHILD_TABLE:
       buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid);
@@ -370,6 +424,32 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
       }
+      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
+      if(pReq->ntbCfg.nBSmaCols > 0) {
+        pReq->ntbCfg.pBSmaCols = (col_id_t *)malloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t));
+        for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
+          buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i);
+        }
+      } else {
+        pReq->ntbCfg.pBSmaCols = NULL;
+      }
+      if(pReq->rollup) {
+        pReq->ntbCfg.pRSmaParam = (SRSmaParam *)malloc(sizeof(SRSmaParam));
+        SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
+        buf = taosDecodeFixedU32(buf, (uint32_t*)&param->xFilesFactor);
+        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeFixedI8(buf, &param->nFuncIds);
+        if (param->nFuncIds > 0) {
+          for (int8_t i = 0; i < param->nFuncIds; ++i) {
+            buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
+          }
+        } else {
+          param->pFuncIds = NULL;
+        }
+        buf = taosDecodeFixedI64(buf, &param->delay);
+      } else {
+        pReq->ntbCfg.pRSmaParam = NULL;
+      }
       break;
     default:
       ASSERT(0);
@@ -428,8 +508,12 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->delay) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
 
   for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
     SField *pField = taosArrayGet(pReq->pColumns, i);
@@ -445,7 +529,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
-  if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1;
+  if (tEncodeBinary(&encoder, pReq->comment, pReq->commentLen) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -460,8 +544,12 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
 
   pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
   pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
@@ -492,6 +580,12 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     }
   }
 
+  if (pReq->commentLen > 0) {
+    pReq->comment = malloc(pReq->commentLen);
+    if (pReq->comment == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
+  }
+
   if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
   tEndDecode(&decoder);
 
@@ -589,6 +683,123 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) {
   pReq->pFields = NULL;
 }
 
+int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->intervalUnit) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->slidingUnit) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->timezone) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dstVgId) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->interval) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->offset) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->sliding) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->exprLen) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->tagsFilterLen) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->astLen) < 0) return -1;
+  if (pReq->exprLen > 0) {
+    if (tEncodeBinary(&encoder, pReq->expr, pReq->exprLen) < 0) return -1;
+  }
+  if (pReq->tagsFilterLen > 0) {
+    if (tEncodeBinary(&encoder, pReq->tagsFilter, pReq->tagsFilterLen) < 0) return -1;
+  }
+  if (pReq->sqlLen > 0) {
+    if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1;
+  }
+  if (pReq->astLen > 0) {
+    if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1;
+  }
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->intervalUnit) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->slidingUnit) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->timezone) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dstVgId) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->interval) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->offset) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->sliding) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->exprLen) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->tagsFilterLen) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->astLen) < 0) return -1;
+  if (pReq->exprLen > 0) {
+    pReq->expr = malloc(pReq->exprLen);
+    if (pReq->expr == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->expr) < 0) return -1;
+  }
+  if (pReq->tagsFilterLen > 0) {
+    pReq->tagsFilter = malloc(pReq->tagsFilterLen);
+    if (pReq->tagsFilter == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->tagsFilter) < 0) return -1;
+  }
+  if (pReq->sqlLen > 0) {
+    pReq->sql = malloc(pReq->sqlLen);
+    if (pReq->sql == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1;
+  }
+  if (pReq->astLen > 0) {
+    pReq->ast = malloc(pReq->astLen);
+    if (pReq->ast == NULL) return -1;
+    if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) {
+  tfree(pReq->expr);
+  tfree(pReq->tagsFilter);
+  tfree(pReq->sql);
+  tfree(pReq->ast);
+}
+
+int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tCoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -2627,7 +2838,7 @@ int32_t tSerializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pRsp
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI32(&encoder, pRsp->code) < 0) return -1;    
+  if (tEncodeI32(&encoder, pRsp->code) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -2654,13 +2865,13 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR
   if (tStartEncode(&encoder) < 0) return -1;
   if (pRsp->rspList) {
     int32_t num = taosArrayGetSize(pRsp->rspList);
-    if (tEncodeI32(&encoder, num) < 0) return -1;    
+    if (tEncodeI32(&encoder, num) < 0) return -1;
     for (int32_t i = 0; i < num; ++i) {
       SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
-      if (tEncodeI32(&encoder, rsp->code) < 0) return -1;    
+      if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
     }
   } else {
-    if (tEncodeI32(&encoder, 0) < 0) return -1;    
+    if (tEncodeI32(&encoder, 0) < 0) return -1;
   }
   tEndEncode(&encoder);
 
@@ -2670,7 +2881,7 @@ int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchR
 }
 
 int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
-  SCoder decoder = {0};
+  SCoder  decoder = {0};
   int32_t num = 0;
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
@@ -2693,7 +2904,6 @@ int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatc
   return 0;
 }
 
-
 int32_t tSerializeSVCreateTSmaReq(void **buf, SVCreateTSmaReq *pReq) {
   int32_t tlen = 0;
 
@@ -2742,7 +2952,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->outputTbName) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
+  if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
+  if (tEncodeI32(&encoder, astLen) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 
@@ -2790,30 +3001,36 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
 }
 
 int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) {
-  if (tStartEncode(pEncoder) < 0) return -1;
+  /*if (tStartEncode(pEncoder) < 0) return -1;*/
   if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
   if (tEncodeI32(pEncoder, pTask->level) < 0) return -1;
   if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->pipeEnd) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->parallel) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->pipeSource) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->pipeSink) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->parallelizable) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->nextOpDst) < 0) return -1;
+  // if (tEncodeI8(pEncoder, pTask->numOfRunners) < 0) return -1;
   if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1;
   if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1;
-  tEndEncode(pEncoder);
+  /*tEndEncode(pEncoder);*/
   return pEncoder->pos;
 }
 
 int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+  /*if (tStartDecode(pDecoder) < 0) return -1;*/
   if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1;
   if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->pipeEnd) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->parallel) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->pipeSource) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->pipeSink) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->parallelizable) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->nextOpDst) < 0) return -1;
+  // if (tDecodeI8(pDecoder, &pTask->numOfRunners) < 0) return -1;
   if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1;
-  tEndDecode(pDecoder);
+  /*tEndDecode(pDecoder);*/
   return 0;
 }
 

source/common/src/trow.c

@@ -78,7 +78,7 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit
     setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
     pCol->len += TYPE_BYTES[pCol->type];
   }
-  if(setBitmap) {
+  if (setBitmap) {
     tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
   }
 }
@@ -118,8 +118,8 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) {
 
 #endif
 
-STSRow* tdRowDup(STSRow *row) {
-  STSRow* trow = malloc(TD_ROW_LEN(row));
+STSRow *tdRowDup(STSRow *row) {
+  STSRow *trow = malloc(TD_ROW_LEN(row));
   if (trow == NULL) return NULL;
 
   tdRowCpy(trow, row);
@@ -176,7 +176,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
 
   SDataCol *pDataCol = &(pCols->cols[0]);
   if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts,  pCols->numOfRows, pCols->maxPoints);
+    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints);
   }
 
   while (dcol < pCols->numOfCols) {
@@ -378,9 +378,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
   }
 }
 
-
-
-STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) {
+STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) {
 #if 0
   ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2));
   ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1));
@@ -473,6 +471,44 @@ STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema
   }
   taosArrayDestroy(stashRow);
   return buffer;
-  #endif
+#endif
   return NULL;
 }
+
+SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
+  SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints);
+  if (pRet == NULL) return NULL;
+
+  pRet->numOfCols = pDataCols->numOfCols;
+  pRet->sversion = pDataCols->sversion;
+  if (keepData) pRet->numOfRows = pDataCols->numOfRows;
+
+  for (int i = 0; i < pDataCols->numOfCols; i++) {
+    pRet->cols[i].type = pDataCols->cols[i].type;
+    pRet->cols[i].bitmap = pDataCols->cols[i].bitmap;
+    pRet->cols[i].colId = pDataCols->cols[i].colId;
+    pRet->cols[i].bytes = pDataCols->cols[i].bytes;
+    pRet->cols[i].offset = pDataCols->cols[i].offset;
+
+    if (keepData) {
+      if (pDataCols->cols[i].len > 0) {
+        if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) {
+          tdFreeDataCols(pRet);
+          return NULL;
+        }
+        pRet->cols[i].len = pDataCols->cols[i].len;
+        memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
+        if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
+          int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints;
+          memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
+        }
+        if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
+          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints);
+          memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
+        }
+      }
+    }
+  }
+
+  return pRet;
+}
\ No newline at end of file

source/dnode/mgmt/mnode/src/mmMsg.c

@@ -123,6 +123,8 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, (NodeMsgFp)mmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, 0);
@@ -142,6 +144,8 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, (NodeMsgFp)mmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
 
   // Requests handled by VNODE
   dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
@@ -149,5 +153,6 @@ void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, (NodeMsgFp)mmProcessWriteMsg, 0);
 }

source/dnode/mgmt/vnode/src/vmInt.c

@@ -343,7 +343,7 @@ void vmGetMgmtFp(SMgmtWrapper *pWrapper) {
   mgmtFp.requiredFp = vmRequire;
 
   vmInitMsgHandles(pWrapper);
-  pWrapper->name = "vnodes";
+  pWrapper->name = "vnode";
   pWrapper->fp = mgmtFp;
 }
 

source/dnode/mgmt/vnode/src/vmMsg.c

@@ -268,14 +268,19 @@ void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
   dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, 0);
+  dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, 0);
 
   dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);
   dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0);

source/dnode/mnode/impl/inc/mndDef.h

@@ -103,6 +103,8 @@ typedef enum {
   TRN_TYPE_CREATE_STB = 4001,
   TRN_TYPE_ALTER_STB = 4002,
   TRN_TYPE_DROP_STB = 4003,
+  TRN_TYPE_CREATE_SMA = 4004,
+  TRN_TYPE_DROP_SMA = 4005,
   TRN_TYPE_STB_SCOPE_END,
 } ETrnType;
 
@@ -305,6 +307,31 @@ typedef struct {
   SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
 } SVgObj;
 
+typedef struct {
+  char    name[TSDB_TABLE_FNAME_LEN];
+  char    stb[TSDB_TABLE_FNAME_LEN];
+  char    db[TSDB_DB_FNAME_LEN];
+  int64_t createdTime;
+  int64_t uid;
+  int64_t stbUid;
+  int64_t dbUid;
+  int8_t  intervalUnit;
+  int8_t  slidingUnit;
+  int8_t  timezone;
+  int32_t dstVgId;  // for stream
+  int64_t interval;
+  int64_t offset;
+  int64_t sliding;
+  int32_t exprLen;  // strlen + 1
+  int32_t tagsFilterLen;
+  int32_t sqlLen;
+  int32_t astLen;
+  char*   expr;
+  char*   tagsFilter;
+  char*   sql;
+  char*   ast;
+} SSmaObj;
+
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   char     db[TSDB_DB_FNAME_LEN];
@@ -314,12 +341,16 @@ typedef struct {
   int64_t  dbUid;
   int32_t  version;
   int32_t  nextColId;
+  float    xFilesFactor;
+  int32_t  aggregationMethod;
+  int32_t  delay;
   int32_t  numOfColumns;
   int32_t  numOfTags;
+  int32_t  commentLen;
   SSchema* pColumns;
   SSchema* pTags;
+  char*    comment;
   SRWLatch lock;
-  char     comment[TSDB_STB_COMMENT_LEN];
 } SStbObj;
 
 typedef struct {
@@ -697,12 +728,12 @@ typedef struct {
   char*   logicalPlan;
   char*   physicalPlan;
   SArray* tasks;  // SArray<SArray<SStreamTask>>
+  SArray* outputName;
 } SStreamObj;
 
 int32_t tEncodeSStreamObj(SCoder* pEncoder, const SStreamObj* pObj);
 int32_t tDecodeSStreamObj(SCoder* pDecoder, SStreamObj* pObj);
 
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndInt.h

@@ -122,9 +122,9 @@ typedef struct SMnode {
   SMsgCb            msgCb;
 } SMnode;
 
-void     mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
-uint64_t mndGenerateUid(char *name, int32_t len);
-void     mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
+void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
+int64_t mndGenerateUid(char *name, int32_t len);
+void    mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/inc/tq.h → source/dnode/mnode/impl/inc/mndSma.h

@@ -13,54 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TQ_H_
-#define _TQ_H_
+#ifndef _TD_MND_SMA_H_
+#define _TD_MND_SMA_H_
 
-#include "executor.h"
-#include "meta.h"
-#include "taoserror.h"
-#include "tcommon.h"
-#include "tmallocator.h"
-#include "tmsg.h"
-#include "trpc.h"
-#include "ttimer.h"
-#include "tutil.h"
-#include "vnode.h"
-#include "wal.h"
+#include "mndInt.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct STQ STQ;
-
-// memory allocator provided by vnode
-typedef struct {
-  SMemAllocatorFactory* pAllocatorFactory;
-  SMemAllocator*        pAllocator;
-} STqMemRef;
-
-// init once
-int  tqInit();
-void tqCleanUp();
-
-// open in each vnode
-STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac);
-void tqClose(STQ*);
-
-// required by vnode
-int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version);
-int tqCommit(STQ*);
-
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
-int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-int32_t tqProcessRebReq(STQ* pTq, char* msg);
-int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg);
-
-int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
+int32_t  mndInitSma(SMnode *pMnode);
+void     mndCleanupSma(SMnode *pMnode);
+SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName);
+void     mndReleaseSma(SMnode *pMnode, SSmaObj *pSma);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TQ_H_*/
+#endif /*_TD_MND_SMA_H_*/

source/dnode/mnode/impl/inc/mndSnode.h

@@ -24,6 +24,7 @@ extern "C" {
 
 int32_t mndInitSnode(SMnode *pMnode);
 void    mndCleanupSnode(SMnode *pMnode);
+SEpSet  mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndStream.h

@@ -31,6 +31,8 @@ void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndDef.c

@@ -16,6 +16,7 @@
 #include "mndDef.h"
 
 int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
+  int32_t outputNameSz = 0;
   if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
   if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
@@ -43,6 +44,15 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   } else {
     tEncodeI32(pEncoder, 0);
   }
+
+  if (pObj->outputName != NULL) {
+    outputNameSz = taosArrayGetSize(pObj->outputName);
+  }
+  if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1;
+  for (int32_t i = 0; i < outputNameSz; i++) {
+    char *name = taosArrayGetP(pObj->outputName, i);
+    if (tEncodeCStr(pEncoder, name) < 0) return -1;
+  }
   return pEncoder->pos;
 }
 
@@ -76,5 +86,16 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   } else {
     pObj->tasks = NULL;
   }
+  int32_t outputNameSz;
+  if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1;
+  pObj->outputName = taosArrayInit(outputNameSz, sizeof(void *));
+  if (pObj->outputName == NULL) {
+    return -1;
+  }
+  for (int32_t i = 0; i < outputNameSz; i++) {
+    char *name;
+    if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1;
+    taosArrayPush(pObj->outputName, &name);
+  }
   return 0;
 }

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -16,10 +16,13 @@
 #define _DEFAULT_SOURCE
 #include "mndInfoSchema.h"
 
+#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+#define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
+
 static const SInfosTableSchema dnodesSchema[] = {{.name = "id",             .bytes = 2,   .type = TSDB_DATA_TYPE_SMALLINT},
                                                  {.name = "endpoint",       .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
                                                  {.name = "vnodes",         .bytes = 2,   .type = TSDB_DATA_TYPE_SMALLINT},
-                                                 {.name = "max_vnodes",.bytes = 2,   .type = TSDB_DATA_TYPE_SMALLINT},
+                                                 {.name = "max_vnodes",     .bytes = 2,   .type = TSDB_DATA_TYPE_SMALLINT},
                                                  {.name = "status",         .bytes = 10 + VARSTR_HEADER_SIZE,  .type = TSDB_DATA_TYPE_BINARY},
                                                  {.name = "create_time",    .bytes = 8,   .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                  {.name = "note",           .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
@@ -38,7 +41,7 @@ static const SInfosTableSchema qnodesSchema[] = {{.name = "id",             .byt
                                                  {.name = "endpoint",       .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
                                                  {.name = "create_time",    .bytes = 8,   .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                 };
-static const SInfosTableSchema userDBSchema[] = {{.name = "name",             .bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userDBSchema[] = {{.name = "name",             .bytes = SYSTABLE_SCH_DB_NAME_LEN,   .type = TSDB_DATA_TYPE_BINARY},
                                                  {.name = "create_time",      .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                  {.name = "vgroups",          .bytes = 2,    .type = TSDB_DATA_TYPE_SMALLINT},
                                                  {.name = "ntables",          .bytes = 8,    .type = TSDB_DATA_TYPE_BIGINT},
@@ -58,21 +61,21 @@ static const SInfosTableSchema userDBSchema[] = {{.name = "name",             .b
 //                                                 {.name = "update",           .bytes = 1,   .type = TSDB_DATA_TYPE_TINYINT},  // disable update
                                                 };
 static const SInfosTableSchema userFuncSchema[] = {{.name = "name",           .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
-                                                   {.name = "create_time",   .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
+                                                   {.name = "create_time",    .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                    {.name = "ntables",        .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
                                                    {.name = "precision",      .bytes = 2,    .type = TSDB_DATA_TYPE_BINARY},
                                                    {.name = "status",         .bytes = 10,   .type = TSDB_DATA_TYPE_BINARY},
                                                   };
 static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name",          .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
-                                                  {.name = "table_name",       .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
+                                                  {.name = "table_name",       .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "index_database",   .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
-                                                  {.name = "index_name",       .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
+                                                  {.name = "index_name",       .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "column_name",      .bytes = 64,   .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "index_type",       .bytes = 10,   .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "index_extensions", .bytes = 256,  .type = TSDB_DATA_TYPE_BINARY},
                                                  };
-static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name",         .bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE,   .type = TSDB_DATA_TYPE_BINARY},
-                                                   {.name = "stable_name",     .bytes = (TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE,  .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name",     .bytes = SYSTABLE_SCH_TABLE_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
+                                                   {.name = "db_name",         .bytes = SYSTABLE_SCH_DB_NAME_LEN,   .type = TSDB_DATA_TYPE_BINARY},
                                                    {.name = "create_time",     .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                    {.name = "columns",         .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
                                                    {.name = "tags",            .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
@@ -80,25 +83,25 @@ static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name",         .
                                                    {.name = "last_update",     .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                    {.name = "table_comment",   .bytes = 1024 + VARSTR_HEADER_SIZE,    .type = TSDB_DATA_TYPE_INT},
                                                   };
-static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name",  .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name",  .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
                                                       {.name = "user_name",    .bytes = 23,   .type = TSDB_DATA_TYPE_BINARY},
-                                                      {.name = "dest_table",   .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
-                                                      {.name = "create_time", .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
+                                                      {.name = "dest_table",   .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
+                                                      {.name = "create_time",  .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                       {.name = "sql",          .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
                                                      };
 static const SInfosTableSchema userTblsSchema[] = {
-                                                   {.name = "table_name",      .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
-                                                   {.name = "db_name",         .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
-                                                   {.name = "create_time",    .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
+                                                   {.name = "table_name",      .bytes = SYSTABLE_SCH_TABLE_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
+                                                   {.name = "db_name",         .bytes = SYSTABLE_SCH_DB_NAME_LEN,   .type = TSDB_DATA_TYPE_BINARY},
+                                                   {.name = "create_time",     .bytes = 8,    .type = TSDB_DATA_TYPE_TIMESTAMP},
                                                    {.name = "columns",         .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
-                                                   {.name = "stable_name",     .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
+                                                   {.name = "stable_name",     .bytes = SYSTABLE_SCH_TABLE_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
                                                    {.name = "uid",             .bytes = 8,    .type = TSDB_DATA_TYPE_BIGINT},
-                                                   {.name = "vgroup_id",           .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
+                                                   {.name = "vgroup_id",       .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
                                                    {.name = "ttl",             .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
-                                                   {.name = "table_comment",           .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
+                                                   {.name = "table_comment",    .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
                                                   };
 static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name",                .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
-                                                      {.name = "table_name",             .bytes = 192,  .type = TSDB_DATA_TYPE_BINARY},
+                                                      {.name = "table_name",             .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
                                                       {.name = "distributed_histogram",  .bytes = 500,  .type = TSDB_DATA_TYPE_BINARY},
                                                       {.name = "min_of_rows",            .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
                                                       {.name = "max_of_rows",            .bytes = 4,    .type = TSDB_DATA_TYPE_INT},
@@ -117,7 +120,7 @@ static const SInfosTableSchema userUsersSchema[] = {{.name = "name",           .
                                                     {.name = "account",        .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE,    .type = TSDB_DATA_TYPE_BINARY},
                                                    };
 static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id",        .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
-                                                  {.name = "db_name",          .bytes = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE,   .type = TSDB_DATA_TYPE_BINARY},
+                                                  {.name = "db_name",          .bytes = SYSTABLE_SCH_DB_NAME_LEN,   .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "tables",           .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
                                                   {.name = "status",           .bytes = 12 + VARSTR_HEADER_SIZE,  .type = TSDB_DATA_TYPE_BINARY},
                                                   {.name = "onlines",          .bytes = 4,   .type = TSDB_DATA_TYPE_INT},

source/dnode/mnode/impl/src/mndScheduler.c

@@ -20,6 +20,7 @@
 #include "mndMnode.h"
 #include "mndOffset.h"
 #include "mndShow.h"
+#include "mndSnode.h"
 #include "mndStb.h"
 #include "mndStream.h"
 #include "mndSubscribe.h"
@@ -31,20 +32,23 @@
 #include "tname.h"
 #include "tuuid.h"
 
-int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet) {
+extern bool tsStreamSchedV;
+
+int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet* pEpSet, tmsg_t type, int32_t nodeId) {
   SCoder encoder;
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
   tEncodeSStreamTask(&encoder, pTask);
-  int32_t tlen = sizeof(SMsgHead) + encoder.pos;
+  int32_t size = encoder.pos;
+  int32_t tlen = sizeof(SMsgHead) + size;
   tCoderClear(&encoder);
   void* buf = malloc(tlen);
   if (buf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-  ((SMsgHead*)buf)->streamTaskId = pTask->taskId;
+  ((SMsgHead*)buf)->streamTaskId = htonl(nodeId);
   void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, tlen, TD_ENCODER);
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, abuf, size, TD_ENCODER);
   tEncodeSStreamTask(&encoder, pTask);
   tCoderClear(&encoder);
 
@@ -52,7 +56,7 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet
   memcpy(&action.epSet, pEpSet, sizeof(SEpSet));
   action.pCont = buf;
   action.contLen = tlen;
-  action.msgType = TDMT_SND_TASK_DEPLOY;
+  action.msgType = type;
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     rpcFreeCont(buf);
     return -1;
@@ -69,12 +73,27 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SS
     terrno = TSDB_CODE_QRY_INVALID_INPUT;
     return -1;
   }
-  mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet);
+  mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
   return 0;
 }
 
+SSnodeObj* mndSchedFetchSnode(SMnode* pMnode) {
+  SSnodeObj* pObj = NULL;
+  pObj = sdbFetch(pMnode->pSdb, SDB_SNODE, NULL, (void**)&pObj);
+  return pObj;
+}
+
 int32_t mndAssignTaskToSnode(SMnode* pMnode, STrans* pTrans, SStreamTask* pTask, SSubplan* plan,
                              const SSnodeObj* pSnode) {
+  int32_t msgLen;
+  plan->execNode.nodeId = pSnode->id;
+  plan->execNode.epSet = mndAcquireEpFromSnode(pMnode, pSnode);
+
+  if (qSubPlanToString(plan, &pTask->qmsg, &msgLen) < 0) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return -1;
+  }
+  mndPersistTaskDeployReq(pTrans, pTask, &plan->execNode.epSet, TDMT_SND_TASK_DEPLOY, 0);
   return 0;
 }
 
@@ -90,6 +109,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 
   int32_t totLevel = LIST_LENGTH(pPlan->pSubplans);
   pStream->tasks = taosArrayInit(totLevel, sizeof(SArray));
+  int32_t lastUsedVgId = 0;
 
   for (int32_t level = 0; level < totLevel; level++) {
     SArray*        taskOneLevel = taosArrayInit(0, sizeof(SStreamTask));
@@ -97,9 +117,9 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
     int32_t        opNum = LIST_LENGTH(inner->pNodeList);
     ASSERT(opNum == 1);
 
-    SSubplan* plan = nodesListGetNode(inner->pNodeList, level);
+    SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
     if (level == 0) {
-      ASSERT(plan->type == SUBPLAN_TYPE_SCAN);
+      ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN);
       void* pIter = NULL;
       while (1) {
         pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@@ -109,12 +129,15 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
           continue;
         }
 
+        lastUsedVgId = pVgroup->vgId;
         pStream->vgNum++;
         // send to vnode
 
         SStreamTask* pTask = streamTaskNew(pStream->uid, level);
+        pTask->pipeSource = 1;
+        pTask->pipeSink = level == totLevel - 1 ? 1 : 0;
+        pTask->parallelizable = 1;
         // TODO: set to
-        pTask->parallel = 4;
         if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
           sdbRelease(pSdb, pVgroup);
           qDestroyQueryPlan(pPlan);
@@ -122,35 +145,37 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         }
         taosArrayPush(taskOneLevel, pTask);
       }
-
-    } else if (plan->subplanType == SUBPLAN_TYPE_SCAN) {
-      // duplicatable
-
-      int32_t parallel = 0;
-      // if no snode, parallel set to fetch thread num in vnode
-
-      // if has snode, set to shared thread num in snode
-      parallel = SND_SHARED_THREAD_NUM;
-
-      SStreamTask* pTask = streamTaskNew(pStream->uid, level);
-      pTask->parallel = parallel;
-      // TODO:get snode id and ep
-      if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
-        sdbRelease(pSdb, pVgroup);
-        qDestroyQueryPlan(pPlan);
-        return -1;
-      }
-      taosArrayPush(taskOneLevel, pTask);
     } else {
-      // not duplicatable
       SStreamTask* pTask = streamTaskNew(pStream->uid, level);
-
-      // TODO: get snode
-      if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
-        sdbRelease(pSdb, pVgroup);
-        qDestroyQueryPlan(pPlan);
-        return -1;
+      pTask->pipeSource = 0;
+      pTask->pipeSink = level == totLevel - 1 ? 1 : 0;
+      pTask->parallelizable = plan->subplanType == SUBPLAN_TYPE_SCAN;
+      pTask->nextOpDst = STREAM_NEXT_OP_DST__VND;
+
+      if (tsStreamSchedV) {
+        ASSERT(lastUsedVgId != 0);
+        SVgObj* pVg = mndAcquireVgroup(pMnode, lastUsedVgId);
+        if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVg) < 0) {
+          sdbRelease(pSdb, pVg);
+          qDestroyQueryPlan(pPlan);
+          return -1;
+        }
+        sdbRelease(pSdb, pVg);
+      } else {
+        SSnodeObj* pSnode = mndSchedFetchSnode(pMnode);
+        if (pSnode != NULL) {
+          if (mndAssignTaskToSnode(pMnode, pTrans, pTask, plan, pSnode) < 0) {
+            sdbRelease(pSdb, pSnode);
+            qDestroyQueryPlan(pPlan);
+            return -1;
+          }
+          sdbRelease(pMnode->pSdb, pSnode);
+        } else {
+          // TODO: assign to one vg
+          ASSERT(0);
+        }
       }
+
       taosArrayPush(taskOneLevel, pTask);
     }
     taosArrayPush(pStream->tasks, taskOneLevel);

source/dnode/mnode/impl/src/mndShow.c

@@ -427,6 +427,8 @@ char *mndShowStr(int32_t showType) {
       return "show topics";
     case TSDB_MGMT_TABLE_FUNC:
       return "show functions";
+      case TSDB_MGMT_TABLE_INDEX:
+      return "show indexes";
     default:
       return "undefined";
   }

source/dnode/mnode/impl/src/mndSnode.c

@@ -60,6 +60,15 @@ int32_t mndInitSnode(SMnode *pMnode) {
 
 void mndCleanupSnode(SMnode *pMnode) {}
 
+SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode) {
+  SEpSet epSet;
+  memcpy(epSet.eps->fqdn, pSnode->pDnode->fqdn, 128);
+  epSet.eps->port = pSnode->pDnode->port;
+  epSet.numOfEps = 1;
+  epSet.inUse = 0;
+  return epSet;
+}
+
 static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) {
   SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId);
   if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {

source/dnode/mnode/impl/src/mndStb.c

@@ -13,20 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
 #include "mndStb.h"
 #include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
+#include "mndInfoSchema.h"
 #include "mndMnode.h"
 #include "mndShow.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
-#include "mndInfoSchema.h"
 #include "tname.h"
 
-#define TSDB_STB_VER_NUMBER 1
+#define TSDB_STB_VER_NUMBER   1
 #define TSDB_STB_RESERVE_SIZE 64
 
 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
@@ -86,8 +85,12 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, STB_ENCODE_OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->version, STB_ENCODE_OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, STB_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), STB_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, STB_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pStb->delay, STB_ENCODE_OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, STB_ENCODE_OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, STB_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, STB_ENCODE_OVER)
 
   for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
     SSchema *pSchema = &pStb->pColumns[i];
@@ -105,7 +108,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
     SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_ENCODE_OVER)
   }
 
-  SDB_SET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_ENCODE_OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_ENCODE_OVER)
   SDB_SET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_ENCODE_OVER)
   SDB_SET_DATALEN(pRaw, dataPos, STB_ENCODE_OVER)
 
@@ -148,8 +151,14 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, STB_DECODE_OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->version, STB_DECODE_OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, STB_DECODE_OVER)
+  int32_t xFilesFactor = 0;
+  SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, STB_DECODE_OVER)
+  pStb->xFilesFactor = xFilesFactor / 10000.0f;
+  SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, STB_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pStb->delay, STB_DECODE_OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, STB_DECODE_OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, STB_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, STB_DECODE_OVER)
 
   pStb->pColumns = calloc(pStb->numOfColumns, sizeof(SSchema));
   pStb->pTags = calloc(pStb->numOfTags, sizeof(SSchema));
@@ -173,7 +182,11 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
     SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, STB_DECODE_OVER)
   }
 
-  SDB_GET_BINARY(pRaw, dataPos, pStb->comment, TSDB_STB_COMMENT_LEN, STB_DECODE_OVER)
+  if (pStb->commentLen > 0) {
+    pStb->comment = calloc(pStb->commentLen, 1);
+    if (pStb->comment == NULL) goto STB_DECODE_OVER;
+    SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, STB_DECODE_OVER)
+  }
   SDB_GET_RESERVE(pRaw, dataPos, TSDB_STB_RESERVE_SIZE, STB_DECODE_OVER)
 
   terrno = 0;
@@ -183,6 +196,7 @@ STB_DECODE_OVER:
     mError("stb:%s, failed to decode from raw:%p since %s", pStb->name, pRaw, terrstr());
     tfree(pStb->pColumns);
     tfree(pStb->pTags);
+    tfree(pStb->comment);
     tfree(pRow);
     return NULL;
   }
@@ -200,6 +214,7 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
   mTrace("stb:%s, perform delete action, row:%p", pStb->name, pStb);
   tfree(pStb->pColumns);
   tfree(pStb->pTags);
+  tfree(pStb->comment);
   return 0;
 }
 
@@ -502,6 +517,13 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
   stbObj.nextColId = 1;
   stbObj.numOfColumns = pCreate->numOfColumns;
   stbObj.numOfTags = pCreate->numOfTags;
+  stbObj.commentLen = pCreate->commentLen;
+  stbObj.comment = calloc(stbObj.commentLen, 1);
+  if (stbObj.comment == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  memcpy(stbObj.comment, pCreate->comment, stbObj.commentLen);
 
   stbObj.pColumns = malloc(stbObj.numOfColumns * sizeof(SSchema));
   stbObj.pTags = malloc(stbObj.numOfTags * sizeof(SSchema));
@@ -1162,7 +1184,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
 
 static int32_t mndDropStb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) {
   int32_t code = -1;
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK,TRN_TYPE_DROP_STB, &pReq->rpcMsg);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_DROP_STB, &pReq->rpcMsg);
   if (pTrans == NULL) goto DROP_STB_OVER;
 
   mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
@@ -1534,20 +1556,19 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
     cols = 0;
 
     SName name = {0};
+    char stbName[TSDB_TABLE_NAME_LEN] = {0};
+    mndExtractTableName(pStb->name, stbName);
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, stbName);
+    cols++;
+
     char  db[TSDB_DB_NAME_LEN] = {0};
     tNameFromString(&name, pStb->db, T_NAME_ACCT|T_NAME_DB);
     tNameGetDbName(&name, db);
-
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     STR_TO_VARSTR(pWrite, db);
     cols++;
 
-    char stbName[TSDB_TABLE_NAME_LEN] = {0};
-    mndExtractTableName(pStb->name, stbName);
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, stbName);
-    cols++;
-
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     *(int64_t *)pWrite = pStb->createdTime;
     cols++;
@@ -1569,7 +1590,11 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, pStb->comment);
+    if (pStb->commentLen != 0) {
+      STR_TO_VARSTR(pWrite, pStb->comment);
+    } else {
+      STR_TO_VARSTR(pWrite, "");
+    }
     cols++;
 
     numOfRows++;

source/dnode/mnode/impl/src/mndStream.c

@@ -21,6 +21,7 @@
 #include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndTopic.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
@@ -33,6 +34,7 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream);
 static int32_t mndProcessCreateStreamReq(SNodeMsg *pReq);
+static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp);
 /*static int32_t mndProcessDropStreamReq(SNodeMsg *pReq);*/
 /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/
 static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq);
@@ -50,6 +52,8 @@ int32_t mndInitStream(SMnode *pMnode) {
                      .deleteFp = (SdbDeleteFp)mndStreamActionDelete};
 
   mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq);
+  mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp);
+  mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp);
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
 
@@ -68,7 +72,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
 
   SCoder encoder;
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
-  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+  if (tEncodeSStreamObj(&encoder, pStream) < 0) {
     tCoderClear(&encoder);
     goto STREAM_ENCODE_OVER;
   }
@@ -83,7 +87,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
   if (buf == NULL) goto STREAM_ENCODE_OVER;
 
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
-  if (tEncodeSStreamObj(NULL, pStream) < 0) {
+  if (tEncodeSStreamObj(&encoder, pStream) < 0) {
     tCoderClear(&encoder);
     goto STREAM_ENCODE_OVER;
   }
@@ -135,7 +139,7 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
   SDB_GET_BINARY(pRaw, dataPos, buf, tlen, STREAM_DECODE_OVER);
 
   SCoder decoder;
-  tCoderInit(&decoder, TD_LITTLE_ENDIAN, NULL, 0, TD_DECODER);
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, tlen + 1, TD_DECODER);
   if (tDecodeSStreamObj(&decoder, pStream) < 0) {
     goto STREAM_DECODE_OVER;
   }
@@ -191,6 +195,11 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
   sdbRelease(pSdb, pStream);
 }
 
+static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp) {
+  mndTransProcessRsp(pRsp);
+  return 0;
+}
+
 static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) {
   SName name = {0};
   tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@@ -209,6 +218,90 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
   return 0;
 }
 
+static SArray *mndExtractNamesFromAst(const SNode *pAst) {
+  if (pAst->type != QUERY_NODE_SELECT_STMT) return NULL;
+
+  SArray *names = taosArrayInit(0, sizeof(void *));
+  if (names == NULL) {
+    return NULL;
+  }
+  SSelectStmt *pSelect = (SSelectStmt *)pAst;
+  SNodeList   *pNodes = pSelect->pProjectionList;
+  SListCell   *pCell = pNodes->pHead;
+  while (pCell != NULL) {
+    if (pCell->pNode->type != QUERY_NODE_FUNCTION) {
+      continue;
+    }
+    SFunctionNode *pFunction = (SFunctionNode *)pCell->pNode;
+    char          *name = strdup(pFunction->node.aliasName);
+    taosArrayPush(names, &name);
+    pCell = pCell->pNext;
+  }
+  return names;
+}
+
+static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
+  if (NULL == ast) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode  *pAst = NULL;
+  int32_t code = nodesStringToNode(ast, &pAst);
+
+  SQueryPlan *pPlan = NULL;
+  if (TSDB_CODE_SUCCESS == code) {
+    SPlanContext cxt = {
+        .pAstRoot = pAst,
+        .topicQuery = false,
+        .streamQuery = true,
+    };
+    code = qCreateQueryPlan(&cxt, &pPlan, NULL);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesNodeToString(pPlan, false, pStr, NULL);
+  }
+  nodesDestroyNode(pAst);
+  nodesDestroyNode(pPlan);
+  terrno = code;
+  return code;
+}
+
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
+  SNode *pAst = NULL;
+  if (nodesStringToNode(ast, &pAst) < 0) {
+    return -1;
+  }
+  SArray *names = mndExtractNamesFromAst(pAst);
+  printf("|");
+  for (int i = 0; i < taosArrayGetSize(names); i++) {
+    printf(" %15s |", (char *)taosArrayGetP(names, i));
+  }
+  printf("\n=======================================================\n");
+
+  pStream->outputName = names;
+
+  if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) {
+    mError("topic:%s, failed to get plan since %s", pStream->name, terrstr());
+    return -1;
+  }
+
+  if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
+    mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
+    return -1;
+  }
+
+  SSdbRaw *pRedoRaw = mndStreamActionEncode(pStream);
+  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+  sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+  return 0;
+}
+
 static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamReq *pCreate, SDbObj *pDb) {
   mDebug("stream:%s to create", pCreate->name);
   SStreamObj streamObj = {0};
@@ -220,30 +313,22 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.dbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
-  streamObj.physicalPlan = "";
-  streamObj.logicalPlan = "";
+  /*streamObj.physicalPlan = "";*/
+  streamObj.logicalPlan = "not implemented";
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
   if (pTrans == NULL) {
     mError("stream:%s, failed to create since %s", pCreate->name, terrstr());
     return -1;
   }
   mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
 
-  if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) {
-    mError("stream:%ld, schedule stream since %s", streamObj.uid, terrstr());
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
+    mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
     return -1;
   }
 
-  SSdbRaw *pRedoRaw = mndStreamActionEncode(&streamObj);
-  if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
-    mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
-    return -1;
-  }
-  sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
-
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);

source/dnode/mnode/impl/src/mndTopic.c

@@ -236,7 +236,7 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   return 0;
 }
 
-static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) {
+static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) {
   if (NULL == pCreate->ast) {
     return TSDB_CODE_SUCCESS;
   }

source/dnode/mnode/impl/src/mndTrans.c

@@ -406,6 +406,10 @@ static const char *mndTransType(ETrnType type) {
       return "alter-stb";
     case TRN_TYPE_DROP_STB:
       return "drop-stb";
+    case TRN_TYPE_CREATE_SMA:
+      return "create-sma";
+    case TRN_TYPE_DROP_SMA:
+      return "drop-sma";
     default:
       return "invalid";
   }

source/dnode/mnode/impl/src/mnode.c

@@ -28,6 +28,7 @@
 #include "mndProfile.h"
 #include "mndQnode.h"
 #include "mndShow.h"
+#include "mndSma.h"
 #include "mndSnode.h"
 #include "mndStb.h"
 #include "mndStream.h"
@@ -204,6 +205,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-stb", mndInitSma, mndCleanupSma) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
@@ -409,15 +411,15 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
 }
 
 // Note: uid 0 is reserved
-uint64_t mndGenerateUid(char *name, int32_t len) {
+int64_t mndGenerateUid(char *name, int32_t len) {
   int32_t hashval = MurmurHash3_32(name, len);
 
   do {
     int64_t  us = taosGetTimestampUs();
-    uint64_t x = (us & 0x000000FFFFFFFFFF) << 24;
-    uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
+    int64_t x = (us & 0x000000FFFFFFFFFF) << 24;
+    int64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul));
     if (uuid) {
-      return uuid;
+      return abs(uuid);
     }
   } while (true);
 }

source/dnode/mnode/impl/test/CMakeLists.txt

@@ -12,5 +12,6 @@ add_subdirectory(dnode)
 add_subdirectory(mnode)
 add_subdirectory(db)
 add_subdirectory(stb)
+add_subdirectory(sma)
 add_subdirectory(func)
 add_subdirectory(topic)

source/dnode/mnode/impl/test/sma/CMakeLists.txt

+aux_source_directory(. SMA_SRC)
+add_executable(mnode_test_sma ${SMA_SRC})
+target_link_libraries(
+    mnode_test_sma
+    PUBLIC sut
+)
+
+add_test(
+    NAME mnode_test_sma
+    COMMAND mnode_test_sma
+)

source/dnode/mnode/impl/test/sma/sma.cpp

+/**
+ * @file sma.cpp
+ * @author slguan (slguan@taosdata.com)
+ * @brief MNODE module sma tests
+ * @version 1.0
+ * @date 2022-03-23
+ *
+ * @copyright Copyright (c) 2022
+ *
+ */
+
+#include "sut.h"
+
+class MndTestSma : public ::testing::Test {
+ protected:
+  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_sma", 9035); }
+  static void TearDownTestSuite() { test.Cleanup(); }
+
+  static Testbase test;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  void* BuildCreateDbReq(const char* dbname, int32_t* pContLen);
+  void* BuildDropDbReq(const char* dbname, int32_t* pContLen);
+  void* BuildCreateStbReq(const char* stbname, int32_t* pContLen);
+  void* BuildDropStbReq(const char* stbname, int32_t* pContLen);
+  void* BuildCreateSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr,
+                          const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen);
+  void* BuildDropSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen);
+};
+
+Testbase MndTestSma::test;
+
+void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
+  SCreateDbReq createReq = {0};
+  strcpy(createReq.db, dbname);
+  createReq.numOfVgroups = 2;
+  createReq.cacheBlockSize = 16;
+  createReq.totalBlocks = 10;
+  createReq.daysPerFile = 10;
+  createReq.daysToKeep0 = 3650;
+  createReq.daysToKeep1 = 3650;
+  createReq.daysToKeep2 = 3650;
+  createReq.minRows = 100;
+  createReq.maxRows = 4096;
+  createReq.commitTime = 3600;
+  createReq.fsyncPeriod = 3000;
+  createReq.walLevel = 1;
+  createReq.precision = 0;
+  createReq.compression = 2;
+  createReq.replications = 1;
+  createReq.quorum = 1;
+  createReq.update = 0;
+  createReq.cacheLastRow = 0;
+  createReq.ignoreExist = 1;
+
+  int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSCreateDbReq(pReq, contLen, &createReq);
+
+  *pContLen = contLen;
+  return pReq;
+}
+
+void* MndTestSma::BuildDropDbReq(const char* dbname, int32_t* pContLen) {
+  SDropDbReq dropdbReq = {0};
+  strcpy(dropdbReq.db, dbname);
+
+  int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSDropDbReq(pReq, contLen, &dropdbReq);
+
+  *pContLen = contLen;
+  return pReq;
+}
+
+void* MndTestSma::BuildCreateStbReq(const char* stbname, int32_t* pContLen) {
+  SMCreateStbReq createReq = {0};
+  createReq.numOfColumns = 3;
+  createReq.numOfTags = 1;
+  createReq.igExists = 0;
+  createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
+  createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
+  strcpy(createReq.name, stbname);
+
+  {
+    SField field = {0};
+    field.bytes = 8;
+    field.type = TSDB_DATA_TYPE_TIMESTAMP;
+    strcpy(field.name, "ts");
+    taosArrayPush(createReq.pColumns, &field);
+  }
+
+  {
+    SField field = {0};
+    field.bytes = 2;
+    field.type = TSDB_DATA_TYPE_TINYINT;
+    strcpy(field.name, "col1");
+    taosArrayPush(createReq.pColumns, &field);
+  }
+
+  {
+    SField field = {0};
+    field.bytes = 8;
+    field.type = TSDB_DATA_TYPE_BIGINT;
+    strcpy(field.name, "col2");
+    taosArrayPush(createReq.pColumns, &field);
+  }
+
+  {
+    SField field = {0};
+    field.bytes = 2;
+    field.type = TSDB_DATA_TYPE_TINYINT;
+    strcpy(field.name, "tag1");
+    taosArrayPush(createReq.pTags, &field);
+  }
+
+  int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
+  void*   pHead = rpcMallocCont(tlen);
+  tSerializeSMCreateStbReq(pHead, tlen, &createReq);
+  tFreeSMCreateStbReq(&createReq);
+  *pContLen = tlen;
+  return pHead;
+}
+
+void* MndTestSma::BuildDropStbReq(const char* stbname, int32_t* pContLen) {
+  SMDropStbReq dropstbReq = {0};
+  strcpy(dropstbReq.name, stbname);
+
+  int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropstbReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSMDropStbReq(pReq, contLen, &dropstbReq);
+
+  *pContLen = contLen;
+  return pReq;
+}
+
+void* MndTestSma::BuildCreateSmaReq(const char* smaname, const char* stbname, int8_t igExists, const char* expr,
+                                    const char* tagsFilter, const char* sql, const char* ast, int32_t* pContLen) {
+  SMCreateSmaReq createReq = {0};
+  strcpy(createReq.name, smaname);
+  strcpy(createReq.stb, stbname);
+  createReq.igExists = igExists;
+  createReq.intervalUnit = 1;
+  createReq.slidingUnit = 2;
+  createReq.timezone = 3;
+  createReq.dstVgId = 4;
+  createReq.interval = 10;
+  createReq.offset = 5;
+  createReq.sliding = 6;
+  createReq.expr = (char*)expr;
+  createReq.exprLen = strlen(createReq.expr) + 1;
+  createReq.tagsFilter = (char*)tagsFilter;
+  createReq.tagsFilterLen = strlen(createReq.tagsFilter) + 1;
+  createReq.sql = (char*)sql;
+  createReq.sqlLen = strlen(createReq.sql) + 1;
+  createReq.ast = (char*)ast;
+  createReq.astLen = strlen(createReq.ast) + 1;
+
+  int32_t tlen = tSerializeSMCreateSmaReq(NULL, 0, &createReq);
+  void*   pHead = rpcMallocCont(tlen);
+  tSerializeSMCreateSmaReq(pHead, tlen, &createReq);
+  *pContLen = tlen;
+  return pHead;
+}
+
+void* MndTestSma::BuildDropSmaReq(const char* smaname, int8_t igNotExists, int32_t* pContLen) {
+  SMDropSmaReq dropsmaReq = {0};
+  dropsmaReq.igNotExists = igNotExists;
+  strcpy(dropsmaReq.name, smaname);
+
+  int32_t contLen = tSerializeSMDropSmaReq(NULL, 0, &dropsmaReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSMDropSmaReq(pReq, contLen, &dropsmaReq);
+
+  *pContLen = contLen;
+  return pReq;
+}
+
+TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
+  const char* dbname = "1.d1";
+  const char* stbname = "1.d1.stb";
+  const char* smaname = "1.d1.sma";
+  int32_t     contLen = 0;
+  void*       pReq;
+  SRpcMsg*    pRsp;
+
+  {
+    pReq = BuildCreateDbReq(dbname, &contLen);
+    pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
+    ASSERT_EQ(pRsp->code, 0);
+  }
+
+  {
+    pReq = BuildCreateStbReq(stbname, &contLen);
+    pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
+    ASSERT_EQ(pRsp->code, 0);
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, dbname);
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 1);
+  }
+#if 0
+  {
+    pReq = BuildCreateSmaReq(smaname, stbname, 0, "expr", "tagsFilter", "sql", "ast", &contLen);
+    pRsp = test.SendReq(TDMT_MND_CREATE_SMA, pReq, contLen);
+    ASSERT_EQ(pRsp->code, 0);
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname);
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 1);
+  }
+
+  // restart
+  test.Restart();
+
+  {
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname);
+    CHECK_META("show indexes", 3);
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 1);
+
+    CheckBinary("sma", TSDB_INDEX_NAME_LEN);
+    CheckTimestamp();
+    CheckBinary("stb", TSDB_TABLE_NAME_LEN);
+  }
+
+  {
+     pReq = BuildDropSmaReq(smaname, 0, &contLen);
+    pRsp = test.SendReq(TDMT_MND_DROP_SMA, pReq, contLen);
+    ASSERT_EQ(pRsp->code, 0);
+    test.SendShowMetaReq(TSDB_MGMT_TABLE_INDEX, dbname);
+    test.SendShowRetrieveReq();
+    EXPECT_EQ(test.GetShowRows(), 0);
+  }
+#endif  
+}

source/dnode/snode/src/snode.c

@@ -57,8 +57,8 @@ void sndMetaDelete(SStreamMeta *pMeta) {
 }
 
 int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) {
-  for (int i = 0; i < pTask->parallel; i++) {
-    pTask->runner.executor[i] = qCreateStreamExecTaskInfo(pTask->qmsg, NULL);
+  for (int i = 0; i < pTask->numOfRunners; i++) {
+    pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL);
   }
   return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *));
 }

source/dnode/vnode/inc/meta.h

@@ -51,7 +51,7 @@ int     metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg);
 int     metaDropTable(SMeta *pMeta, tb_uid_t uid);
 int     metaCommit(SMeta *pMeta);
 int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg);
-int32_t metaDropTSma(SMeta *pMeta, char *indexName);
+int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid);
 
 // For Query
 STbCfg *        metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid);

source/dnode/vnode/inc/tsdb.h

@@ -96,6 +96,7 @@ int    tsdbCommit(STsdb *pTsdb);
  */
 int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg);
 int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, int8_t smaType, char *msg);
+int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
 
 /**
  * @brief Insert RSma(Time-range-wise Rollup SMA) data.

source/dnode/vnode/src/inc/metaDef.h

@@ -34,7 +34,7 @@ void metaCloseDB(SMeta* pMeta);
 int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
 int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
 int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
-int  metaRemoveSmaFromDb(SMeta* pMeta, const char* indexName);
+int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
 
 // SMetaCache
 int  metaOpenCache(SMeta* pMeta);

source/dnode/vnode/src/inc/tqInt.h

@@ -18,8 +18,8 @@
 
 #include "meta.h"
 #include "tlog.h"
-#include "tq.h"
 #include "tqPush.h"
+#include "vnd.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -153,6 +153,11 @@ typedef struct {
   FTqDelete      pDeleter;
 } STqMetaStore;
 
+typedef struct {
+  SMemAllocatorFactory* pAllocatorFactory;
+  SMemAllocator*        pAllocator;
+} STqMemRef;
+
 struct STQ {
   // the collection of groups
   // the handle of meta kvstore
@@ -162,6 +167,7 @@ struct STQ {
   STqMetaStore* tqMeta;
   STqPushMgr*   tqPushMgr;
   SHashObj*     pStreamTasks;
+  SVnode*       pVnode;
   SWal*         pWal;
   SMeta*        pVnodeMeta;
 };

source/dnode/vnode/src/inc/tsdbLog.h

@@ -24,12 +24,12 @@ extern "C" {
 
 extern int32_t tsdbDebugFlag;
 
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", DEBUG_INFO, 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/tsdbSma.h

@@ -16,15 +16,15 @@
 #ifndef _TD_TSDB_SMA_H_
 #define _TD_TSDB_SMA_H_
 
-typedef struct SSmaStat SSmaStat;
-typedef struct SSmaEnv  SSmaEnv;
+typedef struct SSmaStat     SSmaStat;
+typedef struct SSmaEnv      SSmaEnv;
 
 struct SSmaEnv {
   TdThreadRwlock lock;
-  SDiskID          did;
-  TDBEnv           dbEnv;  // TODO: If it's better to put it in smaIndex level?
-  char *           path;   // relative path
-  SSmaStat *       pStat;
+  SDiskID        did;
+  TDBEnv         dbEnv;  // TODO: If it's better to put it in smaIndex level?
+  char          *path;   // relative path
+  SSmaStat      *pStat;
 };
 
 #define SMA_ENV_LOCK(env)       ((env)->lock)

source/dnode/vnode/src/inc/vnd.h

@@ -23,7 +23,6 @@
 #include "tlist.h"
 #include "tlockfree.h"
 #include "tmacro.h"
-#include "tq.h"
 #include "wal.h"
 
 #include "vnode.h"
@@ -34,6 +33,8 @@
 extern "C" {
 #endif
 
+typedef struct STQ STQ;
+
 typedef struct SVState   SVState;
 typedef struct SVBufPool SVBufPool;
 
@@ -81,12 +82,6 @@ struct SVnode {
 
 int vnodeScheduleTask(SVnodeTask* task);
 
-int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
-int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq);
-int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq);
-int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq);
-void    vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp);
-
 #define vFatal(...)                                              \
   do {                                                           \
     if (vDebugFlag & DEBUG_FATAL) {                              \
@@ -171,6 +166,26 @@ void*           vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
 void            vmaFree(SVMemAllocator* pVMA, void* ptr);
 bool            vmaIsFull(SVMemAllocator* pVMA);
 
+// init once
+int  tqInit();
+void tqCleanUp();
+
+// open in each vnode
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig,
+            SMemAllocatorFactory* allocFac);
+void tqClose(STQ*);
+
+// required by vnode
+int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
+int tqCommit(STQ*);
+
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
+int32_t tqProcessRebReq(STQ* pTq, char* msg);
+int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg);
+int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaBDBImpl.c

@@ -259,7 +259,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   return 0;
 }
 
-int metaRemoveSmaFromDb(SMeta *pMeta, const char *indexName) {
+int metaRemoveSmaFromDb(SMeta *pMeta,  int64_t indexUid) {
   // TODO
 #if 0
   DBT key = {0};
@@ -507,7 +507,7 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) {
   tsize += taosEncodeString(buf, pTbCfg->name);
   tsize += taosEncodeFixedU32(buf, pTbCfg->ttl);
   tsize += taosEncodeFixedU32(buf, pTbCfg->keep);
-  tsize += taosEncodeFixedU8(buf, pTbCfg->type);
+  tsize += taosEncodeFixedU8(buf, pTbCfg->info);
 
   if (pTbCfg->type == META_SUPER_TABLE) {
     SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema};
@@ -527,7 +527,7 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
   buf = taosDecodeString(buf, &(pTbCfg->name));
   buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl));
   buf = taosDecodeFixedU32(buf, &(pTbCfg->keep));
-  buf = taosDecodeFixedU8(buf, &(pTbCfg->type));
+  buf = taosDecodeFixedU8(buf, &(pTbCfg->info));
 
   if (pTbCfg->type == META_SUPER_TABLE) {
     SSchemaWrapper sw;

source/dnode/vnode/src/meta/metaIdx.c

@@ -121,11 +121,11 @@ int32_t metaCreateTSma(SMeta *pMeta, SSmaCfg *pCfg) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t metaDropTSma(SMeta *pMeta, char* indexName) {
+int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) {
   // TODO: Validate the cfg
   // TODO: add atomicity
 
-  if (metaRemoveSmaFromDb(pMeta, indexName) < 0) {
+  if (metaRemoveSmaFromDb(pMeta, indexUid) < 0) {
     // TODO: handle error
     return -1;
   }

source/dnode/vnode/src/tq/tq.c

@@ -17,11 +17,14 @@
 #include "tqInt.h"
 #include "tqMetaStore.h"
 
+void tqDebugShowSSData(SArray* dataBlocks);
+
 int32_t tqInit() { return tqPushMgrInit(); }
 
 void tqCleanUp() { tqPushMgrCleanUp(); }
 
-STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig,
+            SMemAllocatorFactory* allocFac) {
   STQ* pTq = malloc(sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -29,6 +32,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, S
   }
   pTq->path = strdup(path);
   pTq->tqConfig = tqConfig;
+  pTq->pVnode = pVnode;
   pTq->pWal = pWal;
   pTq->pVnodeMeta = pVnodeMeta;
 #if 0
@@ -68,8 +72,21 @@ void tqClose(STQ* pTq) {
   // TODO
 }
 
-int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
+int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) {
   if (msgType != TDMT_VND_SUBMIT) return 0;
+  void* data = malloc(msgLen);
+  if (data == NULL) {
+    return -1;
+  }
+  memcpy(data, msg, msgLen);
+  SRpcMsg req = {
+      .msgType = TDMT_VND_STREAM_TRIGGER,
+      .pCont = data,
+      .contLen = msgLen,
+  };
+  tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
+
+#if 0
   void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL);
   while (pIter != NULL) {
     STqPusher* pusher = *(STqPusher**)pIter;
@@ -97,6 +114,7 @@ int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
   // if handle waiting, launch query and response to consumer
   //
   // if no waiting handle, return
+#endif
   return 0;
 }
 
@@ -420,6 +438,21 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
   return 0;
 }
 
+int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
+  ASSERT(parallel <= 8);
+  pTask->numOfRunners = parallel;
+  for (int32_t i = 0; i < parallel; i++) {
+    STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
+    SReadHandle    handle = {
+           .reader = pReadHandle,
+           .meta = pTq->pVnodeMeta,
+    };
+    pTask->runner[i].inputHandle = pReadHandle;
+    pTask->runner[i].executor = qCreateStreamExecTaskInfo(pTask->qmsg, &handle);
+  }
+  return 0;
+}
+
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   SStreamTask* pTask = malloc(sizeof(SStreamTask));
   if (pTask == NULL) {
@@ -427,15 +460,183 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   }
   SCoder decoder;
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, (uint8_t*)msg, msgLen, TD_DECODER);
-  tDecodeSStreamTask(&decoder, pTask);
+  if (tDecodeSStreamTask(&decoder, pTask) < 0) {
+    ASSERT(0);
+  }
   tCoderClear(&decoder);
 
+  tqExpandTask(pTq, pTask, 8);
   taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask));
 
   return 0;
 }
 
+static char* formatTimestamp(char* buf, int64_t val, int precision) {
+  time_t  tt;
+  int32_t ms = 0;
+  if (precision == TSDB_TIME_PRECISION_NANO) {
+    tt = (time_t)(val / 1000000000);
+    ms = val % 1000000000;
+  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+    tt = (time_t)(val / 1000000);
+    ms = val % 1000000;
+  } else {
+    tt = (time_t)(val / 1000);
+    ms = val % 1000;
+  }
+
+  /* comment out as it make testcases like select_with_tags.sim fail.
+    but in windows, this may cause the call to localtime crash if tt < 0,
+    need to find a better solution.
+    if (tt < 0) {
+      tt = 0;
+    }
+    */
+
+#ifdef WINDOWS
+  if (tt < 0) tt = 0;
+#endif
+  if (tt <= 0 && ms < 0) {
+    tt--;
+    if (precision == TSDB_TIME_PRECISION_NANO) {
+      ms += 1000000000;
+    } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+      ms += 1000000;
+    } else {
+      ms += 1000;
+    }
+  }
+
+  struct tm* ptm = localtime(&tt);
+  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
+
+  if (precision == TSDB_TIME_PRECISION_NANO) {
+    sprintf(buf + pos, ".%09d", ms);
+  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
+    sprintf(buf + pos, ".%06d", ms);
+  } else {
+    sprintf(buf + pos, ".%03d", ms);
+  }
+
+  return buf;
+}
+void tqDebugShowSSData(SArray* dataBlocks) {
+  char    pBuf[128];
+  int32_t sz = taosArrayGetSize(dataBlocks);
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i);
+    int32_t      colNum = pDataBlock->info.numOfCols;
+    int32_t      rows = pDataBlock->info.rows;
+    for (int32_t j = 0; j < rows; j++) {
+      printf("|");
+      for (int32_t k = 0; k < colNum; k++) {
+        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+        switch (pColInfoData->info.type) {
+          case TSDB_DATA_TYPE_TIMESTAMP:
+            formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
+            printf(" %25s |", pBuf);
+            break;
+          case TSDB_DATA_TYPE_INT:
+          case TSDB_DATA_TYPE_UINT:
+            printf(" %15d |", *(int32_t*)var);
+            break;
+          case TSDB_DATA_TYPE_BIGINT:
+          case TSDB_DATA_TYPE_UBIGINT:
+            printf(" %15ld |", *(int64_t*)var);
+            break;
+        }
+      }
+      printf("\n");
+    }
+  }
+}
+
+int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen) {
+  void* pIter = NULL;
+
+  while (1) {
+    pIter = taosHashIterate(pTq->pStreamTasks, pIter);
+    if (pIter == NULL) break;
+    SStreamTask* pTask = (SStreamTask*)pIter;
+    if (!pTask->pipeSource) continue;
+
+    int32_t workerId = 0;
+    void*   exec = pTask->runner[workerId].executor;
+    qSetStreamInput(exec, data, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+    SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+    while (1) {
+      SSDataBlock* output;
+      uint64_t     ts;
+      if (qExecTask(exec, &output, &ts) < 0) {
+        ASSERT(false);
+      }
+      if (output == NULL) {
+        break;
+      }
+      taosArrayPush(pRes, output);
+    }
+    if (pTask->pipeSink) {
+      // write back
+      /*printf("reach end\n");*/
+      tqDebugShowSSData(pRes);
+    } else {
+      int32_t tlen = sizeof(SStreamExecMsgHead) + tEncodeDataBlocks(NULL, pRes);
+      void*   buf = rpcMallocCont(tlen);
+      if (buf == NULL) {
+        return -1;
+      }
+      void* abuf = POINTER_SHIFT(buf, sizeof(SStreamExecMsgHead));
+      tEncodeDataBlocks(abuf, pRes);
+      tmsg_t type;
+
+      if (pTask->nextOpDst == STREAM_NEXT_OP_DST__VND) {
+        type = TDMT_VND_TASK_EXEC;
+      } else {
+        type = TDMT_SND_TASK_EXEC;
+      }
+
+      SRpcMsg reqMsg = {
+          .pCont = buf,
+          .contLen = tlen,
+          .code = 0,
+          .msgType = type,
+      };
+      tmsgSendReq(&pTq->pVnode->msgCb, &pTask->NextOpEp, &reqMsg);
+    }
+  }
+  return 0;
+}
+
 int32_t tqProcessTaskExec(STQ* pTq, SRpcMsg* msg) {
-  //
+  SStreamTaskExecReq* pReq = msg->pCont;
+
+  int32_t taskId = pReq->head.streamTaskId;
+  int32_t workerType = pReq->head.workerType;
+
+  SStreamTask* pTask = taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
+  // assume worker id is 1
+  int32_t workerId = 1;
+  void*   exec = pTask->runner[workerId].executor;
+  int32_t sz = taosArrayGetSize(pReq->data);
+  printf("input data:\n");
+  tqDebugShowSSData(pReq->data);
+  SArray* pRes = taosArrayInit(0, sizeof(void*));
+  for (int32_t i = 0; i < sz; i++) {
+    SSDataBlock* input = taosArrayGet(pReq->data, i);
+    SSDataBlock* output;
+    uint64_t     ts;
+    qSetStreamInput(exec, input, STREAM_DATA_TYPE_SSDATA_BLOCK);
+    if (qExecTask(exec, &output, &ts) < 0) {
+      ASSERT(0);
+    }
+    if (output == NULL) {
+      break;
+    }
+    taosArrayPush(pRes, &output);
+  }
+  printf("output data:\n");
+  tqDebugShowSSData(pRes);
+
   return 0;
 }

source/dnode/vnode/src/tq/tqRead.c

@@ -126,6 +126,36 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   if (pArray == NULL) {
     return NULL;
   }
+  int32_t colMeta = 0;
+  int32_t colNeed = 0;
+  while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) {
+    SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta];
+    int16_t  colIdSchema = pColSchema->colId;
+    int16_t  colIdNeed = *(int16_t*)taosArrayGet(pHandle->pColIdList, colNeed);
+    if (colIdSchema < colIdNeed) {
+      colMeta++;
+    } else if (colIdSchema > colIdNeed) {
+      colNeed++;
+    } else {
+      SColumnInfoData colInfo = {0};
+      int             sz = numOfRows * pColSchema->bytes;
+      colInfo.info.bytes = pColSchema->bytes;
+      colInfo.info.colId = pColSchema->colId;
+      colInfo.info.type = pColSchema->type;
+
+      colInfo.pData = calloc(1, sz);
+      if (colInfo.pData == NULL) {
+        // TODO free
+        taosArrayDestroy(pArray);
+        return NULL;
+      }
+
+      blockDataEnsureColumnCapacity(&colInfo, numOfRows);
+      taosArrayPush(pArray, &colInfo);
+      colMeta++;
+      colNeed++;
+    }
+  }
 
   int j = 0;
   for (int32_t i = 0; i < colNumNeed; i++) {
@@ -163,11 +193,23 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
     tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block
+    int32_t colTot = taosArrayGetSize(pArray);
+    for (int32_t i = 0; i < colTot; i++) {
+      SColumnInfoData* pColData = taosArrayGet(pArray, i);
+      SCellVal         sVal = {0};
+      if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
+        break;
+      }
+      memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes);
+    }
+#if 0
     for (int32_t i = 0; i < colNumNeed; i++) {
       SColumnInfoData* pColData = taosArrayGet(pArray, i);
       STColumn*        pCol = schemaColAt(pTschema, i);
       // TODO
-      ASSERT(pCol->colId == pColData->info.colId);
+      if(pCol->colId != pColData->info.colId) {
+        continue;
+      }
       // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
       SCellVal sVal = {0};
       if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
@@ -176,6 +218,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       }
       memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
     }
+#endif
     curRow++;
   }
   return pArray;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -403,18 +403,12 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
       SColumnInfoData colInfo = {{0}, 0};
       colInfo.info = pCond->colList[i];
 
-      colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes);
-      if (!IS_VAR_DATA_TYPE(colInfo.info.type)) {
-        colInfo.nullbitmap = calloc(1, BitmapLen(pReadHandle->outputCapacity));
-      }
-
-      if (colInfo.pData == NULL || (colInfo.nullbitmap == NULL && (!IS_VAR_DATA_TYPE(colInfo.info.type)))) {
+      int32_t code = blockDataEnsureColumnCapacity(&colInfo, pReadHandle->outputCapacity);
+      if (code != TSDB_CODE_SUCCESS) {
         goto _end;
       }
 
       taosArrayPush(pReadHandle->pColumns, &colInfo);
-
-
       pReadHandle->statis[i].colId = colInfo.info.colId;
     }
 
@@ -1383,7 +1377,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 }
 
 static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
-  char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1;
 
   SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
@@ -1409,43 +1402,38 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
       continue;
     }
 
-    int32_t bytes = pColInfo->info.bytes;
-
-    if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-      pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
-    } else {
-      pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
-    }
-
     if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) {
-      if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
-        memmove(pData, (char*)src->pData + bytes * start, bytes * num);
-      } else {  // handle the var-string
-        char* dst = pData;
+      if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) {  // todo opt performance
+//        memmove(pData, (char*)src->pData + bytes * start, bytes * num);
+        for(int32_t k = start; k < num + start; ++k) {
+          SCellVal sVal = {0};
+          if (tdGetColDataOfRow(&sVal, src, k) < 0) {
+            TASSERT(0);
+          }
 
+          if (sVal.valType == TD_VTYPE_NULL) {
+            colDataAppend(pColInfo, k, NULL, true);
+          } else {
+            colDataAppend(pColInfo, k, sVal.val, false);
+          }
+        }
+      } else {  // handle the var-string
         // todo refactor, only copy one-by-one
         for (int32_t k = start; k < num + start; ++k) {
-          SCellVal    sVal = {0};
+          SCellVal sVal = {0};
           if(tdGetColDataOfRow(&sVal, src, k) < 0){
             TASSERT(0);
           }
-          memcpy(dst, sVal.val, varDataTLen(sVal.val));
-          dst += bytes;
+
+          colDataAppend(pColInfo, k, sVal.val, false);
         }
       }
 
       j++;
       i++;
     } else { // pColInfo->info.colId < src->colId, it is a NULL data
-      if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-        char* dst = pData;
-
-        for(int32_t k = start; k < num + start; ++k) {
-          setVardataNull(dst, pColInfo->info.type);
-          dst += bytes;
-        }
-      } else {
-        setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num);
+      for(int32_t k = start; k < num + start; ++k) {  // TODO opt performance
+        colDataAppend(pColInfo, k, NULL, true);
       }
       i++;
     }
@@ -1453,23 +1441,9 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
 
   while (i < requiredNumOfCols) { // the remain columns are all null data
     SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-    if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-      pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
-    } else {
-      pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes;
-    }
-
-    if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
-      char* dst = pData;
-
-      for(int32_t k = start; k < num + start; ++k) {
-        setVardataNull(dst, pColInfo->info.type);
-        dst += pColInfo->info.bytes;
-      }
-    } else {
-      setNullN(pData, pColInfo->info.type, pColInfo->info.bytes, num);
+    for(int32_t k = start; k < num + start; ++k) {
+      colDataAppend(pColInfo, k, NULL, true); // TODO add a fast version to set a number of consecutive NULL value.
     }
-
     i++;
   }
 
@@ -1479,14 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
   return numOfRows + num;
 }
 
-// TODO fix bug for reverse copy data
-// TODO handle the null data
+// TODO fix bug for reverse copy data problem
 // Note: row1 always has high priority
 static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1,
                                STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
                                bool forceSetNull) {
 #if 1
-  char*       pData = NULL;
   STSchema*   pSchema;
   STSRow*     row;
   int16_t     colId;
@@ -1528,12 +1500,6 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
   while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) {
     SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
 
-    if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-      pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
-    } else {
-      pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
-    }
-
     int32_t colIdOfRow1;
     if(j >= numOfColsOfRow1) {
       colIdOfRow1 = INT32_MAX;
@@ -1596,43 +1562,11 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
 
     if (colId == pColInfo->info.colId) {
       if (tdValTypeIsNorm(sVal.valType)) {
-        switch (pColInfo->info.type) {
-          case TSDB_DATA_TYPE_BINARY:
-          case TSDB_DATA_TYPE_NCHAR:
-            memcpy(pData, sVal.val, varDataTLen(sVal.val));
-            break;
-          case TSDB_DATA_TYPE_BOOL:
-          case TSDB_DATA_TYPE_TINYINT:
-          case TSDB_DATA_TYPE_UTINYINT:
-            *(uint8_t *)pData = *(uint8_t *)sVal.val;
-            break;
-          case TSDB_DATA_TYPE_SMALLINT:
-          case TSDB_DATA_TYPE_USMALLINT:
-            *(uint16_t *)pData = *(uint16_t *)sVal.val;
-            break;
-          case TSDB_DATA_TYPE_INT:
-          case TSDB_DATA_TYPE_UINT:
-            *(uint32_t *)pData = *(uint32_t *)sVal.val;
-            break;
-          case TSDB_DATA_TYPE_BIGINT:
-          case TSDB_DATA_TYPE_UBIGINT:
-            *(uint64_t *)pData = *(uint64_t *)sVal.val;
-            break;
-          case TSDB_DATA_TYPE_FLOAT:
-            SET_FLOAT_PTR(pData, sVal.val);
-            break;
-          case TSDB_DATA_TYPE_DOUBLE:
-            SET_DOUBLE_PTR(pData, sVal.val);
-            break;
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            *(TSKEY*)pData = *(TSKEY*)sVal.val;
-            break;
-          default:
-            memcpy(pData, sVal.val, pColInfo->info.bytes);
-        }
+        colDataAppend(pColInfo, numOfRows, sVal.val, false);
       } else if (forceSetNull) {
         colDataAppend(pColInfo, numOfRows, NULL, true);
       }
+
       i++;
 
       if(row == row1) {

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -727,9 +727,9 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
       }
 
       ASSERT(pBlockCol->colId == pDataCol->colId);
-      // set the bitmap
-      pDataCol->bitmap = pBlockCol->bitmap;
     }
+    // set the bitmap
+    pDataCol->bitmap = pBlockCol->bitmap;
 
     if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
   }

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -25,6 +25,7 @@ static const char *TSDB_SMA_DNAME[] = {
 #define SMA_STORAGE_TSDB_TIMES  10
 #define SMA_STORAGE_SPLIT_HOURS 24
 #define SMA_KEY_LEN             18  // tableUid_colId_TSKEY 8+2+8
+#define SMA_DROP_EXPIRED_TIME   10  // default is 10 seconds
 
 #define SMA_STATE_HASH_SLOT      4
 #define SMA_STATE_ITEM_HASH_SLOT 32
@@ -60,10 +61,11 @@ typedef struct {
 typedef struct {
   /**
    * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
-   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
-   * without information about its previous state.
    *    - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
    * Streaming Module or TSDB local persistence.
+   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
+   * without information about its previous state.
+   *    - TSDB_SMA_STAT_DROPPED: 1)sma dropped
    */
   int8_t    state;           // ETsdbSmaStat
   SHashObj *expiredWindows;  // key: skey of time window, value: N/A
@@ -80,6 +82,7 @@ struct SSmaStat {
 // expired window
 static int32_t  tsdbUpdateExpiredWindow(STsdb *pTsdb, ETsdbSmaType smaType, char *msg);
 static int32_t  tsdbInitSmaStat(SSmaStat **pSmaStat);
+static void *   tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
 static int32_t  tsdbDestroySmaState(SSmaStat *pSmaStat);
 static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did);
 static int32_t  tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv);
@@ -109,7 +112,55 @@ static void    tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[])
 static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg);
 static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg);
 
+// mgmt interface
+static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid);
+
 // implementation
+static FORCE_INLINE int8_t tsdbSmaStat(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    return atomic_load_8(&pStatItem->state);
+  }
+  return TSDB_SMA_STAT_UNKNOWN;
+}
+
+static FORCE_INLINE bool tsdbSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) {
+  if(!pStatItem) {
+    return false;
+  }
+
+  if (state) {
+    *state = atomic_load_8(&pStatItem->state);
+    return *state == TSDB_SMA_STAT_OK;
+  }
+  return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK;
+}
+
+static FORCE_INLINE bool tsdbSmaStatIsExpired(SSmaStatItem *pStatItem) {
+  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true;
+}
+
+static FORCE_INLINE bool tsdbSmaStatIsDropped(SSmaStatItem *pStatItem) {
+  return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true;
+}
+
+static FORCE_INLINE void tsdbSmaStatSetOK(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK);
+  }
+}
+
+static FORCE_INLINE void tsdbSmaStatSetExpired(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED);
+  }
+}
+
+static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) {
+  if (pStatItem) {
+    atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED);
+  }
+}
+
 static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
   snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP,
            TSDB_SMA_DNAME[smaType]);
@@ -252,6 +303,16 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) {
   return pItem;
 }
 
+static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
+  if (pSmaStatItem != NULL) {
+    tdDestroyTSma(pSmaStatItem->pSma);
+    tfree(pSmaStatItem->pSma);
+    taosHashCleanup(pSmaStatItem->expiredWindows);
+    tfree(pSmaStatItem);
+  }
+  return NULL;
+}
+
 /**
  * @brief Release resources allocated for its member fields, not including itself.
  *
@@ -264,12 +325,7 @@ int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) {
     void *item = taosHashIterate(pSmaStat->smaStatItems, NULL);
     while (item != NULL) {
       SSmaStatItem *pItem = *(SSmaStatItem **)item;
-      if (pItem != NULL) {
-        tdDestroyTSma(pItem->pSma);
-        tfree(pItem->pSma);
-        taosHashCleanup(pItem->expiredWindows);
-        tfree(pItem);
-      }
+      tsdbFreeSmaStatItem(pItem);
       item = taosHashIterate(pSmaStat->smaStatItems, item);
     }
     taosHashCleanup(pSmaStat->smaStatItems);
@@ -437,6 +493,15 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind
                skey, indexUid);
       return TSDB_CODE_FAILED;
     }
+    // TODO: use a standalone interface to received state upate notification from stream computing module.
+    /**
+     * @brief state
+     *  - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK.
+     *  - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g.
+     * when batch data caculation not finised)
+     *  - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB.
+     */
+    pItem->state = TSDB_SMA_STAT_OK;
   } else {
     // error handling
     tsdbUnRefSmaStat(pTsdb, pStat);
@@ -711,6 +776,8 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
   STsdbCfg *        pCfg = REPO_CFG(pTsdb);
   STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
   SSmaEnv *         pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
+  int64_t indexUid = SMA_TEST_INDEX_UID;
+
 
   if (pEnv == NULL) {
     terrno = TSDB_CODE_INVALID_PTR;
@@ -730,13 +797,28 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
     return TSDB_CODE_FAILED;
   }
 
-  int64_t indexUid = SMA_TEST_INDEX_UID;
+  SSmaStat *pStat = SMA_ENV_STAT(pTsdb->pTSmaEnv);
+  SSmaStatItem *pItem = NULL;
+
+  tsdbRefSmaStat(pTsdb, pStat);
+
+  if (pStat && pStat->smaStatItems) {
+    pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid));
+  }
+
+  if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) {
+    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    tsdbUnRefSmaStat(pTsdb, pStat);
+    return TSDB_CODE_FAILED;
+  }
+
   char    rPath[TSDB_FILENAME_LEN] = {0};
   char    aPath[TSDB_FILENAME_LEN] = {0};
   snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
   tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath);
   if (!taosCheckExistFile(aPath)) {
     if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
+      tsdbUnRefSmaStat(pTsdb, pStat);
       return TSDB_CODE_FAILED;
     }
   }
@@ -754,12 +836,14 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
     tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
              tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
     tsdbDestroyTSmaWriteH(&tSmaH);
+    tsdbUnRefSmaStat(pTsdb, pStat);
     return TSDB_CODE_FAILED;
   }
 
   if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) {
     tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
     tsdbDestroyTSmaWriteH(&tSmaH);
+    tsdbUnRefSmaStat(pTsdb, pStat);
     return TSDB_CODE_FAILED;
   }
   // TODO:tsdbEndTSmaCommit();
@@ -768,9 +852,60 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) {
   tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey);
 
   tsdbDestroyTSmaWriteH(&tSmaH);
+  tsdbUnRefSmaStat(pTsdb, pStat);
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief Drop tSma data and local cache
+ *        - insert/query reference
+ * @param pTsdb 
+ * @param msg 
+ * @return int32_t 
+ */
+static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) {
+  SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv);
+
+  // clear local cache
+  if (pEnv) {
+    tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid);
+
+    SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
+    if ((pItem != NULL) || ((pItem = *(SSmaStatItem **)pItem) != NULL)) {
+      if (tsdbSmaStatIsDropped(pItem)) {
+        tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid);
+        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
+      }
+
+      tsdbWLockSma(pEnv);
+      if (tsdbSmaStatIsDropped(pItem)) {
+        tsdbUnLockSma(pEnv);
+        tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid);
+        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
+      }
+      tsdbSmaStatSetDropped(pItem);
+      tsdbUnLockSma(pEnv);
+
+      int32_t nSleep = 0;
+      while (true) {
+        if (T_REF_VAL_GET(SMA_ENV_STAT(pEnv)) <= 0) {
+          break;
+        }
+        taosSsleep(1);
+        if (++nSleep > SMA_DROP_EXPIRED_TIME) {
+          break;
+        };
+      }
+
+      tsdbFreeSmaStatItem(pItem);
+      tsdbDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", REPO_ID(pTsdb), indexUid);
+    }
+  }
+  // clear sma data files
+  // TODO: 
+
+}
+
 static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) {
   STsdb *pTsdb = pSmaH->pTsdb;
 
@@ -784,35 +919,64 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData,
 static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) {
   STsdbCfg *        pCfg = REPO_CFG(pTsdb);
   STSmaDataWrapper *pData = (STSmaDataWrapper *)msg;
-  STSmaWriteH       tSmaH = {0};
+  SSmaEnv *         pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv);
 
-  tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData);
+  if (pEnv == NULL) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    tsdbWarn("vgId:%d insert tSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb));
+    return terrno;
+  }
 
   if (pData->dataLen <= 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
+    return TSDB_CODE_FAILED;
   }
 
-  // Step 1: Judge the storage level
-  int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
-  int32_t daysPerFile = storageLevel == SMA_STORAGE_LEVEL_TSDB ? SMA_STORAGE_TSDB_DAYS : pCfg->daysPerFile;
+  STSmaWriteH tSmaH = {0};
 
-  // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
-  //         - Set and open the DFile or the B+Tree file
+  if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData) != 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  int64_t indexUid = SMA_TEST_INDEX_UID;
+  char    rPath[TSDB_FILENAME_LEN] = {0};
+  char    aPath[TSDB_FILENAME_LEN] = {0};
+  snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
+  tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath);
+  if (!taosCheckExistFile(aPath)) {
+    if (tfsMkdirRecurAt(REPO_TFS(pTsdb), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_FAILED;
+    }
+  }
 
+  // Step 1: Judge the storage level and days
+  int32_t storageLevel = tsdbGetSmaStorageLevel(pData->interval, pData->intervalUnit);
+  int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel);
   int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision));
 
-  // Save all the TSma data to one file
+  // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file
+  //         - Set and open the DFile or the B+Tree file
   // TODO: tsdbStartTSmaCommit();
-  tsdbSetTSmaDataFile(&tSmaH, pData, storageLevel, fid);
+  tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid);
+  if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) {
+    tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb),
+             tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
+    tsdbDestroyTSmaWriteH(&tSmaH);
+    return TSDB_CODE_FAILED;
+  }
 
-  tsdbInsertTSmaDataSection(&tSmaH, pData);
+  if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) {
+    tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    tsdbDestroyTSmaWriteH(&tSmaH);
+    return TSDB_CODE_FAILED;
+  }
   // TODO:tsdbEndTSmaCommit();
 
-  // reset the SSmaStat
-  tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pRSmaEnv), pData->indexUid, pData->skey);
+  // Step 3: reset the SSmaStat
+  tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey);
 
+  tsdbDestroyTSmaWriteH(&tSmaH);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -934,6 +1098,15 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_
 #endif
 
 #if 1
+  int8_t smaStat = 0;
+  if (!tsdbSmaStatIsOK(pItem, &smaStat)) {  // TODO: multiple check for large scale sma query
+    tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv));
+    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    tsdbWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, REPO_ID(pTsdb), indexUid,
+             tstrerror(terrno), smaStat);
+    return TSDB_CODE_FAILED;
+  }
+
   if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) {
     // TODO: mark this window as expired.
     tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb),
@@ -1086,6 +1259,20 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) {
   return code;
 }
 
+/**
+ * @brief Get tSma data
+ * 
+ * @param pTsdb 
+ * @param pData 
+ * @param indexUid 
+ * @param interval 
+ * @param intervalUnit 
+ * @param tableUid 
+ * @param colId 
+ * @param querySKey 
+ * @param nMaxResult 
+ * @return int32_t 
+ */
 int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit,
                         tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) {
   int32_t code = TSDB_CODE_SUCCESS;
@@ -1094,4 +1281,19 @@ int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid,
     tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
   }
   return code;
+}
+
+/**
+ * @brief Drop tSma Data and caches
+ * 
+ * @param pTsdb 
+ * @param msg 
+ * @return int32_t 
+ */
+int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) {
+    tsdbWarn("vgId:%d drop tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno));
+  }
+  return code;
 }
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeMain.c

@@ -115,7 +115,8 @@ static int vnodeOpenImpl(SVnode *pVnode) {
 
   // Open tsdb
   sprintf(dir, "%s/tsdb", pVnode->path);
-  pVnode->pTsdb = tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
+  pVnode->pTsdb =
+      tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
   if (pVnode->pTsdb == NULL) {
     // TODO: handle error
     return -1;
@@ -131,7 +132,7 @@ static int vnodeOpenImpl(SVnode *pVnode) {
 
   // Open TQ
   sprintf(dir, "%s/tq", pVnode->path);
-  pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode));
+  pVnode->pTq = tqOpen(dir, pVnode, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode));
   if (pVnode->pTq == NULL) {
     // TODO: handle error
     return -1;

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -14,6 +14,7 @@
  */
 
 #include "vnodeQuery.h"
+#include "executor.h"
 #include "vnd.h"
 
 static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg);
@@ -66,6 +67,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
       return tqProcessPollReq(pVnode->pTq, pMsg);
     case TDMT_VND_TASK_EXEC:
       return tqProcessTaskExec(pVnode->pTq, pMsg);
+    case TDMT_VND_STREAM_TRIGGER:
+      return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen);
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:
@@ -161,7 +164,6 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
     memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols);
   }
 
-
 _exit:
 
   rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp);
@@ -177,7 +179,6 @@ _exit:
   }
   tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
 
-
   tFreeSTableMetaRsp(&metaRsp);
   if (pSW != NULL) {
     tfree(pSW->pSchema);

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -13,12 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tq.h"
 #include "vnd.h"
 
 void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
   SNodeMsg *pMsg;
-  SRpcMsg *pRpc;
+  SRpcMsg  *pRpc;
 
   for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
     pMsg = *(SNodeMsg **)taosArrayGet(pMsgs, i);
@@ -60,7 +59,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
   // todo: change the interface here
   int64_t ver;
   taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver);
-  if (tqPushMsg(pVnode->pTq, ptr, pMsg->msgType, ver) < 0) {
+  if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) {
     // TODO: handle error
   }
 
@@ -86,10 +85,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       for (int i = 0; i < reqNum; i++) {
         SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
 
-        char tableFName[TSDB_TABLE_FNAME_LEN];
+        char      tableFName[TSDB_TABLE_FNAME_LEN];
         SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
         sprintf(tableFName, "%s.%s", pCreateTbReq->dbFName, pCreateTbReq->name);
-        
+
         int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName);
         if (code) {
           SVCreateTbRsp rsp;
@@ -97,7 +96,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
           taosArrayPush(vCreateTbBatchRsp.rspList, &rsp);
         }
-        
+
         if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
           // TODO: handle error
           vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name);
@@ -117,10 +116,10 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       taosArrayDestroy(vCreateTbBatchReq.pArray);
       if (vCreateTbBatchRsp.rspList) {
         int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp);
-        void *msg = rpcMallocCont(contLen);
+        void   *msg = rpcMallocCont(contLen);
         tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp);
         taosArrayDestroy(vCreateTbBatchRsp.rspList);
-        
+
         *pRsp = calloc(1, sizeof(SRpcMsg));
         (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP;
         (*pRsp)->pCont = msg;
@@ -169,6 +168,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       }
     } break;
     case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
+#if 0
       SSmaCfg vCreateSmaReq = {0};
       if (tDeserializeSVCreateTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateSmaReq) == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -190,26 +190,37 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       // }
       tdDestroyTSma(&vCreateSmaReq.tSma);
       // TODO: return directly or go on follow steps?
+#endif
     } break;
     case TDMT_VND_CANCEL_SMA: {  // timeRangeSMA
     } break;
     case TDMT_VND_DROP_SMA: {  // timeRangeSMA
+#if 0    
       SVDropTSmaReq vDropSmaReq = {0};
       if (tDeserializeSVDropTSmaReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vDropSmaReq) == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         return -1;
       }
 
-      if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexName) < 0) {
-        // TODO: handle error
-        return -1;
-      }
       // TODO: send msg to stream computing to drop tSma
       // if ((send msg to stream computing) < 0) {
       //   tdDestroyTSma(&vCreateSmaReq);
       //   return -1;
       // }
+      // 
+
+      if (metaDropTSma(pVnode->pMeta, vDropSmaReq.indexUid) < 0) {
+        // TODO: handle error
+        return -1;
+      }
+
+      if(tsdbDropTSmaData(pVnode->pTsdb, vDropSmaReq.indexUid) < 0) {
+        // TODO: handle error
+        return -1;
+      }
+
       // TODO: return directly or go on follow steps?
+#endif
     } break;
     default:
       ASSERT(0);

source/dnode/vnode/test/tqSerializerTest.cpp

-#include <gtest/gtest.h>
-#include <cstring>
-#include <iostream>
-#include <queue>
-
-#include "tq.h"
-
-using namespace std;
-
-TEST(TqSerializerTest, basicTest) {
-  TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle));
-  
-}

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -33,6 +33,61 @@ int main(int argc, char **argv) {
   return RUN_ALL_TESTS();
 }
 
+TEST(testCase, unionEncodeDecodeTest) {
+  typedef struct {
+    union {
+      uint8_t info;
+      struct {
+        uint8_t rollup : 1;  // 1 means rollup sma
+        uint8_t type : 7;
+      };
+    };
+    col_id_t  nBSmaCols;
+    col_id_t* pBSmaCols;
+  } SUnionTest;
+
+  SUnionTest sut = {0};
+  sut.rollup = 1;
+  sut.type = 1;
+
+  sut.nBSmaCols = 2;
+  sut.pBSmaCols = (col_id_t*)malloc(sut.nBSmaCols * sizeof(col_id_t));
+  for (col_id_t i = 0; i < sut.nBSmaCols; ++i) {
+    sut.pBSmaCols[i] = i + 100;
+  }
+
+  void* buf = malloc(1024);
+  void *  pBuf = buf;
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedU8(&buf, sut.info);
+  tlen += taosEncodeFixedI16(&buf, sut.nBSmaCols);
+  for (col_id_t i = 0; i < sut.nBSmaCols; ++i) {
+    tlen += taosEncodeFixedI16(&buf, sut.pBSmaCols[i]);
+  }
+
+  SUnionTest dut = {0};
+  pBuf = taosDecodeFixedU8(pBuf, &dut.info);
+  pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols);
+  if(dut.nBSmaCols > 0) {
+    dut.pBSmaCols = (col_id_t*)malloc(dut.nBSmaCols * sizeof(col_id_t));
+    for(col_id_t i=0; i < dut.nBSmaCols; ++i) {
+      pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i);
+    }
+  } else {
+    dut.pBSmaCols = NULL;
+  }
+
+  printf("sut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", sut.rollup, sut.type, sut.info);
+  printf("dut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", dut.rollup, dut.type, dut.info);
+
+  ASSERT_EQ(sut.rollup, dut.rollup);
+  ASSERT_EQ(sut.type, dut.type);
+  ASSERT_EQ(sut.nBSmaCols, dut.nBSmaCols);
+  for (col_id_t i = 0; i< sut.nBSmaCols; ++i) {
+    ASSERT_EQ(*(col_id_t*)(sut.pBSmaCols + i), sut.pBSmaCols[i]);
+    ASSERT_EQ(*(col_id_t*)(sut.pBSmaCols + i), dut.pBSmaCols[i]);
+  }
+}
 #if 1
 TEST(testCase, tSma_Meta_Encode_Decode_Test) {
   // encode
@@ -217,8 +272,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   taosArrayDestroy(pUids);
 
   // resource release
-  metaRemoveSmaFromDb(pMeta, smaIndexName1);
-  metaRemoveSmaFromDb(pMeta, smaIndexName2);
+  metaRemoveSmaFromDb(pMeta, indexUid1);
+  metaRemoveSmaFromDb(pMeta, indexUid2);
 
   tdDestroyTSma(&tSma);
   metaClose(pMeta);

source/libs/executor/src/executor.c

@@ -16,7 +16,7 @@
 #include "executor.h"
 #include "executorimpl.h"
 #include "planner.h"
-#include "tq.h"
+#include "vnode.h"
 
 static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t type, char* id) {
   ASSERT(pOperator != NULL);
@@ -30,7 +30,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
       qError("join not supported for stream block scan, %s" PRIx64, id);
       return TSDB_CODE_QRY_APP_ERROR;
     }
-
+    pOperator->status = OP_NOT_OPENED;
     return doSetStreamBlock(pOperator->pDownstream[0], input, type, id);
   } else {
     SStreamBlockScanInfo* pInfo = pOperator->info;
@@ -52,9 +52,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
 
       SSDataBlock* pDataBlock = input;
       pInfo->pRes->info = pDataBlock->info;
-      for(int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) {
-        pInfo->pRes->pDataBlock = pDataBlock->pDataBlock;
-      }
+      taosArrayClear(pInfo->pRes->pDataBlock);
+      taosArrayAddAll(pInfo->pRes->pDataBlock, pDataBlock->pDataBlock);
 
       // set current block valid.
       pInfo->blockValid = true;
@@ -121,7 +120,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
 
   // traverse to the streamscan node to add this table id
   SOperatorInfo* pInfo = pTaskInfo->pRoot;
-  while(pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+  while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     pInfo = pInfo->pDownstream[0];
   }
 

source/libs/parser/src/parTranslater.c

@@ -348,7 +348,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) {
 
 static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
   if (pVal->isDuration) {
-    if (parseAbsoluteDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) {
+    if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, pVal->node.resType.precision) != TSDB_CODE_SUCCESS) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
     }
   } else {