merge 3.0

CMakeLists.txt

@@ -7,7 +7,7 @@ project(
 )
 
 if (NOT DEFINED TD_SOURCE_DIR)
-  set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} )
+  set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
 endif()
 
 set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")

contrib/CMakeLists.txt

@@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
     cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
-# gnu regex
-if(${BUILD_GNUREGEX})
-    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+# iconv
+if(${BUILD_WITH_ICONV})
+    cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# msvc regex
+if(${BUILD_MSVCREGEX})
+    cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wcwidth
+if(${BUILD_WCWIDTH})
+    cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
@@ -99,8 +114,27 @@ if(${BUILD_TEST})
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
+    )
+    if(${TD_WINDOWS})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
+        )
+    endif(${TD_WINDOWS})
+    if(${TD_LINUX})
+        target_include_directories(
+            gtest
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
         )
+    endif(${TD_LINUX})
+    if(${TD_DARWIN})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
+        )
+    endif(${TD_DARWIN})
+    
+    
 endif(${BUILD_TEST})
 
 # cJson
@@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT})
     add_subdirectory(nuraft)
 endif(${BUILD_WITH_NURAFT})
 
+# pthread
+if(${BUILD_PTHREAD})
+    set(CMAKE_BUILD_TYPE release)
+    add_definitions(-DPTW32_STATIC_LIB) 
+    add_subdirectory(pthread)
+    set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
+    add_library(pthread STATIC IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+endif()
+
+# iconv
+if(${BUILD_WITH_ICONV})
+    add_subdirectory(iconv)
+endif(${BUILD_WITH_ICONV})
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    add_subdirectory(wingetopt)
+endif(${BUILD_WINGETOPT})
+
+# msvcregex
+if(${BUILD_MSVCREGEX})
+    add_library(msvcregex STATIC "")
+    target_sources(msvcregex
+        PRIVATE "msvcregex/regex.c"
+    )
+    target_include_directories(msvcregex
+        PRIVATE "msvcregex"
+    )
+    target_link_libraries(msvcregex 
+        INTERFACE Shell32
+    )
+    SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
+endif(${BUILD_MSVCREGEX})
+
+# msvcregex
+if(${BUILD_WCWIDTH})
+    add_library(wcwidth STATIC "")
+    target_sources(wcwidth
+        PRIVATE "wcwidth/wcwidth.c"
+    )
+    target_include_directories(wcwidth
+        PRIVATE "wcwidth"
+    )
+    SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
+endif(${BUILD_WCWIDTH})
+
 # CRAFT
 if(${BUILD_WITH_CRAFT})
     add_library(craft STATIC IMPORTED GLOBAL)
@@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
     target_link_libraries(sqlite 
         INTERFACE m 
         INTERFACE pthread 
+    )
+    if(NOT TD_WINDOWS)
+        target_link_libraries(sqlite 
             INTERFACE dl
         )
+    endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
 # pthread

contrib/test/craft/CMakeLists.txt

 add_executable(simulate_vnode "simulate_vnode.c")
 target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
+if(${BUILD_WINGETOPT})
+    target_link_libraries(simulate_vnode PUBLIC wingetopt)
+    target_include_directories(
+        simulate_vnode
+        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+    )
+endif()
\ No newline at end of file

contrib/test/tdev/src/main.c

@@ -6,43 +6,39 @@
 #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
 #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
 
-#define tPutA(buf, val)              \
-  ({                                 \
-    memcpy(buf, &val, sizeof(val));  \
-    POINTER_SHIFT(buf, sizeof(val)); \
-  })
+static inline void tPutA(void **buf, uint64_t val) {
+  memcpy(buf, &val, sizeof(val));
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutB(buf, val)                         \
-  ({                                            \
-    ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
-    ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
-    ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
-    ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
-    ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
-    ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
-    ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;  \
-    ((uint8_t *)buf)[0] = (val)&0xff;           \
-    POINTER_SHIFT(buf, sizeof(val));            \
-  })
+static inline void tPutB(void **buf, uint64_t val) {
+  ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
+  ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
+  ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
+  ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
+  ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
+  ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
+  ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
+  ((uint8_t *)buf)[0] = (val)&0xff;
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutC(buf, val)                \
-  ({                                   \
-    if (buf) {                         \
-      ((uint64_t *)buf)[0] = (val);    \
-      POINTER_SHIFT(buf, sizeof(val)); \
-    }                                  \
-    NULL;                                 \
-  })
+static inline void tPutC(void **buf, uint64_t val) {
+  if (buf) {
+    ((uint64_t *)buf)[0] = (val);
+    POINTER_SHIFT(buf, sizeof(val));
+  }
+  *buf = NULL;
+}
 
-#define tPutD(buf, val)                        \
-  ({                                           \
-    uint64_t tmp = val;                        \
-    for (size_t i = 0; i < sizeof(val); i++) { \
-      ((uint8_t *)buf)[i] = tmp & 0xff;        \
-      tmp >>= 8;                               \
-    }                                          \
-    POINTER_SHIFT(buf, sizeof(val));           \
-  })
+static inline void tPutD(void **buf, uint64_t val) {
+  uint64_t tmp = val;
+  for (size_t i = 0; i < sizeof(val); i++) {
+    ((uint8_t *)buf)[i] = tmp & 0xff;
+    tmp >>= 8;
+  }
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
 static inline void tPutE(void **buf, uint64_t val) {
   if (buf) {
@@ -61,7 +57,7 @@ static void func(T t) {
   switch (t) {
     case A:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutA(pBuf, val);
+        tPutA(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -69,7 +65,7 @@ static void func(T t) {
       break;
     case B:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutB(pBuf, val);
+        tPutB(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -77,7 +73,7 @@ static void func(T t) {
       break;
     case C:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutC(pBuf, val);
+        tPutC(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -85,7 +81,7 @@ static void func(T t) {
       break;
     case D:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutD(pBuf, val);
+        tPutD(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }

example/CMakeLists.txt

@@ -30,6 +30,7 @@ target_link_libraries(demoapi
 )
 
 target_include_directories(tmq
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

example/src/tmq.c

@@ -17,8 +17,8 @@
 #include <stdio.h>
 #include <string.h>
 #include <time.h>
-#include <unistd.h>
 #include "taos.h"
+#include "osSleep.h"
 
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
@@ -48,7 +48,6 @@ int32_t init_env() {
     return -1;
   }
   taos_free_result(pRes);
-  sleep(1);
 
   pRes = taos_query(pConn, "use abc1");
   if (taos_errno(pRes) != 0) {
@@ -60,7 +59,6 @@ int32_t init_env() {
   pRes =
       taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
   if (taos_errno(pRes) != 0) {
-    assert(0);
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
@@ -104,8 +102,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
-  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
+  pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -163,9 +161,10 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "group.id", "tg2");
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
-  tmq_conf_set(conf, "td.connect.db", "abc1");
+  /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
   tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+  assert(tmq);
   return tmq;
 }
 

examples/c/CMakeLists.txt

 PROJECT(TDengine)
 
 IF (TD_LINUX)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo apitest.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
@@ -13,7 +13,7 @@ IF (TD_LINUX)
   TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
 ENDIF ()
 IF (TD_DARWIN)
-  INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc  ${TD_COMMUNITY_DIR}/inc)
+  INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc  ${TD_SOURCE_DIR}/inc)
   AUX_SOURCE_DIRECTORY(. SRC)
   ADD_EXECUTABLE(demo demo.c)
   TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)

include/client/taos.h

@@ -27,9 +27,6 @@ typedef void   TAOS;
 typedef void   TAOS_STMT;
 typedef void   TAOS_RES;
 typedef void **TAOS_ROW;
-#if 0
-typedef void   TAOS_STREAM;
-#endif
 typedef void   TAOS_SUB;
 
 // Data type definition
@@ -88,7 +85,11 @@ typedef struct taosField {
   int32_t bytes;
 } TAOS_FIELD;
 
-#define DLL_EXPORT
+#ifdef WINDOWS
+  #define DLL_EXPORT    __declspec(dllexport)
+#else
+  #define DLL_EXPORT 
+#endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
@@ -192,12 +193,6 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-#if 0
-DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
-                                         int64_t stime, void *param, void (*callback)(void *));
-DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
-#endif
-
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
 
@@ -234,15 +229,11 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
-DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
+DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
-#if 0
-DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups);
-DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups);
-#endif
 DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
@@ -269,7 +260,7 @@ DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
 // TODO
 #if 0
-DLL_EXPORT char   *tmq_get_block_table_name(TAOS_RES *res);
+DLL_EXPORT char   *tmq_get_table_name(TAOS_RES *res);
 #endif
 
 #if 0

include/common/tcommon.h

@@ -99,6 +99,15 @@ typedef struct SColumnInfoData {
   };
 } SColumnInfoData;
 
+typedef struct SQueryTableDataCond {
+  STimeWindow  twindow;
+  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  bool         loadExternalRows;  // load external rows or not
+  int32_t      type;              // data block load type:
+} SQueryTableDataCond;
+
 void*   blockDataDestroy(SSDataBlock* pBlock);
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
@@ -229,7 +238,6 @@ typedef struct SResSchame {
   char    name[TSDB_COL_NAME_LEN];
 } SResSchema;
 
-// TODO move away to executor.h
 typedef struct SExprBasicInfo {
   SResSchema   resSchema;
   int16_t      numOfParams;  // argument value of each function

include/common/tglobal.h

@@ -123,9 +123,9 @@ extern SDiskCfg tsDiskCfg[];
 
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
-                      const char *apolloUrl, SArray *pArgs, bool tsc);
-int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc);
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
+                      char *apolloUrl, SArray *pArgs, bool tsc);
+int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
 void    taosCleanupCfg();
 void    taosCfgDynamicOptions(const char *option, const char *value);
 void    taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);

include/common/tmsg.h

@@ -229,6 +229,14 @@ typedef struct {
 typedef struct {
   int32_t totalLen;
   int32_t len;
+  // head of SSubmitBlk
+  int64_t uid;        // table unique id
+  int64_t suid;       // stable id
+  int32_t sversion;   // data schema version
+  int32_t dataLen;    // data part length, not including the SSubmitBlk head
+  int32_t schemaLen;  // schema length, if length is 0, no schema exists
+  int16_t numOfRows;  // total number of rows in current submit block
+  // head of SSubmitBlk
   const void* pMsg;
 } SSubmitMsgIter;
 
@@ -237,6 +245,15 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
 int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
 STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 
+// TODO: KEEP one suite of iterator API finally.
+// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
+// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
+// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter
+int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
+int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
+int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
+STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter);
+
 typedef struct {
   int32_t index;  // index of failed block in submit blocks
   int32_t vnode;  // vnode index of failed block
@@ -275,7 +292,6 @@ typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
   float   xFilesFactor;
-  int32_t aggregationMethod;
   int32_t delay;
   int32_t ttl;
   int32_t numOfColumns;
@@ -1316,7 +1332,7 @@ typedef struct {
 typedef struct {
   int64_t consumerId;
   char    cgroup[TSDB_CGROUP_LEN];
-  SArray* topicNames;  // SArray<char*>
+  SArray* topicNames;  // SArray<char**>
 } SCMSubscribeReq;
 
 static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@@ -1495,10 +1511,8 @@ typedef struct {
   int32_t    delay;
   int32_t    qmsg1Len;
   int32_t    qmsg2Len;
-  func_id_t* pFuncIds;
-  char*      qmsg1;  // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1
-  char*      qmsg2;  // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2
-  int8_t     nFuncIds;
+  char*      qmsg1;  // pAst1:qmsg1:SRetention1 => trigger aggr task1
+  char*      qmsg2;  // pAst2:qmsg2:SRetention2 => trigger aggr task2
 } SRSmaParam;
 
 typedef struct SVCreateTbReq {
@@ -2052,80 +2066,6 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
   return buf;
 }
 
-typedef struct {
-  int64_t leftForVer;
-  int32_t vgId;
-  int32_t epoch;
-  int64_t consumerId;
-  char    topicName[TSDB_TOPIC_FNAME_LEN];
-} SMqCancelConnReq;
-
-static FORCE_INLINE int32_t tEncodeSMqCancelConnReq(void** buf, const SMqCancelConnReq* pReq) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
-  tlen += taosEncodeFixedI32(buf, pReq->vgId);
-  tlen += taosEncodeFixedI32(buf, pReq->epoch);
-  tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeString(buf, pReq->topicName);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqCancelConnReq(void* buf, SMqCancelConnReq* pReq) {
-  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
-  buf = taosDecodeFixedI32(buf, &pReq->vgId);
-  buf = taosDecodeFixedI32(buf, &pReq->epoch);
-  buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeStringTo(buf, pReq->topicName);
-  return buf;
-}
-
-typedef struct {
-  int8_t reserved;
-} SMqCancelConnRsp;
-
-typedef struct {
-  int64_t leftForVer;
-  int32_t vgId;
-  int64_t oldConsumerId;
-  int64_t newConsumerId;
-  char*   topic;
-} SMqMVRebReq;
-
-static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
-  tlen += taosEncodeFixedI32(buf, pReq->vgId);
-  tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
-  tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
-  tlen += taosEncodeString(buf, pReq->topic);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) {
-  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
-  buf = taosDecodeFixedI32(buf, &pReq->vgId);
-  buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
-  buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
-  buf = taosDecodeString(buf, &pReq->topic);
-  return buf;
-}
-
-typedef struct {
-  SMsgHead header;
-  int32_t  vgId;
-  int64_t  consumerId;
-  char     topicName[TSDB_TOPIC_FNAME_LEN];
-  char     cgroup[TSDB_CGROUP_LEN];
-} SMqSetCVgRsp;
-
-typedef struct {
-  SMsgHead header;
-  int32_t  vgId;
-  int64_t  consumerId;
-  char     topicName[TSDB_TOPIC_FNAME_LEN];
-  char     cgroup[TSDB_CGROUP_LEN];
-} SMqMVRebRsp;
-
 typedef struct {
   int32_t vgId;
   int64_t offset;
@@ -2152,6 +2092,24 @@ typedef struct {
   SSchema* pSchema;
 } SSchemaWrapper;
 
+static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
+  SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
+  if (pSW == NULL) return pSW;
+  pSW->nCols = pSchemaWrapper->nCols;
+  pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
+  if (pSW->pSchema == NULL) {
+    taosMemoryFree(pSW);
+    return NULL;
+  }
+  memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema));
+  return pSW;
+}
+
+static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) {
+  taosMemoryFree(pSchemaWrapper->pSchema);
+  taosMemoryFree(pSchemaWrapper);
+}
+
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pSchema->type);
@@ -2162,13 +2120,13 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema
   return tlen;
 }
 
-static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) {
+static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) {
   buf = taosDecodeFixedI8(buf, &pSchema->type);
   buf = taosDecodeFixedI8(buf, &pSchema->flags);
   buf = taosDecodeFixedI32(buf, &pSchema->bytes);
   buf = taosDecodeFixedI16(buf, &pSchema->colId);
   buf = taosDecodeStringTo(buf, pSchema->name);
-  return buf;
+  return (void*)buf;
 }
 
 static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
@@ -2198,7 +2156,7 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
   return tlen;
 }
 
-static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) {
+static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
   buf = taosDecodeFixedU32(buf, &pSW->nCols);
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) {
@@ -2208,7 +2166,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pS
   for (int32_t i = 0; i < pSW->nCols; i++) {
     buf = taosDecodeSSchema(buf, &pSW->pSchema[i]);
   }
-  return buf;
+  return (void*)buf;
 }
 
 static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) {
@@ -2615,6 +2573,10 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
       void*   data = taosArrayGetP(pRsp->blockData, i);
       tlen += taosEncodeFixedI32(buf, bLen);
       tlen += taosEncodeBinary(buf, data, bLen);
+      if (pRsp->withSchema) {
+        SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i);
+        tlen += taosEncodeSSchemaWrapper(buf, pSW);
+      }
     }
   }
   return tlen;
@@ -2627,6 +2589,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
   buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
   pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
   pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
   if (pRsp->blockNum != 0) {
     buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
     buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
@@ -2639,6 +2602,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
       buf = taosDecodeBinary(buf, &data, bLen);
       taosArrayPush(pRsp->blockDataLen, &bLen);
       taosArrayPush(pRsp->blockData, &data);
+      if (pRsp->withSchema) {
+        SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
+        buf = taosDecodeSSchemaWrapper(buf, pSW);
+        taosArrayPush(pRsp->blockSchema, &pSW);
+      }
     }
   }
   return (void*)buf;

include/common/tmsgdef.h

@@ -145,7 +145,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-mq-ask-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)

include/libs/function/function.h

@@ -202,7 +202,7 @@ typedef struct SqlFunctionCtx {
   SPoint1                end;
   SFuncExecFuncs         fpSet;
   SScalarFuncExecFuncs   sfp;
-  SExprInfo             *pExpr;
+  struct SExprInfo      *pExpr;
   struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   int32_t                curBufPage;

include/libs/function/functionMgt.h

@@ -125,15 +125,15 @@ typedef struct SFmGetFuncInfoParam {
   struct SCatalog* pCtg;
   void *pRpc;
   const SEpSet* pMgmtEps;
+  char* pErrBuf;
+  int32_t errBufLen;
 } SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
-
-int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
 
 bool fmIsAggFunc(int32_t funcId);
 bool fmIsScalarFunc(int32_t funcId);

include/libs/nodes/querynodes.h

@@ -25,8 +25,10 @@ extern "C" {
 #include "tvariant.h"
 
 #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
-#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
-#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
+#define TABLE_META_SIZE(pMeta) \
+  (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
+#define VGROUPS_INFO_SIZE(pInfo) \
+  (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
 
 typedef struct SRawExprNode {
   ENodeType nodeType;
@@ -46,13 +48,10 @@ typedef struct SExprNode {
   ENodeType type;
   SDataType resType;
   char      aliasName[TSDB_COL_NAME_LEN];
-  SNodeList* pAssociationList;
+  SArray*   pAssociation;
 } SExprNode;
 
-typedef enum EColumnType {
-  COLUMN_TYPE_COLUMN = 1,
-  COLUMN_TYPE_TAG
-} EColumnType;
+typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType;
 
 typedef struct SColumnNode {
   SExprNode   node;  // QUERY_NODE_COLUMN
@@ -81,7 +80,6 @@ typedef struct SValueNode {
   char*     literal;
   bool      isDuration;
   bool      translate;
-  bool genByCalc;
   int16_t   placeholderNo;
   union {
     bool     b;
@@ -100,7 +98,6 @@ typedef struct SOperatorNode {
   SNode*        pRight;
 } SOperatorNode;
 
-
 typedef struct SLogicConditionNode {
   SExprNode           node;  // QUERY_NODE_LOGIC_CONDITION
   ELogicConditionType condType;
@@ -119,9 +116,7 @@ typedef struct SFunctionNode {
   int32_t    funcId;
   int32_t    funcType;
   SNodeList* pParameterList;
-
-  int8_t udfFuncType; //TODO: fill by parser/planner
-  int32_t bufSize; //TODO: fill by parser/planner
+  int32_t    udfBufSize;
 } SFunctionNode;
 
 typedef struct STableNode {
@@ -138,7 +133,7 @@ typedef struct SRealTableNode {
   STableNode         table;  // QUERY_NODE_REAL_TABLE
   struct STableMeta* pMeta;
   SVgroupsInfo*      pVgroupList;
-  char useDbName[TSDB_DB_NAME_LEN];
+  char               qualDbName[TSDB_DB_NAME_LEN];  // SHOW qualDbName.TABLES
   double             ratio;
 } SRealTableNode;
 
@@ -147,9 +142,7 @@ typedef struct STempTableNode {
   SNode*     pSubquery;
 } STempTableNode;
 
-typedef enum EJoinType {
-  JOIN_TYPE_INNER = 1
-} EJoinType;
+typedef enum EJoinType { JOIN_TYPE_INNER = 1 } EJoinType;
 
 typedef struct SJoinTableNode {
   STableNode table;  // QUERY_NODE_JOIN_TABLE
@@ -159,9 +152,7 @@ typedef struct SJoinTableNode {
   SNode*     pOnCond;
 } SJoinTableNode;
 
-typedef enum EGroupingSetType {
-  GP_TYPE_NORMAL = 1
-} EGroupingSetType;
+typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
 
 typedef struct SGroupingSetNode {
   ENodeType        type;  // QUERY_NODE_GROUPING_SET
@@ -169,16 +160,9 @@ typedef struct SGroupingSetNode {
   SNodeList*       pParameterList;
 } SGroupingSetNode;
 
-typedef enum EOrder {
-  ORDER_ASC = 1,
-  ORDER_DESC
-} EOrder;
+typedef enum EOrder { ORDER_ASC = 1, ORDER_DESC } EOrder;
 
-typedef enum ENullOrder {
-  NULL_ORDER_DEFAULT = 1,
-  NULL_ORDER_FIRST,
-  NULL_ORDER_LAST
-} ENullOrder;
+typedef enum ENullOrder { NULL_ORDER_DEFAULT = 1, NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder;
 
 typedef struct SOrderByExprNode {
   ENodeType  type;  // QUERY_NODE_ORDER_BY_EXPR
@@ -245,12 +229,10 @@ typedef struct SSelectStmt {
   char        stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t     precision;
   bool        isEmptyResult;
+  bool        hasAggFuncs;
 } SSelectStmt;
 
-typedef enum ESetOperatorType {
-  SET_OP_TYPE_UNION_ALL = 1,
-  SET_OP_TYPE_UNION
-} ESetOperatorType;
+typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
 
 typedef struct SSetOperator {
   ENodeType        type;  // QUERY_NODE_SET_OPERATOR
@@ -274,7 +256,6 @@ typedef enum ESqlClause {
   SQL_CLAUSE_ORDER_BY
 } ESqlClause;
 
-
 typedef enum {
   PAYLOAD_TYPE_KV = 0,
   PAYLOAD_TYPE_RAW = 1,
@@ -284,7 +265,7 @@ typedef struct SVgDataBlocks {
   SVgroupInfo vg;
   int32_t     numOfTables;  // number of tables in current submit block
   uint32_t    size;
-  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
+  char*       pData;  // SMsgDesc + SSubmitReq + SSubmitBlk + ...
 } SVgDataBlocks;
 
 typedef struct SVnodeModifOpStmt {
@@ -327,9 +308,9 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
-void* nodesGetValueFromNode(SValueNode *pNode);
-char* nodesGetStrValueFromNode(SValueNode *pNode);
-char *getFillModeString(EFillMode mode);
+void* nodesGetValueFromNode(SValueNode* pNode);
+char* nodesGetStrValueFromNode(SValueNode* pNode);
+char* getFillModeString(EFillMode mode);
 void  valueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
 
 #ifdef __cplusplus

include/libs/sync/sync.h

@@ -158,6 +158,8 @@ typedef enum {
 
 int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
+bool syncEnvIsStart();
+
 extern int32_t sDebugFlag;
 
 //-----------------------------------------

include/util/taoserror.h

@@ -573,7 +573,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x2602)
 #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN          TAOS_DEF_ERROR_CODE(0, 0x2603)
 #define TSDB_CODE_PAR_WRONG_VALUE_TYPE          TAOS_DEF_ERROR_CODE(0, 0x2604)
-#define TSDB_CODE_PAR_INVALID_FUNTION           TAOS_DEF_ERROR_CODE(0, 0x2605)
 #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION  TAOS_DEF_ERROR_CODE(0, 0x2608)
 #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT    TAOS_DEF_ERROR_CODE(0, 0x2609)
 #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION   TAOS_DEF_ERROR_CODE(0, 0x260A)
@@ -633,6 +632,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_FUNTION_PARA_NUM         TAOS_DEF_ERROR_CODE(0, 0x2801)
 #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE        TAOS_DEF_ERROR_CODE(0, 0x2802)
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
+#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2604)
 
 #ifdef __cplusplus
 }

include/util/tconfig.h

@@ -30,6 +30,7 @@ typedef enum {
   CFG_STYPE_CFG_FILE,
   CFG_STYPE_ENV_FILE,
   CFG_STYPE_ENV_VAR,
+  CFG_STYPE_ENV_CMD,
   CFG_STYPE_APOLLO_URL,
   CFG_STYPE_ARG_LIST,
   CFG_STYPE_TAOS_OPTIONS
@@ -82,7 +83,7 @@ typedef struct SConfig {
 } SConfig;
 
 SConfig *cfgInit();
-int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
+int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
 int32_t  cfgLoadFromArray(SConfig *pCfg, SArray *pArgs);  // SConfigPair
 void     cfgCleanup(SConfig *pCfg);
 
@@ -105,6 +106,8 @@ const char *cfgDtypeStr(ECfgDataType type);
 
 void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
 
+int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tenv.h

+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_ENV_H_
+#define _TD_ENV_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen);
+int32_t taosEnvToCfg(const char *envStr, char *cfgStr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_ENV_H_*/

include/util/tjson.h

@@ -49,6 +49,8 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
 int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
 
 SJson*  tjsonGetObjectItem(const SJson* pJson, const char* pName);
+int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
+int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
 int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
 int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
 int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
@@ -81,6 +83,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
 bool   tjsonValidateJson(const char* pJson);
+const char* tjsonGetError();
 
 #ifdef __cplusplus
 }

include/util/ttimer.h

@@ -41,10 +41,6 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *han
 
 void taosTmrCleanUp(void *handle);
 
-int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);
-
-void taosUninitTimer();
-
 #ifdef __cplusplus
 }
 #endif

source/client/CMakeLists.txt

 aux_source_directory(src CLIENT_SRC)
-add_library(taos SHARED ${CLIENT_SRC})
+if(TD_WINDOWS)
+    add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in)
+else()
+    add_library(taos SHARED ${CLIENT_SRC})
+endif ()
 target_include_directories(
     taos
     PUBLIC "${TD_SOURCE_DIR}/include/client"
@@ -10,6 +14,13 @@ target_link_libraries(
     INTERFACE api
     PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
 )
+if(TD_WINDOWS)
+    set_target_properties(taos 
+        PROPERTIES 
+        LINK_FLAGS 
+        /DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/taos.def
+    )
+endif ()
 
 set_target_properties(
     taos

source/client/inc/clientInt.h

@@ -231,6 +231,10 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver
   msg->resIter++;
   if (msg->resIter < msg->rsp.blockNum) {
     SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    if (msg->rsp.withSchema) {
+      SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
+      setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
+    }
     setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
     return &msg->resInfo;
   }

source/client/src/clientEnv.c

@@ -254,12 +254,12 @@ void taos_init_imp(void) {
 
   deltaToUtcInitOnce();
 
-  if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) {
+  if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
     tscInitRes = -1;
     return;
   }
 
-  if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
     tscInitRes = -1;
     return;
   }

source/client/src/clientHb.c

@@ -14,9 +14,9 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "scheduler.h"
 #include "trpc.h"
 
 static SClientHbMgr clientHbMgr = {0};
@@ -110,7 +110,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
 static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
   SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == info) {
-    tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType);
+    tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid,
+            pRsp->connKey.connType);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -471,10 +472,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
     pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
   }
 
-//  if (code) {
-//    taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq);
-//    taosMemoryFreeClear(pBatchReq);
-//  }
+  //  if (code) {
+  //    taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq);
+  //    taosMemoryFreeClear(pBatchReq);
+  //  }
 
   return pBatchReq;
 }
@@ -641,7 +642,6 @@ void appHbMgrCleanup(void) {
     taosHashCleanup(pTarget->activeInfo);
     pTarget->activeInfo = NULL;
 
-
     pIter = taosHashIterate(pTarget->connInfo, NULL);
     while (pIter != NULL) {
       SHbConnInfo *info = pIter;
@@ -668,13 +668,13 @@ int hbMgrInit() {
   hbMgrInitHandle();
 
   // init backgroud thread
-  //hbCreateThread();
+  /*hbCreateThread();*/
 
   return 0;
 }
 
 void hbMgrCleanUp() {
-  //hbStopThread();
+  // hbStopThread();
 
   // destroy all appHbMgr
   int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0);

source/client/src/clientImpl.c

@@ -226,16 +226,14 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
   pRequest->type = pQuery->msgType;
-  SPlanContext cxt = {
-    .queryId = pRequest->requestId,
+  SPlanContext cxt = {.queryId = pRequest->requestId,
                       .acctId = pRequest->pTscObj->acctId,
                       .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
                       .pAstRoot = pQuery->pRoot,
                       .showRewrite = pQuery->showRewrite,
                       .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
                       .pMsg = pRequest->msgBuf,
-    .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
-  };
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
   int32_t      code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
   if (TSDB_CODE_SUCCESS == code) {
     code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
@@ -247,6 +245,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   ASSERT(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
+  // TODO handle memory leak
   pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
   pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
 
@@ -840,12 +839,12 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     return code;
   }
 
-  char* p = (char*) pResultInfo->pData;
+  char* p = (char*)pResultInfo->pData;
 
-  int32_t dataLen = *(int32_t*) p;
+  int32_t dataLen = *(int32_t*)p;
   p += sizeof(int32_t);
 
-  uint64_t groupId = *(uint64_t*) p;
+  uint64_t groupId = *(uint64_t*)p;
   p += sizeof(uint64_t);
 
   int32_t* colLength = (int32_t*)p;

source/client/src/clientMain.c

@@ -120,7 +120,7 @@ const char *taos_errstr(TAOS_RES *res) {
     return (const char *)tstrerror(terrno);
   }
 
-  if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
+  if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
     return (const char *)tstrerror(pRequest->code);

source/client/src/taos.def

+taos_cleanup
+taos_options
+taos_set_config
+taos_connect
+taos_connect_l
+taos_connect_auth
+taos_close
+taos_data_type
+taos_stmt_init
+taos_stmt_prepare
+taos_stmt_set_tbname_tags
+taos_stmt_set_tbname
+taos_stmt_set_sub_tbname
+taos_stmt_is_insert
+taos_stmt_num_params
+taos_stmt_get_param
+taos_stmt_bind_param
+taos_stmt_bind_param_batch
+taos_stmt_bind_single_param_batch
+taos_stmt_add_batch
+taos_stmt_execute
+taos_stmt_use_result
+taos_stmt_close
+taos_stmt_errstr
+taos_stmt_affected_rows
+taos_stmt_affected_rows_once
+taos_query
+taos_query_l
+taos_fetch_row
+taos_result_precision
+taos_free_result
+taos_field_count
+taos_num_fields
+taos_affected_rows
+taos_fetch_fields
+taos_select_db
+taos_print_row
+taos_stop_query
+taos_is_null
+taos_is_update_query
+taos_fetch_block
+taos_fetch_block_s
+taos_fetch_raw_block
+taos_get_column_data_offset
+taos_validate_sql
+taos_reset_current_db
+taos_fetch_lengths
+taos_result_block
+taos_get_server_info
+taos_get_client_info
+taos_errstr
+taos_errno
+taos_query_a
+taos_fetch_rows_a
+taos_subscribe
+taos_consume
+taos_unsubscribe
+taos_load_table_info
+taos_schemaless_insert
+tmq_list_new
+tmq_list_append
+tmq_list_destroy
+tmq_list_get_size
+tmq_list_to_c_array
+tmq_consumer_new
+tmq_err2str
+tmq_subscribe
+tmq_unsubscribe
+tmq_subscription
+tmq_consumer_poll
+tmq_consumer_close
+tmq_commit
+tmq_conf_new
+tmq_conf_set
+tmq_conf_destroy
+tmq_conf_set_offset_commit_cb
+tmq_get_topic_name
+tmq_get_vgroup_id
+tmq_create_stream
+taos_check_server_status
\ No newline at end of file

source/client/src/taos.rc.in

+1 VERSIONINFO
+ FILEVERSION ${TD_VER_NUMBER}
+ PRODUCTVERSION ${TD_VER_NUMBER}
+ FILEFLAGSMASK 0x17L
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x0L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "FileDescription", "Native C Driver for TDengine"
+            VALUE "FileVersion", "${TD_VER_NUMBER}"
+            VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})"
+            VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data"
+            VALUE "OriginalFilename", ""
+            VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})"
+            VALUE "ProductVersion", "${TD_VER_NUMBER}"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
\ No newline at end of file

source/common/src/tdatablock.c

@@ -78,9 +78,13 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo
   ASSERT(pColumnInfoData != NULL);
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     return pColumnInfoData->varmeta.length;
+  } else {
+    if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) {
+      return 0;
     } else {
       return pColumnInfoData->info.bytes * numOfRows;
     }
+  }
 }
 
 int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {

source/common/src/tglobal.c

@@ -220,7 +220,7 @@ struct SConfig *taosGetCfg() {
   return tsCfg;
 }
 
-static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
+static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) {
   char cfgDir[PATH_MAX] = {0};
   char cfgFile[PATH_MAX + 100] = {0};
 
@@ -231,6 +231,8 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
     tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
   }
 
+  if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl);
+
   if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
     uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
     return -1;
@@ -251,6 +253,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
     return -1;
   }
 
+  if (cfgLoad(pCfg, CFG_STYPE_ENV_CMD, envCmd) != 0) {
+    uError("failed to load from cmd env variables since %s", terrstr());
+    return -1;
+  }
+
   return 0;
 }
 
@@ -325,8 +332,8 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
   if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
   if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
-  if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
+  if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1;
   if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
@@ -572,8 +579,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   return 0;
 }
 
-int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile,
-                      const char *apolloUrl, SArray *pArgs, bool tsc) {
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
+                      char *apolloUrl, SArray *pArgs, bool tsc) {
   osDefaultInit();
 
   SConfig *pCfg = cfgInit();
@@ -588,7 +595,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
     if (taosAddServerLogCfg(pCfg) != 0) return -1;
   }
 
-  if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+  if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
     uError("failed to load cfg since %s", terrstr());
     cfgCleanup(pCfg);
     return -1;
@@ -625,7 +632,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
   return 0;
 }
 
-int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) {
+int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
   if (tsCfg != NULL) return 0;
   tsCfg = cfgInit();
 
@@ -640,7 +647,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
   }
   taosAddSystemCfg(tsCfg);
 
-  if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) {
+  if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
     uError("failed to load cfg since %s", terrstr());
     cfgCleanup(tsCfg);
     tsCfg = NULL;

source/common/src/tmsg.c

@@ -94,6 +94,86 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
   }
 }
 
+// TODO: KEEP one suite of iterator API finally.
+// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
+// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
+// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter
+
+int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
+  if (pMsg == NULL) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    return -1;
+  }
+
+  pIter->totalLen = htonl(pMsg->length);
+  ASSERT(pIter->totalLen > 0);
+  pIter->len = 0;
+  pIter->pMsg = pMsg;
+  if (pIter->totalLen <= sizeof(SSubmitReq)) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
+  ASSERT(pIter->len >= 0);
+
+  if (pIter->len == 0) {
+    pIter->len += sizeof(SSubmitReq);
+  } else {
+    if (pIter->len >= pIter->totalLen) {
+      ASSERT(0);
+    }
+
+    SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+    pIter->len += (sizeof(SSubmitBlk) + pIter->dataLen + pIter->schemaLen);
+    ASSERT(pIter->len > 0);
+  }
+
+  if (pIter->len > pIter->totalLen) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    *pPBlock = NULL;
+    return -1;
+  }
+
+  if (pIter->len == pIter->totalLen) {
+    *pPBlock = NULL;
+  } else {
+    *pPBlock = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+    pIter->uid = htobe64((*pPBlock)->uid);
+    pIter->suid = htobe64((*pPBlock)->suid);
+    pIter->sversion = htonl((*pPBlock)->sversion);
+    pIter->dataLen = htonl((*pPBlock)->dataLen);
+    pIter->schemaLen = htonl((*pPBlock)->schemaLen);
+    pIter->numOfRows = htons((*pPBlock)->numOfRows);
+  }
+  return 0;
+}
+
+int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+  if (pMsgIter->dataLen <= 0) return -1;
+  pIter->totalLen = pMsgIter->dataLen;
+  pIter->len = 0;
+  pIter->row = (STSRow *)(pBlock->data + pMsgIter->schemaLen);
+  return 0;
+}
+
+STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) {
+  STSRow *row = pIter->row;
+
+  if (pIter->len >= pIter->totalLen) {
+    return NULL;
+  } else {
+    pIter->len += TD_ROW_LEN(row);
+    if (pIter->len < pIter->totalLen) {
+      pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row));
+    }
+    return row;
+  }
+}
+
 int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) {
   if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1;
   if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
@@ -442,10 +522,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         tlen += taosEncodeFixedI32(buf, param->delay);
-        tlen += taosEncodeFixedI8(buf, param->nFuncIds);
-        for (int8_t i = 0; i < param->nFuncIds; ++i) {
-          tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
-        }
         tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
         if (param->qmsg1Len > 0) {
           tlen += taosEncodeString(buf, param->qmsg1);
@@ -475,10 +551,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
         tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         tlen += taosEncodeFixedI32(buf, param->delay);
-        tlen += taosEncodeFixedI8(buf, param->nFuncIds);
-        for (int8_t i = 0; i < param->nFuncIds; ++i) {
-          tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
-        }
       }
       break;
     default:
@@ -521,13 +593,6 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
-        buf = taosDecodeFixedI8(buf, &param->nFuncIds);
-        if (param->nFuncIds > 0) {
-          param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t));
-          for (int8_t i = 0; i < param->nFuncIds; ++i) {
-            buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
-          }
-        }
         buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
         if (param->qmsg1Len > 0) {
           buf = taosDecodeString(buf, &param->qmsg1);
@@ -561,15 +626,6 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
         buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
-        buf = taosDecodeFixedI8(buf, &param->nFuncIds);
-        if (param->nFuncIds > 0) {
-          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
-          for (int8_t i = 0; i < param->nFuncIds; ++i) {
-            buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
-          }
-        } else {
-          param->pFuncIds = NULL;
-        }
       } else {
         pReq->ntbCfg.pRSmaParam = NULL;
       }
@@ -632,7 +688,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->delay) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
@@ -687,7 +742,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
   if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;

source/dnode/mgmt/exe/dmMain.c

@@ -17,13 +17,24 @@
 #include "dmImp.h"
 #include "tconfig.h"
 
+#define DM_APOLLO_URL    "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'."
+#define DM_CFG_DIR       "Configuration directory."
+#define DM_DMP_CFG       "Dump configuration."
+#define DM_ENV_CMD       "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
+#define DM_ENV_FILE      "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
+#define DM_NODE_TYPE     "Startup type of the node, default is 0."
+#define DM_MACHINE_CODE  "Get machine code."
+#define DM_VERSION       "Print program version."
+#define DM_EMAIL         "<support@taosdata.com>"
 static struct {
   bool         dumpConfig;
   bool         generateGrant;
   bool         printAuth;
   bool         printVersion;
+  bool         printHelp;
   char         envFile[PATH_MAX];
   char         apolloUrl[PATH_MAX];
+  const char **envCmd;
   SArray      *pArgs;  // SConfigPair
   SDnode      *pDnode;
   EDndNodeType ntype;
@@ -56,6 +67,9 @@ static void dmSetSignalHandle() {
 }
 
 static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
+  int32_t cmdEnvIndex = 0;
+  global.envCmd = taosMemoryMalloc(argc-1);
+  memset(global.envCmd, 0, argc-1);
   for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
       if (i < argc - 1) {
@@ -70,7 +84,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
       }
     } else if (strcmp(argv[i], "-a") == 0) {
       tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
-    } else if (strcmp(argv[i], "-e") == 0) {
+    } else if (strcmp(argv[i], "-E") == 0) {
       tstrncpy(global.envFile, argv[++i], PATH_MAX);
     } else if (strcmp(argv[i], "-n") == 0) {
       global.ntype = atoi(argv[++i]);
@@ -84,6 +98,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
       global.dumpConfig = true;
     } else if (strcmp(argv[i], "-V") == 0) {
       global.printVersion = true;
+    } else if (strcmp(argv[i], "-e") == 0) {
+      global.envCmd[cmdEnvIndex] = argv[++i];
+      cmdEnvIndex++;
+    } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?")) {
+      global.printHelp = true;
     } else {
     }
   }
@@ -104,6 +123,21 @@ static void dmPrintVersion() {
   printf("buildInfo: %s\n", buildinfo);
 }
 
+static void dmPrintHelp() {
+  char indent[] = "  ";
+  printf("Usage: taosd [OPTION...] \n\n");
+  printf("%s%s%s%s\n", indent, "-a,", indent, DM_APOLLO_URL);
+  printf("%s%s%s%s\n", indent, "-c,", indent, DM_CFG_DIR);
+  printf("%s%s%s%s\n", indent, "-C,", indent, DM_DMP_CFG);
+  printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
+  printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
+  printf("%s%s%s%s\n", indent, "-n,", indent, DM_NODE_TYPE);
+  printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
+  printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
+
+  printf("\n\nReport bugs to %s.\n", DM_EMAIL);
+}
+
 static void dmDumpCfg() {
   SConfig *pCfg = taosGetCfg();
   cfgDumpCfg(pCfg, 0, true);
@@ -129,7 +163,7 @@ static SDnodeOpt dmGetOpt() {
 static int32_t dmInitLog() {
   char logName[12] = {0};
   snprintf(logName, sizeof(logName), "%slog", dmLogName(global.ntype));
-  return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0);
+  return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
 }
 
 static void dmSetProcInfo(int32_t argc, char **argv) {
@@ -168,6 +202,10 @@ static int32_t dmRunDnode() {
   return code;
 }
 
+static void taosCleanupArgs() {
+  if (global.envCmd != NULL) taosMemoryFree(global.envCmd);
+}
+
 int main(int argc, char const *argv[]) {
   if (!taosCheckSystemIsSmallEnd()) {
     printf("failed to start since on non-small-end machines\n");
@@ -176,26 +214,37 @@ int main(int argc, char const *argv[]) {
 
   if (dmParseArgs(argc, argv) != 0) {
     printf("failed to start since parse args error\n");
+    taosCleanupArgs();
     return -1;
   }
 
   if (global.generateGrant) {
     dmGenerateGrant();
+    taosCleanupArgs();
+    return 0;
+  }
+
+  if (global.printHelp) {
+    dmPrintHelp();
+    taosCleanupArgs();
     return 0;
   }
 
   if (global.printVersion) {
     dmPrintVersion();
+    taosCleanupArgs();
     return 0;
   }
 
   if (dmInitLog() != 0) {
-    dError("failed to start since init log error");
+    printf("failed to start since init log error");
+    taosCleanupArgs();
     return -1;
   }
 
-  if (taosInitCfg(configDir, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
+  if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
     dError("failed to start since read config error");
+    taosCleanupArgs();
     return -1;
   }
 
@@ -203,9 +252,11 @@ int main(int argc, char const *argv[]) {
     dmDumpCfg();
     taosCleanupCfg();
     taosCloseLog();
+    taosCleanupArgs();
     return 0;
   }
 
   dmSetProcInfo(argc, (char **)argv);
+  taosCleanupArgs();
   return dmRunDnode();
 }

source/dnode/mgmt/interface/src/dmFile.c

@@ -124,7 +124,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
   TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s", filepath, terrstr());
+    dError("failed to open file:%s since %s", filepath, terrstr());
     return NULL;
   }
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -202,6 +202,17 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
     return code;
   }
 
+  code = vnodeStart(pImpl);
+  if (code != 0) {
+    tFreeSCreateVnodeReq(&createReq);
+    dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
+    vnodeClose(pImpl);
+    vnodeDestroy(path, pMgmt->pTfs);
+    terrno = code;
+    return code;
+  }
+
+
   code = vmWriteVnodesToFile(pMgmt);
   if (code != 0) {
     tFreeSCreateVnodeReq(&createReq);

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -74,12 +74,6 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
     return -1;
   }
 
-  // sync integration
-  vnodeSyncSetQ(pImpl, NULL);
-  vnodeSyncSetRpc(pImpl, NULL);
-  int32_t ret = vnodeSyncStart(pImpl);
-  assert(ret == 0);
-
   taosWLockLatch(&pMgmt->latch);
   int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
   taosWUnLockLatch(&pMgmt->latch);
@@ -153,6 +147,7 @@ static void *vmOpenVnodeFunc(void *param) {
       pThread->failed++;
     } else {
       vmOpenVnode(pMgmt, pCfg, pImpl);
+      //vnodeStart(pImpl);
       dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
       pThread->opened++;
     }
@@ -364,10 +359,52 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) {
   return 0;
 }
 
+static int32_t vmStart(SMgmtWrapper *pWrapper) {
+  dDebug("vnode-mgmt start to run");
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+
+  taosRLockLatch(&pMgmt->latch);
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    if (ppVnode == NULL || *ppVnode == NULL) continue;
+
+    SVnodeObj *pVnode = *ppVnode;
+    vnodeStart(pVnode->pImpl);
+    pIter = taosHashIterate(pMgmt->hash, pIter);
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+  return 0;
+}
+
+static void vmStop(SMgmtWrapper *pWrapper) {
+#if 0
+  dDebug("vnode-mgmt start to stop");
+  SVnodesMgmt *pMgmt = pWrapper->pMgmt;
+  taosRLockLatch(&pMgmt->latch);
+
+  void *pIter = taosHashIterate(pMgmt->hash, NULL);
+  while (pIter) {
+    SVnodeObj **ppVnode = pIter;
+    if (ppVnode == NULL || *ppVnode == NULL) continue;
+
+    SVnodeObj *pVnode = *ppVnode;
+    vnodeStop(pVnode->pImpl);
+    pIter = taosHashIterate(pMgmt->hash, pIter);
+  }
+
+  taosRUnLockLatch(&pMgmt->latch);
+#endif
+}
+
 void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = vmInit;
   mgmtFp.closeFp = vmCleanup;
+  mgmtFp.startFp = vmStart;
+  mgmtFp.stopFp = vmStop;
   mgmtFp.requiredFp = vmRequire;
 
   vmInitMsgHandle(pWrapper);

source/dnode/mnode/impl/inc/mndDef.h

@@ -126,6 +126,8 @@ typedef enum {
   DND_REASON_OTHERS
 } EDndReason;
 
+typedef void (*TransCbFp)(SMnode* pMnode, void* param);
+
 typedef struct {
   int32_t    id;
   ETrnStage  stage;
@@ -148,6 +150,8 @@ typedef struct {
   int64_t    dbUid;
   char       dbname[TSDB_DB_FNAME_LEN];
   char       lastError[TSDB_TRANS_ERROR_LEN];
+  TransCbFp  transCbFp;
+  void*      transCbParam;
 } STrans;
 
 typedef struct {
@@ -350,7 +354,6 @@ typedef struct {
   int32_t  version;
   int32_t  nextColId;
   float    xFilesFactor;
-  int32_t  aggregationMethod;
   int32_t  delay;
   int32_t  ttl;
   int32_t  numOfColumns;

source/dnode/mnode/impl/inc/mndTrans.h

@@ -44,6 +44,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
 int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
 void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
+void    mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param);
 void    mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -30,7 +30,6 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
 int32_t  mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
 SEpSet   mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
 int32_t  mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
-int32_t  mndGetGlobalVgroupVersion(int32_t *vgId);
 
 void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndDb.c

@@ -234,6 +234,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
   return 0;
 }
 
+static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); }
+
 SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db);
@@ -618,7 +620,7 @@ static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOl
   SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
   if (pRedoRaw == NULL) return -1;
   if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
-  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1;
+  if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
 
   return 0;
 }
@@ -1191,8 +1193,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
   char *p = strchr(usedbReq.db, '.');
   if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
     memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
-    //mndGetGlobalVgroupVersion(); TODO
-    static int32_t vgVersion = 1;
+    int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode);
     if (usedbReq.vgVersion < vgVersion) {
       usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
       if (usedbRsp.pVgroupInfos == NULL) {

source/dnode/mnode/impl/src/mndScheduler.c

@@ -476,7 +476,10 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
   SSdb*       pSdb = pMnode->pSdb;
   SVgObj*     pVgroup = NULL;
-  SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan);
+  SQueryPlan* pPlan = NULL;
+  SSubplan*   plan = NULL;
+  if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
+    pPlan = qStringToQueryPlan(pTopic->physicalPlan);
     if (pPlan == NULL) {
       terrno = TSDB_CODE_QRY_INVALID_INPUT;
       return -1;
@@ -501,7 +504,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
       terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
       return -1;
     }
-  SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
+    plan = nodesListGetNode(inner->pNodeList, 0);
+  }
 
   int64_t             unexistKey = -1;
   SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
@@ -519,38 +523,35 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     }
 
     pSub->vgNum++;
-    plan->execNode.nodeId = pVgroup->vgId;
-    plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
 
     SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
-    pVgEp->epSet = plan->execNode.epSet;
-    pVgEp->vgId = plan->execNode.nodeId;
-
-#if 0
-    SMqConsumerEp consumerEp = {0};
-    consumerEp.status = 0;
-    consumerEp.consumerId = -1;
-    consumerEp.epSet = plan->execNode.epSet;
-    consumerEp.vgId = plan->execNode.nodeId;
-#endif
+    pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
+    pVgEp->vgId = pVgroup->vgId;
+    taosArrayPush(pEpInSub->vgs, &pVgEp);
 
     mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
+    if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
       int32_t msgLen;
+
+      plan->execNode.epSet = pVgEp->epSet;
+      plan->execNode.nodeId = pVgEp->vgId;
+
       if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
         sdbRelease(pSdb, pVgroup);
         qDestroyQueryPlan(pPlan);
         terrno = TSDB_CODE_QRY_INVALID_INPUT;
         return -1;
       }
-    taosArrayPush(pEpInSub->vgs, &pVgEp);
+    } else {
+      pVgEp->qmsg = strdup("");
+    }
 
     ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
 
     /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
-  ASSERT(pEpInSub->vgs->size > 0);
   pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
 
   ASSERT(pEpInSub->vgs->size > 0);

source/dnode/mnode/impl/src/mndStb.c

@@ -87,7 +87,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER)
   SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
@@ -175,7 +174,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
   int32_t xFilesFactor = 0;
   SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER)
   pStb->xFilesFactor = xFilesFactor / 10000.0f;
-  SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
@@ -404,7 +402,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.name = (char *)tNameGetTableName(&name);
   req.ttl = 0;
   req.keep = 0;
-  req.rollup = pStb->aggregationMethod > -1 ? 1 : 0;
+  req.rollup = pStb->pAst1 > 0 ? 1 : 0;
   req.type = TD_SUPER_TABLE;
   req.stbCfg.suid = pStb->uid;
   req.stbCfg.nCols = pStb->numOfColumns;
@@ -433,29 +431,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
 
     pRSmaParam->xFilesFactor = pStb->xFilesFactor;
     pRSmaParam->delay = pStb->delay;
-    pRSmaParam->nFuncIds = 1;  // only 1 aggregation method supported currently
-    pRSmaParam->pFuncIds = (func_id_t *)taosMemoryCalloc(pRSmaParam->nFuncIds, sizeof(func_id_t));
-    if (pRSmaParam->pFuncIds == NULL) {
-      taosMemoryFreeClear(req.stbCfg.pRSmaParam);
-      taosMemoryFreeClear(req.stbCfg.pSchema);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return NULL;
-    }
-    for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) {
-      *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod;
-    }
     if (pStb->ast1Len > 0) {
       if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) {
-        taosMemoryFreeClear(pRSmaParam->pFuncIds);
         taosMemoryFreeClear(req.stbCfg.pRSmaParam);
         taosMemoryFreeClear(req.stbCfg.pSchema);
         return NULL;
       }
     }
     if (pStb->ast2Len > 0) {
-      int32_t qmsgLen2 = 0;
       if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) {
-        taosMemoryFreeClear(pRSmaParam->pFuncIds);
         taosMemoryFreeClear(pRSmaParam->qmsg1);
         taosMemoryFreeClear(req.stbCfg.pRSmaParam);
         taosMemoryFreeClear(req.stbCfg.pSchema);
@@ -470,7 +454,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   SMsgHead *pHead = taosMemoryMalloc(contLen);
   if (pHead == NULL) {
     if (pRSmaParam) {
-      taosMemoryFreeClear(pRSmaParam->pFuncIds);
       taosMemoryFreeClear(pRSmaParam->qmsg1);
       taosMemoryFreeClear(pRSmaParam->qmsg2);
       taosMemoryFreeClear(pRSmaParam);
@@ -488,7 +471,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
 
   *pContLen = contLen;
   if (pRSmaParam) {
-    taosMemoryFreeClear(pRSmaParam->pFuncIds);
     taosMemoryFreeClear(pRSmaParam->qmsg1);
     taosMemoryFreeClear(pRSmaParam->qmsg2);
     taosMemoryFreeClear(pRSmaParam);
@@ -706,7 +688,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
   stbObj.version = 1;
   stbObj.nextColId = 1;
   stbObj.xFilesFactor = pCreate->xFilesFactor;
-  stbObj.aggregationMethod = pCreate->aggregationMethod;
   stbObj.delay = pCreate->delay;
   stbObj.ttl = pCreate->ttl;
   stbObj.numOfColumns = pCreate->numOfColumns;

source/dnode/mnode/impl/src/mndTopic.c

@@ -282,10 +282,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
   topicObj.version = 1;
   topicObj.sql = strdup(pCreate->sql);
   topicObj.sqlLen = strlen(pCreate->sql) + 1;
-  topicObj.ast = strdup(pCreate->ast);
-  topicObj.astLen = strlen(pCreate->ast) + 1;
 
   if (pCreate->ast && pCreate->ast[0]) {
+    topicObj.ast = strdup(pCreate->ast);
+    topicObj.astLen = strlen(pCreate->ast) + 1;
     topicObj.subType = TOPIC_SUB_TYPE__TABLE;
     topicObj.withTbName = 0;
     topicObj.withSchema = 0;
@@ -314,6 +314,9 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
       return -1;
     }
   } else {
+    topicObj.ast = strdup("");
+    topicObj.astLen = 1;
+    topicObj.physicalPlan = strdup("");
     topicObj.subType = TOPIC_SUB_TYPE__DB;
     topicObj.withTbName = 1;
     topicObj.withSchema = 1;

source/dnode/mnode/impl/src/mndTrans.c

@@ -193,9 +193,9 @@ TRANS_ENCODE_OVER:
 static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  SSdbRow *    pRow = NULL;
-  STrans *     pTrans = NULL;
-  char *       pData = NULL;
+  SSdbRow     *pRow = NULL;
+  STrans      *pTrans = NULL;
+  char        *pData = NULL;
   int32_t      dataLen = 0;
   int8_t       sver = 0;
   int32_t      redoLogNum = 0;
@@ -456,7 +456,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
 }
 
 static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
-  SSdb *  pSdb = pMnode->pSdb;
+  SSdb   *pSdb = pMnode->pSdb;
   STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId);
   if (pTrans == NULL) {
     terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
@@ -574,6 +574,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) {
   pTrans->rpcRspLen = contLen;
 }
 
+void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) {
+  pTrans->transCbFp = fp;
+  pTrans->transCbParam = param;
+}
+
 void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
   pTrans->dbUid = pDb->uid;
   memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN);
@@ -626,7 +631,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT
   if (mndIsBasicTrans(pNewTrans)) return 0;
 
   STrans *pTrans = NULL;
-  void *  pIter = NULL;
+  void   *pIter = NULL;
   int32_t code = 0;
 
   while (1) {
@@ -707,6 +712,8 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
   pNew->rpcRefId = pTrans->rpcRefId;
   pNew->rpcRsp = pTrans->rpcRsp;
   pNew->rpcRspLen = pTrans->rpcRspLen;
+  pNew->transCbFp = pTrans->transCbFp;
+  pNew->transCbParam = pTrans->transCbParam;
   pTrans->rpcRsp = NULL;
   pTrans->rpcRspLen = 0;
 
@@ -830,7 +837,7 @@ HANDLE_ACTION_RSP_OVER:
 }
 
 static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
-  SSdb *  pSdb = pMnode->pSdb;
+  SSdb   *pSdb = pMnode->pSdb;
   int32_t arraySize = taosArrayGetSize(pArray);
 
   if (arraySize == 0) return 0;
@@ -1117,6 +1124,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
   }
 
   mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
+
+  if (pTrans->transCbFp != NULL) {
+    (*pTrans->transCbFp)(pMnode, pTrans->transCbParam);
+  }
+
   return continueExec;
 }
 
@@ -1205,11 +1217,11 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
 }
 
 static int32_t mndProcessKillTransReq(SNodeMsg *pReq) {
-  SMnode *      pMnode = pReq->pNode;
+  SMnode       *pMnode = pReq->pNode;
   SKillTransReq killReq = {0};
   int32_t       code = -1;
-  SUserObj *    pUser = NULL;
-  STrans *      pTrans = NULL;
+  SUserObj     *pUser = NULL;
+  STrans       *pTrans = NULL;
 
   if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -1249,7 +1261,7 @@ KILL_OVER:
 
 void mndTransPullup(SMnode *pMnode) {
   STrans *pTrans = NULL;
-  void *  pIter = NULL;
+  void   *pIter = NULL;
 
   while (1) {
     pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@@ -1264,11 +1276,11 @@ void mndTransPullup(SMnode *pMnode) {
 
 static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode *pMnode = pReq->pNode;
-  SSdb *  pSdb = pMnode->pSdb;
+  SSdb   *pSdb = pMnode->pSdb;
   int32_t numOfRows = 0;
   STrans *pTrans = NULL;
   int32_t cols = 0;
-  char *  pWrite;
+  char   *pWrite;
 
   while (numOfRows < rows) {
     pShow->pIter = sdbFetch(pSdb, SDB_TRANS, pShow->pIter, (void **)&pTrans);

source/dnode/mnode/impl/test/db/db.cpp

@@ -89,7 +89,6 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
     void*   pReq = rpcMallocCont(contLen);
     tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
 
-    taosMsleep(1000); // Wait for the vnode to become the leader
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -258,7 +258,6 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateDbReq(dbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(1000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -304,7 +304,6 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -439,7 +438,6 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -499,7 +497,6 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -541,7 +538,6 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -606,7 +602,6 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -660,7 +655,6 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -721,7 +715,6 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -782,7 +775,6 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/vnode/inc/vnode.h

@@ -61,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int     vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
 
+int32_t vnodeStart(SVnode *pVnode);
+void    vnodeStop(SVnode *pVnode);
+
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 
@@ -77,16 +80,15 @@ char       *metaTbCursorNext(SMTbCursor *pTbCur);
 
 // tsdb
 typedef struct STsdb          STsdb;
-typedef struct STsdbQueryCond STsdbQueryCond;
 typedef void                 *tsdbReaderT;
 
 #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
 #define BLOCK_LOAD_TABLE_SEQ_ORDER  2
 #define BLOCK_LOAD_TABLE_RR_ORDER   3
 
-tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
+tsdbReaderT *tsdbQueryTables(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
                              uint64_t taskId);
-tsdbReaderT  tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId,
+tsdbReaderT  tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId,
                                 void *pMemRef);
 int32_t      tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
 bool         isTsdbCacheLastRow(tsdbReaderT *pReader);
@@ -98,6 +100,7 @@ bool         tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
 void         tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
 int32_t      tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
 SArray      *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
+void         tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond);
 void         tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
 int32_t      tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 int32_t      tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
@@ -157,25 +160,11 @@ struct SVnodeCfg {
   int8_t   hashMethod;
 };
 
-struct STsdbQueryCond {
-  STimeWindow  twindow;
-  int32_t      order;  // desc|asc order to iterate the data block
-  int32_t      numOfCols;
-  SColumnInfo *colList;
-  bool         loadExternalRows;  // load external rows or not
-  int32_t      type;              // data block load type:
-};
-
 typedef struct {
   TSKEY    lastKey;
   uint64_t uid;
 } STableKeyInfo;
 
-// sync integration
-void    vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
-void    vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
-int32_t vnodeSyncStart(SVnode *pVnode);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/meta.h

@@ -51,7 +51,7 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64()
 #define META_CHILD_TABLE  TD_CHILD_TABLE
 #define META_NORMAL_TABLE TD_NORMAL_TABLE
 
-int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg);
+int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int             metaDropTable(SMeta* pMeta, tb_uid_t uid);
 int             metaCommit(SMeta* pMeta);
 int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
@@ -74,7 +74,7 @@ tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
-int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
+int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
 int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
 int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);

source/dnode/vnode/src/inc/tsdb.h

@@ -56,8 +56,6 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
 int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
 int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
 void    tsdbCleanupReadHandle(tsdbReaderT queryHandle);
-int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
-
 typedef enum {
   TSDB_FILE_HEAD = 0,  // .head
   TSDB_FILE_DATA,      // .data

source/dnode/vnode/src/inc/tsdbSma.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_TSDB_SMA_H_
+#define _TD_VNODE_TSDB_SMA_H_
+
+#include "os.h"
+#include "thash.h"
+#include "tmsg.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
+
+struct STbDdlH {
+  void         *ahandle;
+  void         *result;
+  __tb_ddl_fn_t fp;
+};
+
+typedef struct {
+  tb_uid_t  suid;
+  SArray   *tbUids;
+  SHashObj *uidHash;
+} STbUidStore;
+
+static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
+  ASSERT(*pStore == NULL);
+  *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
+  if (*pStore == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+void    tsdbUidStoreDestory(STbUidStore *pStore);
+void   *tsdbUidStoreFree(STbUidStore *pStore);
+
+int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq);
+int32_t tsdbFetchTbUidList(void *pTsdb, void **result, void *suid, void *uid);
+int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pUidStore);
+int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_TSDB_SMA_H_*/
\ No newline at end of file

source/dnode/vnode/src/inc/vnodeInt.h

@@ -103,6 +103,8 @@ struct SVnode {
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
+typedef struct STbDdlH STbDdlH;
+
 // sma
 void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
 
@@ -116,6 +118,8 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
 
 #include "vnodeSync.h"
 
+#include "tsdbSma.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -250,7 +250,7 @@ void metaCloseDB(SMeta *pMeta) {
   }
 }
 
-int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
+int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   tb_uid_t       uid;
   SMetaDB       *pMetaDb;
   void          *pKey;
@@ -349,6 +349,12 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     if (ret < 0) {
       return -1;
     }
+    // child table handle for rsma
+    if (pHandle && pHandle->fp) {
+      if (((*pHandle->fp)(pHandle->ahandle, &pHandle->result, &ctbIdxKey.suid, &uid)) < 0) {
+        return -1;
+      };
+    }
   } else if (pTbCfg->type == META_NORMAL_TABLE) {
     pKey = &uid;
     kLen = sizeof(uid);

source/dnode/vnode/src/meta/metaTable.c

@@ -15,7 +15,7 @@
 
 #include "vnodeInt.h"
 
-int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) {
+int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   // Validate the tbOptions
   // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
   //   // TODO: handle error
@@ -24,7 +24,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) {
 
   // TODO: add atomicity
 
-  if (metaSaveTableToDB(pMeta, pTbCfg) < 0) {
+  if (metaSaveTableToDB(pMeta, pTbCfg, pHandle) < 0) {
     // TODO: handle error
     return -1;
   }

source/dnode/vnode/src/tq/tq.c

@@ -197,11 +197,10 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   }
   memcpy(data, msg, msgLen);
 
-  if (msgType == TDMT_VND_SUBMIT) {
+  // make sure msgType == TDMT_VND_SUBMIT
   if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
     return -1;
   }
-  }
 
   SRpcMsg req = {
       .msgType = TDMT_VND_STREAM_TRIGGER,
@@ -411,8 +410,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
+  rsp.withSchema = pExec->withSchema;
   rsp.blockData = taosArrayInit(0, sizeof(void*));
   rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+  rsp.blockSchema = taosArrayInit(0, sizeof(void*));
 
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
@@ -512,6 +513,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           ASSERT(actualLen <= dataStrLen);
           taosArrayPush(rsp.blockDataLen, &actualLen);
           taosArrayPush(rsp.blockData, &buf);
+
+          if (pExec->withSchema) {
+            SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
+            taosArrayPush(rsp.blockSchema, &pSW);
+          }
+
           rsp.blockNum++;
         }
         // db subscribe
@@ -540,6 +547,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           ASSERT(actualLen <= dataStrLen);
           taosArrayPush(rsp.blockDataLen, &actualLen);
           taosArrayPush(rsp.blockData, &buf);
+
+          SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
+          taosArrayPush(rsp.blockSchema, &pSW);
+
           rsp.blockNum++;
         }
       } else {
@@ -586,6 +597,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   // TODO destroy
   taosArrayDestroy(rsp.blockData);
   taosArrayDestroy(rsp.blockDataLen);
+  taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+
   return 0;
 }
 
@@ -827,12 +840,16 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
     for (int32_t i = 0; i < 5; i++) {
       pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+      if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
         SReadHandle handle = {
             .reader = pExec->pExecReader[i],
             .meta = pTq->pVnode->pMeta,
         };
         pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
         ASSERT(pExec->task[i]);
+      } else {
+        pExec->task[i] = NULL;
+      }
     }
     taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
     return 0;

source/dnode/vnode/src/tq/tqRead.c

@@ -33,24 +33,24 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
 
 int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
   pReadHandle->pMsg = pMsg;
-  pMsg->length = htonl(pMsg->length);
-  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
+  // pMsg->length = htonl(pMsg->length);
+  // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
 
   // iterate and convert
-  if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
+  if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1;
   while (true) {
-    if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
+    if (tGetSubmitMsgNextEx(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
     if (pReadHandle->pBlock == NULL) break;
 
-    pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid);
-    pReadHandle->pBlock->suid = htobe64(pReadHandle->pBlock->suid);
-    pReadHandle->pBlock->sversion = htonl(pReadHandle->pBlock->sversion);
-    pReadHandle->pBlock->dataLen = htonl(pReadHandle->pBlock->dataLen);
-    pReadHandle->pBlock->schemaLen = htonl(pReadHandle->pBlock->schemaLen);
-    pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows);
+    // pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid);
+    // pReadHandle->pBlock->suid = htobe64(pReadHandle->pBlock->suid);
+    // pReadHandle->pBlock->sversion = htonl(pReadHandle->pBlock->sversion);
+    // pReadHandle->pBlock->dataLen = htonl(pReadHandle->pBlock->dataLen);
+    // pReadHandle->pBlock->schemaLen = htonl(pReadHandle->pBlock->schemaLen);
+    // pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows);
   }
 
-  if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
+  if (tInitSubmitMsgIterEx(pMsg, &pReadHandle->msgIter) < 0) return -1;
   pReadHandle->ver = ver;
   memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter));
   return 0;
@@ -58,15 +58,17 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t
 
 bool tqNextDataBlock(STqReadHandle* pHandle) {
   while (1) {
-    if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) {
+    if (tGetSubmitMsgNextEx(&pHandle->msgIter, &pHandle->pBlock) < 0) {
       return false;
     }
     if (pHandle->pBlock == NULL) return false;
 
     /*pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid);*/
     /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/
-    ASSERT(pHandle->tbIdHash);
-    void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t));
+    if (pHandle->tbIdHash == NULL) {
+      return true;
+    }
+    void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
     if (ret != NULL) {
       /*printf("retrieve one tb %ld\n", pHandle->pBlock->uid);*/
       /*pHandle->pBlock->tid = htonl(pHandle->pBlock->tid);*/
@@ -88,25 +90,48 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   // TODO set to real sversion
   int32_t sversion = 0;
   if (pHandle->sver != sversion) {
-    pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion);
-
+    pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->msgIter.uid, sversion);
+#if 0
     tb_uid_t quid;
-    STbCfg*  pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid);
+    STbCfg*  pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->msgIter.uid);
     if (pTbCfg->type == META_CHILD_TABLE) {
       quid = pTbCfg->ctbCfg.suid;
     } else {
-      quid = pHandle->pBlock->uid;
+      quid = pHandle->msgIter.uid;
     }
     pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true);
+#endif
+    pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, pHandle->msgIter.suid, sversion, true);
     pHandle->sver = sversion;
   }
 
   STSchema*       pTschema = pHandle->pSchema;
   SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
 
-  *pNumOfRows = pHandle->pBlock->numOfRows;
+  *pNumOfRows = pHandle->msgIter.numOfRows;
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
+  if (colNumNeed == 0) {
+    *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData));
+    if (*ppCols == NULL) {
+      return -1;
+    }
+
+    int32_t colMeta = 0;
+    while (colMeta < pSchemaWrapper->nCols) {
+      SSchema*        pColSchema = &pSchemaWrapper->pSchema[colMeta];
+      SColumnInfoData colInfo = {0};
+      colInfo.info.bytes = pColSchema->bytes;
+      colInfo.info.colId = pColSchema->colId;
+      colInfo.info.type = pColSchema->type;
+
+      if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) {
+        goto FAIL;
+      }
+      taosArrayPush(*ppCols, &colInfo);
+      colMeta++;
+    }
+  } else {
     if (colNumNeed > pSchemaWrapper->nCols) {
       colNumNeed = pSchemaWrapper->nCols;
     }
@@ -140,6 +165,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
         colNeed++;
       }
     }
+  }
 
   int32_t colActual = taosArrayGetSize(*ppCols);
   *pNumOfCols = colActual;
@@ -151,8 +177,8 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
   int32_t curRow = 0;
-  tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
-  while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
+  tInitSubmitBlkIterEx(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter);
+  while ((row = tGetSubmitBlkNextEx(&pHandle->blkIter)) != NULL) {
     tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block
     for (int32_t i = 0; i < colActual; i++) {

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -81,8 +81,8 @@ static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg
 
 static void tsdbFree(STsdb *pTsdb) {
   if (pTsdb) {
-    // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
-    // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
+    tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb));
+    tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb));
     tsdbFreeFS(pTsdb->fs);
     taosMemoryFreeClear(pTsdb->path);
     taosMemoryFree(pTsdb);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -254,7 +254,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S
 
         assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey);
       } else {
-        assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey);
+        info.lastKey = pTsdbReadHandle->window.skey;
       }
 
       taosArrayPush(pTableCheckInfo, &info);
@@ -317,7 +317,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
   return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
 }
 
-static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) {
+static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) {
   pTsdbReadHandle->window = pCond->twindow;
 
   bool    updateTs = false;
@@ -343,7 +343,7 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond*
   }
 }
 
-static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) {
+static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, SQueryTableDataCond* pCond, uint64_t qId, uint64_t taskId) {
   STsdbReadHandle* pReadHandle = taosMemoryCalloc(1, sizeof(STsdbReadHandle));
   if (pReadHandle == NULL) {
     goto _end;
@@ -422,7 +422,7 @@ _end:
   return NULL;
 }
 
-tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT* tsdbQueryTables(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                              uint64_t taskId) {
   STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId);
   if (pTsdbReadHandle == NULL) {
@@ -448,7 +448,7 @@ tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo
   return (tsdbReaderT)pTsdbReadHandle;
 }
 
-void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) {
+void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) {
   STsdbReadHandle* pTsdbReadHandle = queryHandle;
 
   if (emptyQueryTimewindow(pTsdbReadHandle)) {
@@ -485,7 +485,7 @@ void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) {
   resetCheckInfo(pTsdbReadHandle);
 }
 
-void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pCond, STableGroupInfo* groupList) {
+void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, STableGroupInfo* groupList) {
   STsdbReadHandle* pTsdbReadHandle = queryHandle;
 
   pTsdbReadHandle->order = pCond->order;
@@ -526,7 +526,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pC
   //  pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next);
 }
 
-tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                              uint64_t taskId) {
   pCond->twindow = updateLastrowForEachGroup(groupList);
 
@@ -555,7 +555,7 @@ tsdbReaderT tsdbQueryLastRow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo
 }
 
 #if 0
-tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
+tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) {
   STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef);
   if (pTsdbReadHandle == NULL) {
     return NULL;
@@ -618,7 +618,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr
   return pNew;
 }
 
-tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId,
+tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                                           uint64_t taskId) {
   STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList);
 
@@ -1185,10 +1185,11 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     tsdbDebug("%p no data in mem, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
   }
 
-  if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
-      (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-    if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
-        (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
+  bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+
+  if ((ascScan && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
+
+    if ((ascScan && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || (!ascScan && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
 
@@ -1225,8 +1226,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
     assert(pTsdbReadHandle->outputCapacity >= binfo.rows);
     int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo);
 
-    if ((cur->pos == 0 && endPos == binfo.rows - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
-        (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) {
+    if ((cur->pos == 0 && endPos == binfo.rows - 1 && ascScan) || (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ascScan))) {
       pTsdbReadHandle->realNumOfRows = binfo.rows;
 
       cur->rows = binfo.rows;
@@ -1234,7 +1234,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
       cur->mixBlock = false;
       cur->blockCompleted = true;
 
-      if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
+      if (ascScan) {
         cur->lastKey = binfo.window.ekey + 1;
         cur->pos = binfo.rows;
       } else {
@@ -1382,8 +1382,6 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 
 static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
                                        int32_t start, int32_t end) {
-  int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1;
-
   SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0];
   TSKEY*     tsArray = pCols->cols[0].pData;
 
@@ -1394,6 +1392,11 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
     return numOfRows;
   }
 
+  bool    ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order);
+  int32_t trueStart = ascScan ? start : end;
+  int32_t trueEnd   = ascScan ? end : start;
+  int32_t step = ascScan ? 1 : -1;
+
   int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns);
 
   // data in buffer has greater timestamp, copy data in file block
@@ -1411,7 +1414,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
       if (!IS_VAR_DATA_TYPE(pColInfo->info.type)) {  // todo opt performance
         //        memmove(pData, (char*)src->pData + bytes * start, bytes * num);
         int32_t rowIndex = numOfRows;
-        for (int32_t k = start; k <= end; ++k, ++rowIndex) {
+        for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) {
           SCellVal sVal = {0};
           if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) {
             TASSERT(0);
@@ -1427,7 +1430,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
         int32_t rowIndex = numOfRows;
 
         // todo refactor, only copy one-by-one
-        for (int32_t k = start; k < num + start; ++k, ++rowIndex) {
+        for (int32_t k = trueStart; ((ascScan && k <= trueEnd) || (!ascScan && k >= trueEnd)); k += step, ++rowIndex) {
           SCellVal sVal = {0};
           if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) {
             TASSERT(0);
@@ -1444,26 +1447,19 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
       j++;
       i++;
     } else {                                           // pColInfo->info.colId < src->colId, it is a NULL data
-      int32_t rowIndex = numOfRows;
-      for (int32_t k = start; k < num + start; ++k, ++rowIndex) {  // TODO opt performance
-        colDataAppend(pColInfo, rowIndex, NULL, true);
-      }
+      colDataAppendNNULL(pColInfo, numOfRows, num);
       i++;
     }
   }
 
   while (i < requiredNumOfCols) {  // the remain columns are all null data
     SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i);
-    int32_t rowIndex = numOfRows;
-
-    for (int32_t k = start; k < num + start; ++k, ++rowIndex) {
-      colDataAppend(pColInfo, rowIndex, NULL, true);  // TODO add a fast version to set a number of consecutive NULL value.
-    }
+    colDataAppendNNULL(pColInfo, numOfRows, num);
     i++;
   }
 
-  pTsdbReadHandle->cur.win.ekey = tsArray[end];
-  pTsdbReadHandle->cur.lastKey = tsArray[end] + step;
+  pTsdbReadHandle->cur.win.ekey = tsArray[trueEnd];
+  pTsdbReadHandle->cur.lastKey = tsArray[trueEnd] + step;
 
   return numOfRows + num;
 }
@@ -2966,7 +2962,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) {
 //  }
 //
 //  // load the previous row
-//  STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
+//  SQueryTableDataCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER};
 //  if (type == TSDB_PREV_ROW) {
 //    cond.order = TSDB_ORDER_DESC;
 //    cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN};
@@ -3330,21 +3326,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) {
           return NULL;
         }
 
-        // todo refactor
         int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-
-        // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-        if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
-          int32_t emptySize = pHandle->outputCapacity - numOfRows;
-          int32_t reqNumOfCols = (int32_t)taosArrayGetSize(pHandle->pColumns);
-
-          for (int32_t i = 0; i < reqNumOfCols; ++i) {
-            SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
-            memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes,
-                    numOfRows * pColInfo->info.bytes);
-          }
-        }
-
         return pHandle->pColumns;
       }
     }
@@ -3636,6 +3618,8 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch
     tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
     terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
     goto _error;
+  } else {
+    tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId);
   }
 
   if (pTbCfg->type != META_SUPER_TABLE) {

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -14,6 +14,7 @@
  */
 
 #include "vnodeInt.h"
+#include "vnodeSync.h"
 
 int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
   SVnodeInfo info = {0};
@@ -171,6 +172,16 @@ void vnodeClose(SVnode *pVnode) {
   }
 }
 
+// start the sync timer after the queue is ready
+int32_t vnodeStart(SVnode *pVnode) {
+  vnodeSyncSetQ(pVnode, NULL);
+  vnodeSyncSetRpc(pVnode, NULL);
+  vnodeSyncStart(pVnode);
+  return 0;
+}
+
+void vnodeStop(SVnode *pVnode) {}
+
 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
 
 void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -81,6 +81,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
     case TDMT_VND_DROP_TABLE:
       break;
     case TDMT_VND_SUBMIT:
+      tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK);
       pRsp->msgType = TDMT_VND_SUBMIT_RSP;
       vnodeProcessSubmitReq(pVnode, ptr, pRsp);
       break;
@@ -101,7 +102,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
       }
     } break;
     case TDMT_VND_CREATE_SMA: {  // timeRangeSMA
-
       if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
         // TODO
       }
@@ -193,6 +193,9 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
 
 // sync integration
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
+
+  if (syncEnvIsStart()) {
+
     SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
     assert(pSyncNode != NULL);
 
@@ -270,31 +273,25 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
     }
 
     syncNodeRelease(pSyncNode);
-
+  } else {
+    vError("==vnodeProcessSyncReq== error syncEnv stop");
+  }
   return 0;
 }
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) {
   SVCreateTbReq vCreateTbReq = {0};
   tDeserializeSVCreateTbReq(pReq, &vCreateTbReq);
-  if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) {
+  if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq), NULL) < 0) {
     // TODO
     return -1;
   }
 
-  // TODO: remove the debug log
-  SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam;
-  if (param) {
-    printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0,
-           param->qmsg1 ? param->qmsg1 : "");
-    printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0,
-           param->qmsg2 ? param->qmsg2 : "");
-  }
+  tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq);
 
   taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
   taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
   if (vCreateTbReq.stbCfg.pRSmaParam) {
-    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds);
     taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1);
     taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2);
     taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
@@ -309,6 +306,13 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
   SVCreateTbBatchRsp vCreateTbBatchRsp = {0};
   tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq);
   int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray);
+
+  STbDdlH ddlHandle = {
+      .ahandle = pVnode->pTsdb,
+      .result = NULL,
+      .fp = tsdbFetchTbUidList,
+  };
+
   for (int i = 0; i < reqNum; i++) {
     SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i);
 
@@ -324,7 +328,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
       taosArrayPush(vCreateTbBatchRsp.rspList, &rsp);
     }
 
-    if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
+    if (metaCreateTable(pVnode->pMeta, pCreateTbReq, &ddlHandle) < 0) {
       // TODO: handle error
       vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name);
     }
@@ -334,7 +338,6 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
       taosMemoryFree(pCreateTbReq->stbCfg.pSchema);
       taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema);
       if (pCreateTbReq->stbCfg.pRSmaParam) {
-        taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam->pFuncIds);
         taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam);
       }
     } else if (pCreateTbReq->type == TD_CHILD_TABLE) {
@@ -342,12 +345,13 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
     } else {
       taosMemoryFree(pCreateTbReq->ntbCfg.pSchema);
       if (pCreateTbReq->ntbCfg.pRSmaParam) {
-        taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam->pFuncIds);
         taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam);
       }
     }
   }
 
+  tsdbUpdateTbUidList(pVnode->pTsdb, ddlHandle.result);
+
   vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray));
   taosArrayDestroy(vCreateTbBatchReq.pArray);
   if (vCreateTbBatchRsp.rspList) {
@@ -371,7 +375,6 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
   taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
   taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema);
   if (vAlterTbReq.stbCfg.pRSmaParam) {
-    taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam->pFuncIds);
     taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam);
   }
   taosMemoryFree(vAlterTbReq.name);

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -407,7 +407,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
     }
   }
 
-  EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS);
+  // EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS);
 
   EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, pMsg, 0), 0);
 

source/libs/executor/inc/executorimpl.h

@@ -319,16 +319,22 @@ typedef struct SColMatchInfo {
   bool    output;
 } SColMatchInfo;
 
+typedef struct SScanInfo {
+  int32_t numOfAsc;
+  int32_t numOfDesc;
+} SScanInfo;
+
 typedef struct STableScanInfo {
   void*           dataReader;
+
   int32_t         numOfBlocks;  // extract basic running information.
   int32_t         numOfSkipped;
   int32_t         numOfBlockStatis;
   int64_t         numOfRows;
-  int32_t         order;  // scan order
-  int32_t         times;  // repeat counts
+  int64_t         elapsedTime;
+  int32_t         prevGroupId;  // previous table group id
+  SScanInfo       scanInfo;
   int32_t         current;
-  int32_t         reverseTimes;  // 0 by default
   SNode*          pFilterNode;  // filter operator info
   SqlFunctionCtx* pCtx;         // next operator query context
   SResultRowInfo* pResultRowInfo;
@@ -337,9 +343,8 @@ typedef struct STableScanInfo {
   SSDataBlock*    pResBlock;
   SArray*         pColMatchInfo;
   int32_t         numOfOutput;
-  int64_t         elapsedTime;
-  int32_t         prevGroupId;  // previous table group id
 
+  SQueryTableDataCond cond;
   int32_t         scanFlag;     // table scan flag to denote if it is a repeat/reverse/main scan
   int32_t         dataBlockLoadFlag;
   double          sampleRatio;  // data block sample ratio, 1 by default
@@ -399,8 +404,8 @@ typedef struct SOptrBasicInfo {
 // TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset
 typedef struct SAggSupporter {
   SHashObj*      pResultRowHashTable;  // quick locate the window object for each result
-  SHashObj*      pResultRowListSet;    // used to check if current ResultRowInfo has ResultRow object or not
-  SArray*        pResultRowArrayList;  // The array list that contains the Result rows
+//  SHashObj*      pResultRowListSet;    // used to check if current ResultRowInfo has ResultRow object or not
+//  SArray*        pResultRowArrayList;  // The array list that contains the Result rows
   char*          keyBuf;               // window key buffer
   SDiskbasedBuf* pResultBuf;           // query result buffer based on blocked-wised disk file
   int32_t        resultRowSize;        // the result buffer size for each result row, with the meta data size for each row
@@ -620,6 +625,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
                                   int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
                                   SArray* pColList);
 void    getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win);
+int32_t getTableScanOrder(SOperatorInfo* pOperator);
 
 void    doSetOperatorCompleted(SOperatorInfo* pOperator);
 void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
@@ -627,9 +633,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
 
 SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime,
-                                           int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
-                                           SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo,
+                                           SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo);
+
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
                                            int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
 

source/libs/executor/src/executil.c

@@ -221,7 +221,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, boo
 
     p->groupId = *(uint64_t*) key;
     p->pos = *(SResultRowPosition*) pData;
-    memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
+    memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
 
     taosArrayPush(pGroupResInfo->pRows, &p);
   }

source/libs/executor/src/scanoperator.c

@@ -245,6 +245,9 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
 
   relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols);
 
+  // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator.
+  pBlock->info.blockId = 0;
+
   doFilter(pTableScanInfo->pFilterNode, pBlock);
   if (pBlock->info.rows == 0) {
     pCost->filterOutBlocks += 1;
@@ -255,17 +258,15 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
   return TSDB_CODE_SUCCESS;
 }
 
-static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
-  // reverse order time range
+static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   SET_REVERSE_SCAN_FLAG(pTableScanInfo);
 
   switchCtxOrder(pCtx, numOfOutput);
-  SWITCH_ORDER(pTableScanInfo->order);
-  setupQueryRangeForReverseScan(pTableScanInfo);
+//  setupQueryRangeForReverseScan(pTableScanInfo);
 
-  pTableScanInfo->times = 1;
-  pTableScanInfo->current = 0;
-  pTableScanInfo->reverseTimes = 0;
+  STimeWindow* pTWindow = &pTableScanInfo->cond.twindow;
+  TSWAP(pTWindow->skey, pTWindow->ekey, int64_t);
+  pTableScanInfo->cond.order = TSDB_ORDER_DESC;
 }
 
 static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
@@ -294,8 +295,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
       continue;
     }
 
-    // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator.
-    pBlock->info.blockId = 0;
     return pBlock;
   }
 
@@ -311,63 +310,67 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
     return NULL;
   }
 
-  SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo;
   *newgroup = false;
 
-  while (pTableScanInfo->current < pTableScanInfo->times) {
+  while (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) {
     SSDataBlock* p = doTableScanImpl(pOperator, newgroup);
     if (p != NULL) {
       return p;
     }
 
-    if (++pTableScanInfo->current >= pTableScanInfo->times) {
-      if (pTableScanInfo->reverseTimes <= 0 /* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) {
-        return NULL;
-      } else {
-        break;
-      }
-    }
-
-    // do prepare for the next round table scan operation
-    //    STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
-    //    tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
+    pTableScanInfo->current += 1;
 
+    if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) {
       setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
       pTableScanInfo->scanFlag = REPEAT_SCAN;
 
-    //    if (pResultRowInfo->size > 0) {
-    //      pResultRowInfo->curPos = 0;
-    //    }
+      STimeWindow* pWin = &pTableScanInfo->cond.twindow;
+      qDebug("%s start to repeat ascending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
+             GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
 
-    qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
-           GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
+      // do prepare for the next round table scan operation
+      tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond);
+    }
   }
 
-  SSDataBlock* p = NULL;
-  // todo refactor
-  if (pTableScanInfo->reverseTimes > 0) {
-    setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
-    //    STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window);
-    //    tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond);
+  int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
+  if (pTableScanInfo->current < total) {
+    if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
+      prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput);
+      tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond);
+    }
 
-    qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
-           GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
+    STimeWindow* pWin = &pTableScanInfo->cond.twindow;
+    qDebug("%s start to descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
+           GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey);
 
-    if (pResultRowInfo->size > 0) {
-      //      pResultRowInfo->curPos = pResultRowInfo->size - 1;
+    while (pTableScanInfo->current < total) {
+      SSDataBlock* p = doTableScanImpl(pOperator, newgroup);
+      if (p != NULL) {
+        return p;
       }
 
-    p = doTableScanImpl(pOperator, newgroup);
+      pTableScanInfo->current += 1;
+
+      if (pTableScanInfo->current < pTableScanInfo->scanInfo.numOfAsc) {
+        setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
+        pTableScanInfo->scanFlag = REPEAT_SCAN;
+
+        qDebug("%s start to repeat descending order scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
+               GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
+
+        // do prepare for the next round table scan operation
+        tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond);
+      }
+    }
   }
 
-  return p;
+  setTaskStatus(pTaskInfo, TASK_COMPLETED);
+  return NULL;
 }
 
-SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag,
-                                           int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
-                                           SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
-  assert(repeatTime > 0);
-
+SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCond* pCond, int32_t numOfOutput, int32_t dataLoadFlag, const uint8_t* scanInfo,
+    SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
   SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -378,18 +381,19 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int
     return NULL;
   }
 
+  pInfo->cond             = *pCond;
+  pInfo->scanInfo         = (SScanInfo) {.numOfAsc = scanInfo[0], .numOfDesc = scanInfo[1]};
+
   pInfo->interval         = *pInterval;
   pInfo->sampleRatio      = sampleRatio;
   pInfo->dataBlockLoadFlag= dataLoadFlag;
   pInfo->pResBlock        = pResBlock;
   pInfo->pFilterNode      = pCondition;
   pInfo->dataReader       = pDataReader;
-  pInfo->times            = repeatTime;
-  pInfo->reverseTimes     = reverseTime;
-  pInfo->order            = order;
   pInfo->current          = 0;
   pInfo->scanFlag         = MAIN_SCAN;
   pInfo->pColMatchInfo    = pColMatchInfo;
+
   pOperator->name         = "TableScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
   pOperator->blockingOptr = false;
@@ -411,8 +415,6 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) {
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
 
   pInfo->dataReader  = pTsdbReadHandle;
-  pInfo->times = 1;
-  pInfo->reverseTimes = 0;
   pInfo->current     = 0;
   pInfo->prevGroupId = -1;
 

source/libs/function/inc/functionMgtInt.h

@@ -41,7 +41,9 @@ extern "C" {
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
-#define FUNC_UDF_ID_START_OFFSET_VAL 5000
+#define FUNC_UDF_ID_START     5000
+#define FUNC_AGGREGATE_UDF_ID 5001
+#define FUNC_SCALAR_UDF_ID    5002
 
 extern const int funcMgtUdfNum;
 

source/libs/function/src/functionMgt.c

@@ -25,7 +25,6 @@
 
 typedef struct SFuncMgtService {
   SHashObj* pFuncNameHashTable;
-  SArray* pUdfTable; // SUdfInfo
 } SFuncMgtService;
 
 typedef struct SUdfInfo {
@@ -50,18 +49,12 @@ static void doInitFunctionTable() {
       return;
     }
   }
-
-  gFunMgtService.pUdfTable = NULL;
-}
-
-static int8_t getUdfType(int32_t funcId) {
-  SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1);
-  return pUdf->funcType;
 }
 
 static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
   if (fmIsUserDefinedFunc(funcId)) {
-    return getUdfType(funcId);
+    return FUNC_MGT_AGG_FUNC == classification ? FUNC_AGGREGATE_UDF_ID == funcId :
+        (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false);
   }
   if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return false;
@@ -69,33 +62,23 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
   return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
 }
 
-static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
+static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) {
   SFuncInfo* pInfo = NULL;
-  int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo);
-  if (TSDB_CODE_SUCCESS != code || NULL == pInfo) {
-    return -1;
-  }
-  if (NULL == gFunMgtService.pUdfTable) {
-    gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo));
-  }
-  SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType };
-  taosArrayPush(gFunMgtService.pUdfTable, &info);
+  int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo);
+  if (TSDB_CODE_SUCCESS != code) {
+    return code;
+  }
+  if (NULL == pInfo) {
+    snprintf(pParam->pErrBuf, pParam->errBufLen, "Invalid function name: %s", pFunc->functionName);
+    return TSDB_CODE_FUNC_INVALID_FUNTION;
+  }
+  pFunc->funcType = FUNCTION_TYPE_UDF;
+  pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == pInfo->funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID;
+  pFunc->node.resType.type = pInfo->outputType;
+  pFunc->node.resType.bytes = pInfo->outputLen;
+  pFunc->udfBufSize = pInfo->bufSize;
   tFreeSFuncInfo(pInfo);
   taosMemoryFree(pInfo);
-  return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL;
-}
-
-static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
-  void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName));
-  if (NULL == pVal) {
-    return getUdfId(pParam, pFuncName);
-  }
-  return *(int32_t*)pVal;
-}
-
-static int32_t getUdfResultType(SFunctionNode* pFunc) {
-  SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, pFunc->funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1);
-  pFunc->node.resType = pUdf->outputDt;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -104,28 +87,14 @@ int32_t fmFuncMgtInit() {
   return initFunctionCode;
 }
 
-int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
-  *pFuncId = getFuncId(pParam, pFuncName);
-  if (*pFuncId < 0) {
-    return TSDB_CODE_FAILED;
-  }
-  if (fmIsUserDefinedFunc(*pFuncId)) {
-    *pFuncType = FUNCTION_TYPE_UDF;
-  } else {
-    *pFuncType = funcMgtBuiltins[*pFuncId].type;
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  if (fmIsUserDefinedFunc(pFunc->funcId)) {
-    return getUdfResultType(pFunc);
-  }
-
-  if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) {
-    return TSDB_CODE_FAILED;
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) {
+  void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc->functionName, strlen(pFunc->functionName));
+  if (NULL != pVal) {
+    pFunc->funcId = *(int32_t*)pVal;
+    pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type;
+    return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pParam->pErrBuf, pParam->errBufLen);
   }
-  return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pErrBuf, len);
+  return getUdfInfo(pParam, pFunc);
 }
 
 EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
@@ -203,7 +172,7 @@ bool fmIsMultiResFunc(int32_t funcId) {
 }
 
 bool fmIsUserDefinedFunc(int32_t funcId) {
-  return funcId > FUNC_UDF_ID_START_OFFSET_VAL;
+  return funcId > FUNC_UDF_ID_START;
 }
 
 void fmFuncMgtDestroy() {

source/libs/function/src/udfd.c

@@ -551,7 +551,7 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) {
 static int32_t udfdInitLog() {
   char logName[12] = {0};
   snprintf(logName, sizeof(logName), "%slog", "udfd");
-  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
+  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
 }
 
 void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) {
@@ -659,7 +659,7 @@ int main(int argc, char *argv[]) {
     return -1;
   }
 
-  if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
     fnError("failed to start since read config error");
     return -1;
   }

source/libs/function/test/runUdf.c

@@ -31,13 +31,13 @@ static int32_t parseArgs(int32_t argc, char *argv[]) {
 static int32_t initLog() {
   char logName[12] = {0};
   snprintf(logName, sizeof(logName), "%slog", "udfc");
-  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
+  return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0);
 }
 
 int main(int argc, char *argv[]) {
   parseArgs(argc, argv);
   initLog();
-  if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
     fnError("failed to start since read config error");
     return -1;
   }

source/libs/nodes/inc/nodesUtil.h

@@ -27,10 +27,32 @@ extern "C" {
 #define nodesDebug(...) qDebug("NODES: " __VA_ARGS__)
 #define nodesTrace(...) qTrace("NODES: " __VA_ARGS__)
 
-#define NODES_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
-#define NODES_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
-#define NODES_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+#define NODES_ERR_RET(c)              \
+  do {                                \
+    int32_t _code = c;                \
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+      return _code;                   \
+    }                                 \
+  } while (0)
 
+#define NODES_RET(c)                  \
+  do {                                \
+    int32_t _code = c;                \
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+    }                                 \
+    return _code;                     \
+  } while (0)
+
+#define NODES_ERR_JRET(c)            \
+  do {                               \
+    code = c;                        \
+    if (code != TSDB_CODE_SUCCESS) { \
+      terrno = code;                 \
+      goto _return;                  \
+    }                                \
+  } while (0)
 
 #ifdef __cplusplus
 }

source/libs/nodes/src/nodesCodeFuncs.c

@@ -16,8 +16,8 @@
 #include "cmdnodes.h"
 #include "nodesUtil.h"
 #include "plannodes.h"
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 #include "taoserror.h"
 #include "tjson.h"
 
@@ -398,7 +398,8 @@ static int32_t tableMetaToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddObject(pJson, jkTableMetaComInfo, tableComInfoToJson, &pNode->tableInfo);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema), TABLE_TOTAL_COL_NUM(pNode));
+    code = tjsonAddArray(pJson, jkTableMetaColSchemas, schemaToJson, pNode->schema, sizeof(SSchema),
+                         TABLE_TOTAL_COL_NUM(pNode));
   }
 
   return code;
@@ -713,13 +714,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
-static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) {
-  return physiScanNodeToJson(pObj, pJson);
-}
+static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) { return physiScanNodeToJson(pObj, pJson); }
 
-static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
-  return jsonToPhysiScanNode(pJson, pObj);
-}
+static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); }
 
 static const char* jkTableScanPhysiPlanScanCount = "ScanCount";
 static const char* jkTableScanPhysiPlanReverseScanCount = "ReverseScanCount";
@@ -822,13 +819,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
-static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) {
-  return physiScanNodeToJson(pObj, pJson);
-}
+static int32_t physiStreamScanNodeToJson(const void* pObj, SJson* pJson) { return physiScanNodeToJson(pObj, pJson); }
 
-static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) {
-  return jsonToPhysiScanNode(pJson, pObj);
-}
+static int32_t jsonToPhysiStreamScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); }
 
 static const char* jkEndPointFqdn = "Fqdn";
 static const char* jkEndPointPort = "Port";
@@ -1331,13 +1324,9 @@ static int32_t jsonToPhysicDataSinkNode(const SJson* pJson, void* pObj) {
   return jsonToNodeObject(pJson, jkDataSinkInputDataBlockDesc, (SNode**)&pNode->pInputDataBlockDesc);
 }
 
-static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) {
-  return physicDataSinkNodeToJson(pObj, pJson);
-}
+static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return physicDataSinkNodeToJson(pObj, pJson); }
 
-static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) {
-  return jsonToPhysicDataSinkNode(pJson, pObj);
-}
+static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); }
 
 static const char* jkSubplanIdQueryId = "QueryId";
 static const char* jkSubplanIdGroupId = "GroupId";
@@ -1663,7 +1652,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
-static const char* jkValueGenByCalc = "GenByCalc";
+static const char* jkValueLiteralSize = "LiteralSize";
 static const char* jkValueLiteral = "Literal";
 static const char* jkValueDuration = "Duration";
 static const char* jkValueTranslate = "Translate";
@@ -1709,7 +1698,7 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) {
       break;
   }
 
-  return code ;
+  return code;
 }
 
 static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
@@ -1717,9 +1706,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
 
   int32_t code = exprNodeToJson(pObj, pJson);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc);
+    code = tjsonAddIntegerToObject(pJson, jkValueLiteralSize, NULL != pNode->literal ? strlen(pNode->literal) : 0);
   }
-  if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
+  if (TSDB_CODE_SUCCESS == code && NULL != pNode->literal) {
     code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
   }
   if (TSDB_CODE_SUCCESS == code) {
@@ -1789,10 +1778,11 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
   SValueNode* pNode = (SValueNode*)pObj;
 
   int32_t code = jsonToExprNode(pJson, pObj);
+  int32_t literalSize = 0;
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc);
+    code = tjsonGetIntValue(pJson, jkValueLiteralSize, &literalSize);
   }
-  if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) {
+  if (TSDB_CODE_SUCCESS == code && literalSize > 0) {
     code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal);
   }
   if (TSDB_CODE_SUCCESS == code) {
@@ -2014,7 +2004,8 @@ static int32_t vgroupsInfoToJson(const void* pObj, SJson* pJson) {
 
   int32_t code = tjsonAddIntegerToObject(pJson, jkVgroupsInfoNum, pNode->numOfVgroups);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo), pNode->numOfVgroups);
+    code = tjsonAddArray(pJson, jkVgroupsInfoVgroups, vgroupInfoToJson, pNode->vgroups, sizeof(SVgroupInfo),
+                         pNode->numOfVgroups);
   }
 
   return code;
@@ -2380,6 +2371,7 @@ static const char* jkSelectStmtOrderBy = "OrderBy";
 static const char* jkSelectStmtLimit = "Limit";
 static const char* jkSelectStmtSlimit = "Slimit";
 static const char* jkSelectStmtStmtName = "StmtName";
+static const char* jkSelectStmtHasAggFuncs = "HasAggFuncs";
 
 static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
   const SSelectStmt* pNode = (const SSelectStmt*)pObj;
@@ -2418,6 +2410,9 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddStringToObject(pJson, jkSelectStmtStmtName, pNode->stmtName);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkSelectStmtHasAggFuncs, pNode->hasAggFuncs);
+  }
 
   return code;
 }
@@ -2459,6 +2454,9 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetStringValue(pJson, jkSelectStmtStmtName, pNode->stmtName);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkSelectStmtHasAggFuncs, &pNode->hasAggFuncs);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "querynodes.h"
 #include "plannodes.h"
+#include "querynodes.h"
 
 typedef enum ETraversalOrder {
   TRAVERSAL_PREORDER = 1,
@@ -29,7 +29,8 @@ static EDealRes walkExprs(SNodeList* pNodeList, ETraversalOrder order, FNodeWalk
 static EDealRes walkPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext);
 static EDealRes walkPhysiPlans(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext);
 
-static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext, FNodeDispatcher dispatcher) {
+static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker, void* pContext,
+                         FNodeDispatcher dispatcher) {
   if (NULL == pNode) {
     return DEAL_RES_CONTINUE;
   }
@@ -395,7 +396,8 @@ static EDealRes walkScanPhysi(SScanPhysiNode* pScan, ETraversalOrder order, FNod
   return res;
 }
 
-static EDealRes walkTableScanPhysi(STableScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker, void* pContext) {
+static EDealRes walkTableScanPhysi(STableScanPhysiNode* pScan, ETraversalOrder order, FNodeWalker walker,
+                                   void* pContext) {
   EDealRes res = walkScanPhysi((SScanPhysiNode*)pScan, order, walker, pContext);
   if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
     res = walkPhysiPlans(pScan->pDynamicScanFuncs, order, walker, pContext);

source/libs/parser/inc/parInt.h

@@ -20,9 +20,9 @@
 extern "C" {
 #endif
 
-#include "parser.h"
 #include "parToken.h"
 #include "parUtil.h"
+#include "parser.h"
 
 int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery);
 int32_t parse(SParseContext* pParseCxt, SQuery** pQuery);