enh(rpc): fix hb problem

Jenkinsfile2

@@ -88,12 +88,6 @@ def pre_test(){
     cmake .. > /dev/null
     make -j4> /dev/null
     '''
-    sh'''
-    cd ${WKPY}
-    git reset --hard
-    git pull
-    pip3 install .
-    '''
     return 1
 }
 
@@ -103,7 +97,6 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
-      WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
   }
   stages {
       stage('pre_build'){
@@ -124,11 +117,6 @@ pipeline {
               ./test-all.sh b1fq
               '''
               sh'''
-              export LD_LIBRARY_PATH=${WKC}/debug/build/lib
-              cd ${WKC}/tests/system-test
-              ./fulltest.sh
-              '''
-              sh'''
               cd ${WKC}/debug
               ctest
               '''

cmake/cmake.define

 cmake_minimum_required(VERSION 3.16)
 
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
@@ -43,11 +45,14 @@ ENDIF ()
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
     SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
-    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(COMMON_FLAGS "/W3 /D_WIN32")
 
-    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
-      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
-    ENDIF ()
+    # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+    #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    # ENDIF ()
+
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
 
 ELSE ()
     IF (${SANITIZER} MATCHES "true")

cmake/cmake.options

@@ -25,9 +25,28 @@ IF(${TD_WINDOWS})
             ON
           )
 
-ENDIF ()
+    MESSAGE("build msvcregex Win32")
+    option(
+            BUILD_MSVCREGEX
+            "If build msvcregex on Windows"
+            ON
+          )
 
-IF(${TD_LINUX} MATCHES TRUE)
+    MESSAGE("build wcwidth Win32")
+    option(
+            BUILD_WCWIDTH
+            "If build wcwidth on Windows"
+            ON
+          )
+
+    MESSAGE("build wingetopt Win32")
+    option(
+            BUILD_WINGETOPT 
+                "If build wingetopt on Windows"
+            ON
+        )
+
+ENDIF ()
 
 option(
     BUILD_TEST 
@@ -35,8 +54,6 @@ option(
     ON
 )
 
-ENDIF ()
-
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 

cmake/msvcregex_CMakeLists.txt.in

+
+# msvcregex
+ExternalProject_Add(msvcregex
+        GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/pthread_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(pthread
         GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
         GIT_TAG v3.0.3.1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/wcwidth_CMakeLists.txt.in

+
+# wcwidth
+ExternalProject_Add(wcwidth
+        GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/wingetopt_CMakeLists.txt.in

+
+# wingetopt
+ExternalProject_Add(wingetopt
+        GIT_REPOSITORY https://github.com/alex85k/wingetopt.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

contrib/CMakeLists.txt

@@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
     cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
-# gnu regex
-if(${BUILD_GNUREGEX})
-    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+# iconv
+if(${BUILD_WITH_ICONV})
+    cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# msvc regex
+if(${BUILD_MSVCREGEX})
+    cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wcwidth
+if(${BUILD_WCWIDTH})
+    cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
@@ -99,8 +114,27 @@ if(${BUILD_TEST})
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
-        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
     )
+    if(${TD_WINDOWS})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
+        )
+    endif(${TD_WINDOWS})
+    if(${TD_LINUX})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
+        )
+    endif(${TD_LINUX})
+    if(${TD_DARWIN})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
+        )
+    endif(${TD_DARWIN})
+    
+    
 endif(${BUILD_TEST})
 
 # cJson
@@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT})
     add_subdirectory(nuraft)
 endif(${BUILD_WITH_NURAFT})
 
+# pthread
+if(${BUILD_PTHREAD})
+    set(CMAKE_BUILD_TYPE release)
+    add_definitions(-DPTW32_STATIC_LIB) 
+    add_subdirectory(pthread)
+    set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
+    add_library(pthread STATIC IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+endif()
+
+# iconv
+if(${BUILD_WITH_ICONV})
+    add_subdirectory(iconv)
+endif(${BUILD_WITH_ICONV})
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    add_subdirectory(wingetopt)
+endif(${BUILD_WINGETOPT})
+
+# msvcregex
+if(${BUILD_MSVCREGEX})
+    add_library(msvcregex STATIC "")
+    target_sources(msvcregex
+        PRIVATE "msvcregex/regex.c"
+    )
+    target_include_directories(msvcregex
+        PRIVATE "msvcregex"
+    )
+    target_link_libraries(msvcregex 
+        INTERFACE Shell32
+    )
+    SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
+endif(${BUILD_MSVCREGEX})
+
+# msvcregex
+if(${BUILD_WCWIDTH})
+    add_library(wcwidth STATIC "")
+    target_sources(wcwidth
+        PRIVATE "wcwidth/wcwidth.c"
+    )
+    target_include_directories(wcwidth
+        PRIVATE "wcwidth"
+    )
+    SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
+endif(${BUILD_WCWIDTH})
+
 # CRAFT
 if(${BUILD_WITH_CRAFT})
     add_library(craft STATIC IMPORTED GLOBAL)
@@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
     target_link_libraries(sqlite 
         INTERFACE m 
         INTERFACE pthread 
-        INTERFACE dl
     )
+    if(NOT TD_WINDOWS)
+        target_link_libraries(sqlite 
+            INTERFACE dl
+        )
+    endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
 # pthread

contrib/test/craft/CMakeLists.txt

 add_executable(simulate_vnode "simulate_vnode.c")
-target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
+target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
\ No newline at end of file
+if(${BUILD_WINGETOPT})
+    target_link_libraries(simulate_vnode PUBLIC wingetopt)
+    target_include_directories(
+        simulate_vnode
+        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+    )
+endif()
\ No newline at end of file

contrib/test/tdev/src/main.c

@@ -6,43 +6,39 @@
 #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
 #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
 
-#define tPutA(buf, val)              \
-  ({                                 \
-    memcpy(buf, &val, sizeof(val));  \
-    POINTER_SHIFT(buf, sizeof(val)); \
-  })
+static inline void tPutA(void **buf, uint64_t val) {
+  memcpy(buf, &val, sizeof(val));
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutB(buf, val)                         \
-  ({                                            \
-    ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
-    ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
-    ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
-    ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
-    ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
-    ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
-    ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;  \
-    ((uint8_t *)buf)[0] = (val)&0xff;           \
-    POINTER_SHIFT(buf, sizeof(val));            \
-  })
+static inline void tPutB(void **buf, uint64_t val) {
+  ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
+  ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
+  ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
+  ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
+  ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
+  ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
+  ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
+  ((uint8_t *)buf)[0] = (val)&0xff;
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutC(buf, val)                \
-  ({                                   \
-    if (buf) {                         \
-      ((uint64_t *)buf)[0] = (val);    \
-      POINTER_SHIFT(buf, sizeof(val)); \
-    }                                  \
-    NULL;                                 \
-  })
+static inline void tPutC(void **buf, uint64_t val) {
+  if (buf) {
+    ((uint64_t *)buf)[0] = (val);
+    POINTER_SHIFT(buf, sizeof(val));
+  }
+  *buf = NULL;
+}
 
-#define tPutD(buf, val)                        \
-  ({                                           \
-    uint64_t tmp = val;                        \
-    for (size_t i = 0; i < sizeof(val); i++) { \
-      ((uint8_t *)buf)[i] = tmp & 0xff;        \
-      tmp >>= 8;                               \
-    }                                          \
-    POINTER_SHIFT(buf, sizeof(val));           \
-  })
+static inline void tPutD(void **buf, uint64_t val) {
+  uint64_t tmp = val;
+  for (size_t i = 0; i < sizeof(val); i++) {
+    ((uint8_t *)buf)[i] = tmp & 0xff;
+    tmp >>= 8;
+  }
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
 static inline void tPutE(void **buf, uint64_t val) {
   if (buf) {
@@ -61,7 +57,7 @@ static void func(T t) {
   switch (t) {
     case A:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutA(pBuf, val);
+        tPutA(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -69,7 +65,7 @@ static void func(T t) {
       break;
     case B:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutB(pBuf, val);
+        tPutB(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -77,7 +73,7 @@ static void func(T t) {
       break;
     case C:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutC(pBuf, val);
+        tPutC(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -85,7 +81,7 @@ static void func(T t) {
       break;
     case D:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutD(pBuf, val);
+        tPutD(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }

example/CMakeLists.txt

@@ -39,6 +39,7 @@ target_include_directories(tstream
 
 target_include_directories(demoapi
     PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

example/src/demoapi.c

@@ -4,10 +4,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
+// #include <unistd.h>
 #include <inttypes.h>
+#ifndef WINDOWS
 #include <argp.h>
-
+#endif
+#include "osSleep.h"
 #include "taos.h"
 
 #define debugPrint(fmt, ...) \
@@ -32,6 +34,7 @@
 int64_t g_num_of_tb = 2;
 int64_t g_num_of_rec = 3;
 
+#ifndef WINDOWS
 static struct argp_option options[] = {
     {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
     {"records", 'n', "NUMBER", 0,
@@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 }
 
 static struct argp argp = {options, parse_opt, "", ""};
-
+#endif
 static void prepare_data(TAOS* taos) {
     TAOS_RES *res;
     res = taos_query(taos, "drop database if exists test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
 
     res = taos_query(taos, "create database test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
     taos_select_db(taos, "test");
 
     char command[1024] = {0};
@@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) {
     taos_free_result(res);
 
     for (int64_t i = 0; i < g_num_of_tb; i ++) {
+        // sprintf(command, "create table t%"PRId64" using meters "
+        //         "tags(%"PRId64", '%s', '%s', '%s');",
+        //         i, i, (i%2)?"beijing":"shanghai",
+        //         (i%2)?"朝阳区":"黄浦区",
+        //         (i%2)?"长安街":"中山路");
         sprintf(command, "create table t%"PRId64" using meters "
-                "tags(%"PRId64", '%s', '%s', '%s');",
-                i, i, (i%2)?"beijing":"shanghai",
-                (i%2)?"朝阳区":"黄浦区",
-                (i%2)?"长安街":"中山路");
-/*        sprintf(command, "create table t%"PRId64" using meters "
                 "tags(%"PRId64", '%s', '%s', '%s');",
                 i, i,
                 (i%2)?"beijing":"shanghai",
                 (i%2)?"chaoyang":"huangpu",
                 (i%2?"changan street":"jianguo rd"));
-                */
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             okPrint("t%" PRId64 " created\n", i);
@@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) {
                     "'%c%d', '%s%c%d', '%c%d')",
                     i, 1650000000000+j, (float)j, j,
                     'a'+(int)j%25, rand(),
-                    "涛思", 'z' - (int)j%25, rand(),
+                    // "涛思", 'z' - (int)j%25, rand(),
+                    "TAOS", 'z' - (int)j%25, rand(),
                     'b' - (int)j%25, rand()
                     );
             res = taos_query(taos,  command);
@@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %"PRId64"\n",
                                             f, c,
-                                            *(int64_t*)(row[f]+c*sizeof(int64_t)));
+                                            *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
                                 }
                                 break;
 
@@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %d\n",
                                             f, c,
-                                            *(int32_t*)(row[f]+c*sizeof(int32_t)));
+                                            *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
                                 }
                                 break;
 
@@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %f\n",
                                             f, c,
-                                            *(float*)(row[f]+c*sizeof(float)));
+                                            *(float*)((char*)(row[f])+c*sizeof(float)));
                                 }
                                 break;
 
@@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     if (offsets) {
                         for (int c = 0; c < rows; c++) {
                             if (offsets[c] != -1) {
-                                int length = *(int16_t*)(row[f] + offsets[c]);
+                                int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
                                 char *buf = calloc(1, length + 1);
-                                strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
+                                strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
                                 printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
                                         c, f, offsets[c], length, buf);
                                 free(buf);
@@ -314,8 +317,9 @@ int main(int argc, char *argv[]) {
     const char* host = "127.0.0.1";
     const char* user = "root";
     const char* passwd = "taosdata";
-
+#ifndef WINDOWS
     argp_parse(&argp, argc, argv, 0, 0, NULL);
+#endif
     TAOS* taos = taos_connect(host, user, passwd, "", 0);
     if (taos == NULL) {
         printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));

example/src/tmq.c

@@ -101,8 +101,7 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*const char* sql = "select * from tu1";*/
-  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
   pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));

include/client/taos.h

@@ -216,13 +216,14 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
 
 typedef struct tmq_conf_t tmq_conf_t;
 typedef struct tmq_list_t tmq_list_t;
-// typedef struct tmq_message_t tmq_message_t;
 
 typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
+DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
 #if 0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
@@ -262,12 +263,6 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
 
-#if 0
-// temporary used function for demo only
-void    tmqShowMsg(tmq_message_t *tmq_message);
-int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
-#endif
-
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
 DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
@@ -278,12 +273,8 @@ DLL_EXPORT char   *tmq_get_block_table_name(TAOS_RES *res);
 #endif
 
 #if 0
-DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
-DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
-DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
-DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
 #endif
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0

include/common/tglobal.h

@@ -94,6 +94,9 @@ extern bool    tsRetrieveBlockingModel;  // retrieve threads will be blocked
 extern bool    tsKeepOriginalColumnName;
 extern bool    tsDeadLockKillQuery;
 
+// query client
+extern int32_t tsQueryPolicy;
+
 // client
 extern int32_t tsMaxWildCardsLen;
 extern int32_t tsMaxRegexStringLen;

include/common/tmsg.h

@@ -639,13 +639,19 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 
 typedef struct {
-  SArray* epSetList;  // SArray<SEpSet>
+  SArray* addrsList;  // SArray<SQueryNodeAddr>
 } SQnodeListRsp;
 
 int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 void    tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
 
+typedef struct SQueryNodeAddr {
+  int32_t nodeId;  // vgId or qnodeId
+  SEpSet  epSet;
+} SQueryNodeAddr;
+
+
 typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
@@ -991,7 +997,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
 void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
-  int32_t type;
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
   int64_t showId;
@@ -1284,15 +1289,13 @@ typedef struct {
 } SMVCreateStreamRsp, SMSCreateStreamRsp;
 
 typedef struct {
-  char    name[TSDB_TOPIC_FNAME_LEN];
+  char    name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
   int8_t  igExists;
   int8_t  withTbName;
   int8_t  withSchema;
   int8_t  withTag;
-  int8_t  withTagSchema;
   char*   sql;
   char*   ast;
-  int64_t subDbUid;
   char    subscribeDbName[TSDB_DB_NAME_LEN];
 } SCMCreateTopicReq;
 

include/common/tmsgdef.h

@@ -175,11 +175,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@@ -205,6 +201,21 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
+  // sync integration
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

include/common/trow.h

@@ -1359,7 +1359,7 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
   printf(">>>");
   for (int i = 0; i < pSchema->numOfCols; ++i) {
     STColumn *stCol = pSchema->columns + i;
-    SCellVal  sVal = {.valType = 255, .val = NULL};
+    SCellVal  sVal = { 255, NULL};
     if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
       break;
     }

include/common/ttokendef.h

@@ -156,87 +156,96 @@
 #define TK_INTERVAL                       138
 #define TK_TOPIC                          139
 #define TK_AS                             140
-#define TK_DESC                           141
-#define TK_DESCRIBE                       142
-#define TK_RESET                          143
-#define TK_QUERY                          144
-#define TK_EXPLAIN                        145
-#define TK_ANALYZE                        146
-#define TK_VERBOSE                        147
-#define TK_NK_BOOL                        148
-#define TK_RATIO                          149
-#define TK_COMPACT                        150
-#define TK_VNODES                         151
-#define TK_IN                             152
-#define TK_OUTPUTTYPE                     153
-#define TK_AGGREGATE                      154
-#define TK_BUFSIZE                        155
-#define TK_STREAM                         156
-#define TK_INTO                           157
-#define TK_TRIGGER                        158
-#define TK_AT_ONCE                        159
-#define TK_WINDOW_CLOSE                   160
-#define TK_WATERMARK                      161
-#define TK_KILL                           162
-#define TK_CONNECTION                     163
-#define TK_MERGE                          164
-#define TK_VGROUP                         165
-#define TK_REDISTRIBUTE                   166
-#define TK_SPLIT                          167
-#define TK_SYNCDB                         168
-#define TK_NULL                           169
-#define TK_NK_QUESTION                    170
-#define TK_NK_ARROW                       171
-#define TK_ROWTS                          172
-#define TK_TBNAME                         173
-#define TK_QSTARTTS                       174
-#define TK_QENDTS                         175
-#define TK_WSTARTTS                       176
-#define TK_WENDTS                         177
-#define TK_WDURATION                      178
-#define TK_CAST                           179
-#define TK_NOW                            180
-#define TK_TODAY                          181
-#define TK_TIMEZONE                       182
-#define TK_COUNT                          183
-#define TK_FIRST                          184
-#define TK_LAST                           185
-#define TK_LAST_ROW                       186
-#define TK_BETWEEN                        187
-#define TK_IS                             188
-#define TK_NK_LT                          189
-#define TK_NK_GT                          190
-#define TK_NK_LE                          191
-#define TK_NK_GE                          192
-#define TK_NK_NE                          193
-#define TK_MATCH                          194
-#define TK_NMATCH                         195
-#define TK_CONTAINS                       196
-#define TK_JOIN                           197
-#define TK_INNER                          198
-#define TK_SELECT                         199
-#define TK_DISTINCT                       200
-#define TK_WHERE                          201
-#define TK_PARTITION                      202
-#define TK_BY                             203
-#define TK_SESSION                        204
-#define TK_STATE_WINDOW                   205
-#define TK_SLIDING                        206
-#define TK_FILL                           207
-#define TK_VALUE                          208
-#define TK_NONE                           209
-#define TK_PREV                           210
-#define TK_LINEAR                         211
-#define TK_NEXT                           212
-#define TK_GROUP                          213
-#define TK_HAVING                         214
-#define TK_ORDER                          215
-#define TK_SLIMIT                         216
-#define TK_SOFFSET                        217
-#define TK_LIMIT                          218
-#define TK_OFFSET                         219
-#define TK_ASC                            220
-#define TK_NULLS                          221
+#define TK_WITH                           141
+#define TK_SCHEMA                         142
+#define TK_DESC                           143
+#define TK_DESCRIBE                       144
+#define TK_RESET                          145
+#define TK_QUERY                          146
+#define TK_EXPLAIN                        147
+#define TK_ANALYZE                        148
+#define TK_VERBOSE                        149
+#define TK_NK_BOOL                        150
+#define TK_RATIO                          151
+#define TK_COMPACT                        152
+#define TK_VNODES                         153
+#define TK_IN                             154
+#define TK_OUTPUTTYPE                     155
+#define TK_AGGREGATE                      156
+#define TK_BUFSIZE                        157
+#define TK_STREAM                         158
+#define TK_INTO                           159
+#define TK_TRIGGER                        160
+#define TK_AT_ONCE                        161
+#define TK_WINDOW_CLOSE                   162
+#define TK_WATERMARK                      163
+#define TK_KILL                           164
+#define TK_CONNECTION                     165
+#define TK_MERGE                          166
+#define TK_VGROUP                         167
+#define TK_REDISTRIBUTE                   168
+#define TK_SPLIT                          169
+#define TK_SYNCDB                         170
+#define TK_NULL                           171
+#define TK_NK_QUESTION                    172
+#define TK_NK_ARROW                       173
+#define TK_ROWTS                          174
+#define TK_TBNAME                         175
+#define TK_QSTARTTS                       176
+#define TK_QENDTS                         177
+#define TK_WSTARTTS                       178
+#define TK_WENDTS                         179
+#define TK_WDURATION                      180
+#define TK_CAST                           181
+#define TK_NOW                            182
+#define TK_TODAY                          183
+#define TK_TIMEZONE                       184
+#define TK_COUNT                          185
+#define TK_FIRST                          186
+#define TK_LAST                           187
+#define TK_LAST_ROW                       188
+#define TK_BETWEEN                        189
+#define TK_IS                             190
+#define TK_NK_LT                          191
+#define TK_NK_GT                          192
+#define TK_NK_LE                          193
+#define TK_NK_GE                          194
+#define TK_NK_NE                          195
+#define TK_MATCH                          196
+#define TK_NMATCH                         197
+#define TK_CONTAINS                       198
+#define TK_JOIN                           199
+#define TK_INNER                          200
+#define TK_SELECT                         201
+#define TK_DISTINCT                       202
+#define TK_WHERE                          203
+#define TK_PARTITION                      204
+#define TK_BY                             205
+#define TK_SESSION                        206
+#define TK_STATE_WINDOW                   207
+#define TK_SLIDING                        208
+#define TK_FILL                           209
+#define TK_VALUE                          210
+#define TK_NONE                           211
+#define TK_PREV                           212
+#define TK_LINEAR                         213
+#define TK_NEXT                           214
+#define TK_GROUP                          215
+#define TK_HAVING                         216
+#define TK_ORDER                          217
+#define TK_SLIMIT                         218
+#define TK_SOFFSET                        219
+#define TK_LIMIT                          220
+#define TK_OFFSET                         221
+#define TK_ASC                            222
+#define TK_NULLS                          223
+#define TK_ID                             224
+#define TK_NK_BITNOT                      225
+#define TK_INSERT                         226
+#define TK_VALUES                         227
+#define TK_IMPORT                         228
+#define TK_NK_SEMI                        229
+#define TK_FILE                           230
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -244,13 +253,6 @@
 #define TK_NK_HEX                         303   // hex number  0x123
 #define TK_NK_OCT                         304   // oct number
 #define TK_NK_BIN                         305   // bin format data 0b111
-#define TK_NK_FILE                        306
-
-#define TK_NK_BITNOT                      501
-#define TK_INSERT                         502
-#define TK_VALUES                         507
-#define TK_IMPORT                         509
-#define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535
 

include/dnode/bnode/bnode.h

@@ -26,6 +26,9 @@ extern "C" {
 typedef struct SBnode SBnode;
 
 typedef struct {
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 } SBnodeLoad;
 
 typedef struct {

include/libs/catalog/catalog.h

@@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const
  */
 int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
-
 int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
 
 int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);

include/libs/function/function.h

@@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
 typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -113,7 +113,7 @@ typedef struct SResultRowEntryInfo {
   bool     initialized:1;     // output buffer has been initialized
   bool     complete:1;        // query has completed
   uint8_t  isNullRes:6;       // the result is null
-  uint8_t  numOfRes;        // num of output result in current buffer
+  uint8_t  numOfRes;          // num of output result in current buffer
 } SResultRowEntryInfo;
 
 // determine the real data need to calculated the result
@@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData {
   SColumnInfoData  *pPTS;           // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;
+  uint64_t          uid;            // table uid
 } SInputColumnInfoData;
 
 // sql function runtime context
@@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx {
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
   int32_t          numOfParams;
-  SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  SFunctParam     *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t         *ptsList;       // corresponding timestamp array list
   SColumnInfoData *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
   int32_t          offset;

include/libs/nodes/cmdnodes.h

@@ -241,12 +241,20 @@ typedef struct SDropComponentNodeStmt {
   int32_t dnodeId;
 } SDropComponentNodeStmt;
 
+typedef struct STopicOptions {
+  ENodeType type;
+  bool withTable;
+  bool withSchema;
+  bool withTag;
+} STopicOptions;
+
 typedef struct SCreateTopicStmt {
   ENodeType type;
   char topicName[TSDB_TABLE_NAME_LEN];
   char subscribeDbName[TSDB_DB_NAME_LEN];
   bool ignoreExists;
   SNode* pQuery;
+  STopicOptions* pOptions;
 } SCreateTopicStmt;
 
 typedef struct SDropTopicStmt {

include/libs/nodes/nodes.h

@@ -84,6 +84,7 @@ typedef enum ENodeType {
   QUERY_NODE_INDEX_OPTIONS,
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
+  QUERY_NODE_TOPIC_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,

include/libs/nodes/querynodes.h

@@ -237,8 +237,8 @@ typedef struct SSelectStmt {
   SNodeList* pGroupByList; // SGroupingSetNode
   SNode* pHaving;
   SNodeList* pOrderByList; // SOrderByExprNode
-  SNode* pLimit;
-  SNode* pSlimit;
+  SLimitNode* pLimit;
+  SLimitNode* pSlimit;
   char stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t precision;
   bool isEmptyResult;

include/libs/planner/planner.h

@@ -35,6 +35,8 @@ typedef struct SPlanContext {
   int8_t triggerType;
   int64_t watermark;
   bool isStmtQuery;
+  void* pTransporter;
+  struct SCatalog* pCatalog;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.

include/libs/qcom/query.h

@@ -51,6 +51,10 @@ typedef struct STableComInfo {
 
 typedef struct SIndexMeta {
 
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
+
 } SIndexMeta;
 
 
@@ -129,12 +133,6 @@ typedef struct SMsgSendInfo {
   SDataBuf             msgInfo;
 } SMsgSendInfo;
 
-typedef struct SQueryNodeAddr {
-  int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epSet;
-} SQueryNodeAddr;
-
-
 typedef struct SQueryNodeStat {
   int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
-#include "taosdef.h"
-#include "trpc.h"
-#include "wal.h"
+//#include <tdatablock.h>
+#include "cJSON.h"
+#include "tdef.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  uint16_t nodePort;
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
-typedef struct SSyncFSM {
-  void* data;
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                     ESyncState state);
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
 
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
+typedef struct SSyncFSM {
+  void* data;
 
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
 
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN   0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
-typedef struct SStateMgr {
-  void* data;
-
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+struct SWal;
+typedef struct SWal SWal;
 
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
+struct SEpSet;
+typedef struct SEpSet SEpSet;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,27 +140,35 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
-struct SSyncNode;
-typedef struct SSyncNode SSyncNode;
+int32_t     syncInit();
+void        syncCleanUp();
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
+void        syncStop(int64_t rid);
+int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
+ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
 
-int32_t syncInit();
-void    syncCleanUp();
+typedef enum {
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
 
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
-void       syncStop(int64_t rid);
-int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-ESyncState syncGetMyRole(int64_t rid);
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
-// propose with sequence number, to implement linearizable semantics
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
+extern int32_t sDebugFlag;
 
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+//-----------------------------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
 
-extern int32_t sDebugFlag;
+struct SSyncBuffer;
+typedef struct SSyncBuffer SSyncBuffer;
+//-----------------------------------------
 
-const char *syncStr(ESyncState state);
+const char* syncStr(ESyncState state);
 
 #ifdef __cplusplus
 }

include/libs/sync/syncTools.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -43,7 +43,7 @@ typedef struct SRpcMsg {
   int32_t code;
   void *  handle;         // rpc handle returned to app
   void *  ahandle;        // app handle set by client
-  int64_t refId;          //
+  int64_t refId;          // refid, used by server
   int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int     persistHandle;  // persist handle or not
 

include/os/os.h

@@ -22,7 +22,6 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
-#include <semaphore.h>
 
 #include <regex.h>
 
@@ -51,6 +50,15 @@ extern "C" {
 #include <sys/prctl.h>
 #include <argp.h>
 #endif
+#else
+
+#include <time.h>
+#ifndef TD_USE_WINSOCK
+#include <winsock2.h>
+#else
+#include <winsock.h>
+#endif
+
 
 #endif
 

include/os/osAtomic.h

@@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, int32_t val);
+void* atomic_add_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, int32_t val);
+void* atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
+void* atomic_sub_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
+void* atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);

include/os/osDef.h

@@ -191,7 +191,7 @@ extern "C" {
   #define threadlocal __declspec( thread )
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define PRIzu "ld"  
 #else
   #define PRIzu "zu"  

include/os/osDir.h

@@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr;
 
 
 void    taosRemoveDir(const char *dirname);
-bool    taosDirExist(char *dirname);
+bool    taosDirExist(const char *dirname);
 int32_t taosMkDir(const char *dirname);
 int32_t taosMulMkDir(const char *dirname);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
-int32_t taosRealPath(char *dirname, int32_t maxlen);
+int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
 char*   taosDirName(char *dirname);
 char*   taosDirEntryBaseName(char *dirname);
@@ -50,7 +50,7 @@ TdDirPtr      taosOpenDir(const char *dirname);
 TdDirEntryPtr taosReadDir(TdDirPtr pDir);
 bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
 char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
-int32_t       taosCloseDir(TdDirPtr pDir);
+int32_t       taosCloseDir(TdDirPtr *ppDir);
 
 #ifdef __cplusplus
 }

include/os/osEnv.h

@@ -34,6 +34,7 @@ extern int64_t         tsOpenMax;
 extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
+extern char*           tsProcPath;
 
 extern char configDir[];
 extern char tsDataDir[];

include/os/osFile.h

@@ -93,7 +93,6 @@ void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix,
 
 int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
 
-void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
 bool taosValidFile(TdFilePtr pFile);
 
 int32_t taosGetErrorFile(TdFilePtr pFile);

include/os/osLz4.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   int32_t BUILDIN_CLZL(uint64_t val);
   int32_t BUILDIN_CLZ(uint32_t val);
   int32_t BUILDIN_CTZL(uint64_t val);

include/os/osMath.h

@@ -23,7 +23,7 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define TSWAP(a, b, c) \
     do {                \
@@ -33,6 +33,7 @@ extern "C" {
     } while (0)
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
+  #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 

include/os/osSemaphore.h

@@ -22,21 +22,28 @@ extern "C" {
 
 #include <semaphore.h>
 
-#if defined (_TD_DARWIN_64)
-  typedef struct tsem_s *tsem_t;
-  int tsem_init(tsem_t *sem, int pshared, unsigned int value);
-  int tsem_wait(tsem_t *sem);
-  int tsem_post(tsem_t *sem);
-  int tsem_destroy(tsem_t *sem);
+#if defined(_TD_DARWIN_64)
+
+typedef struct tsem_s *tsem_t;
+
+int tsem_init(tsem_t *sem, int pshared, unsigned int value);
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+int tsem_post(tsem_t *sem);
+int tsem_destroy(tsem_t *sem);
+
 #else
-  #define tsem_t sem_t
-  #define tsem_init sem_init
-  int tsem_wait(tsem_t* sem);
-  #define tsem_post sem_post
-  #define tsem_destroy sem_destroy
+
+#define tsem_t       sem_t
+#define tsem_init    sem_init
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+#define tsem_post    sem_post
+#define tsem_destroy sem_destroy
+
 #endif
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
 //  #define TdThreadRwlock TdThreadMutex
 //  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
 //  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
@@ -44,20 +51,20 @@ extern "C" {
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define TdThreadSpinlock TdThreadMutex
-  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
-  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
-  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
+#define TdThreadSpinlock               TdThreadMutex
+#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
+#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
+#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
 int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(TdThread* thread);
+void    taosResetPthread(TdThread *thread);
 bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
-int32_t taosGetAppName(char* name, int32_t* len);
+int32_t taosGetAppName(char *name, int32_t *len);
 
 #ifdef __cplusplus
 }

include/os/osSocket.h

@@ -19,32 +19,53 @@
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-    #define socket SOCKET_FUNC_TAOS_FORBID
-    #define bind   BIND_FUNC_TAOS_FORBID
-    #define listen LISTEN_FUNC_TAOS_FORBID
-    #define accept ACCEPT_FUNC_TAOS_FORBID
-    #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
-    #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
-    #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
-    #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
-    #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
+  #define socket SOCKET_FUNC_TAOS_FORBID
+  #define bind   BIND_FUNC_TAOS_FORBID
+  #define listen LISTEN_FUNC_TAOS_FORBID
+  #define accept ACCEPT_FUNC_TAOS_FORBID
+  #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+  #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
+  #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
+  #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
+  #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
 #endif
 
 #if defined(WINDOWS)
-  #include "winsock2.h"
-  #include <WS2tcpip.h>
-  #include <winbase.h>
-  #include <Winsock2.h>
+  #if BYTE_ORDER == LITTLE_ENDIAN
+    #include <stdlib.h>
+    #define htobe16(x) _byteswap_ushort(x)
+    #define htole16(x) (x)
+    #define be16toh(x) _byteswap_ushort(x)
+    #define le16toh(x) (x)
+                      
+    #define htobe32(x) _byteswap_ulong(x)
+    #define htole32(x) (x)
+    #define be32toh(x) _byteswap_ulong(x)
+    #define le32toh(x) (x)
+                      
+    #define htobe64(x) _byteswap_uint64(x)
+    #define htole64(x) (x)
+    #define be64toh(x) _byteswap_uint64(x)
+    #define le64toh(x) (x)
+  #else
+    #error byte order not supported
+  #endif
+
+  #define __BYTE_ORDER    BYTE_ORDER
+  #define __BIG_ENDIAN    BIG_ENDIAN
+  #define __LITTLE_ENDIAN LITTLE_ENDIAN
+  #define __PDP_ENDIAN    PDP_ENDIAN
+
 #else
+  #include <netinet/in.h>
+  #include <sys/socket.h>
+
+  #if defined(_TD_DARWIN_64)
+    #include <osEok.h>
+  #else
     #include <netinet/in.h>
-    #include <sys/socket.h>
-
-    #if defined(_TD_DARWIN_64)
-        #include <osEok.h>
-    #else
-        #include <netinet/in.h>
-        #include <sys/epoll.h>
-    #endif
+    #include <sys/epoll.h>
+  #endif
 #endif
 
 #ifdef __cplusplus
@@ -52,14 +73,10 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  #define htobe64 htonll
-#endif
-
-#if defined(WINDOWS)
+  typedef int socklen_t;
   #define TAOS_EPOLL_WAIT_TIME 100
   typedef SOCKET eventfd_t;
   #define eventfd(a, b) -1
-  typedef SOCKET EpollFd;
   #define EpollClose(pollFd) epoll_close(pollFd)
   #ifndef EPOLLWAKEUP
     #define EPOLLWAKEUP (1u << 29)
@@ -79,31 +96,29 @@ extern "C" {
 #if defined(_TD_DARWIN_64)
 //  #define htobe64 htonll
 
-#	include <libkern/OSByteOrder.h>
+#include <libkern/OSByteOrder.h>
 
-#	define htobe16(x) OSSwapHostToBigInt16(x)
-#	define htole16(x) OSSwapHostToLittleInt16(x)
-#	define be16toh(x) OSSwapBigToHostInt16(x)
-#	define le16toh(x) OSSwapLittleToHostInt16(x)
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
 
-#	define htobe32(x) OSSwapHostToBigInt32(x)
-#	define htole32(x) OSSwapHostToLittleInt32(x)
-#	define be32toh(x) OSSwapBigToHostInt32(x)
-#	define le32toh(x) OSSwapLittleToHostInt32(x)
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
 
-#	define htobe64(x) OSSwapHostToBigInt64(x)
-#	define htole64(x) OSSwapHostToLittleInt64(x)
-#	define be64toh(x) OSSwapBigToHostInt64(x)
-#	define le64toh(x) OSSwapLittleToHostInt64(x)
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
 
-#	define __BYTE_ORDER    BYTE_ORDER
-#	define __BIG_ENDIAN    BIG_ENDIAN
-#	define __LITTLE_ENDIAN LITTLE_ENDIAN
-#	define __PDP_ENDIAN    PDP_ENDIAN
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
-
 typedef int32_t SocketFd;
 typedef SocketFd  EpollFd;
 
@@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr;
 int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+void taosWinSocketInit();
 
 TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
-TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
 
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen);
 
 void     taosBlockSIGPIPE();
 uint32_t taosGetIpv4FromFqdn(const char *);

include/os/osString.h

@@ -39,7 +39,7 @@ typedef int32_t TdUcs4;
     #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define tstrdup(str) _strdup(str)
 #else
   #define tstrdup(str) strdup(str)

include/os/osThread.h

@@ -22,9 +22,12 @@
 extern "C" {
 #endif
 
+#ifndef WINDOWS
 #ifndef __USE_XOPEN2K
+#define TD_USE_SPINLOCK_AS_MUTEX
 typedef pthread_mutex_t pthread_spinlock_t;
 #endif
+#endif
 
 typedef pthread_t TdThread;
 typedef pthread_spinlock_t TdThreadSpinlock;
@@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey;
 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop pthread_cleanup_pop
 
+
+#ifdef WINDOWS
+#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
+#else
+#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#endif
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
 void taosThreadTestCancel(void);
-int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
-int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
+void taosThreadClear(TdThread *thread);
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -20,8 +20,6 @@
 extern "C" {
 #endif
 
-#include <time.h>
-
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -34,7 +32,7 @@ extern "C" {
     #define mktime MKTIME_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define CLOCK_REALTIME 	0
 
@@ -58,6 +56,8 @@ extern "C" {
 
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
+int32_t taosClockGetTime(int clock_id, struct timespec *pTS);
+
 //@return timestamp in second
 int32_t taosGetTimestampSec();
 
@@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() {
 //@return timestamp in nanosecond
 static FORCE_INLINE int64_t taosGetTimestampNs() {
   struct timespec systemTime = {0};
-  clock_gettime(CLOCK_REALTIME, &systemTime);
+  taosClockGetTime(CLOCK_REALTIME, &systemTime);
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 

include/util/talgo.h

@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
  * @return
  */
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                    __ext_compar_fn_t compar, char* buf, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -98,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @return
  */
 void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                  bool maxroot);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -411,6 +411,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+// sync integration
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)
@@ -615,6 +619,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
 #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2633)
 #define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL      TAOS_DEF_ERROR_CODE(0, 0x2634)
+#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL   TAOS_DEF_ERROR_CODE(0, 0x2635)
+#define TSDB_CODE_PAR_INVALID_DAYS_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2636)
+#define TSDB_CODE_PAR_OFFSET_LESS_ZERO          TAOS_DEF_ERROR_CODE(0, 0x2637)
+#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY  TAOS_DEF_ERROR_CODE(0, 0x2638)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tcache.h

@@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
 /**
  * initialize the cache object
  * @param keyType              key type
- * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
+ * @param refreshTimeInMs      refresh operation interval time, the maximum survival time when one element is expired
  *                             and not referenced by other objects
  * @param extendLifespan       auto extend lifespan, if accessed
  * @param fn                   free resource callback function
  * @return
  */
-SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
+SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName);
 
 /**
@@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
  * @param pCacheObj
  * @return
  */
-size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
+size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj);
 
 /**
  *  move all data node into trash, clear node in trash can if it is not referenced by any clients
@@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
  */
 void taosStopCacheRefreshWorker();
 
-SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
-bool taosCacheIterNext(SCacheIter* pIter);
-void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
-void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
-void taosCacheDestroyIter(SCacheIter* pIter);
+SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj);
+bool        taosCacheIterNext(SCacheIter *pIter);
+void       *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen);
+void       *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen);
+void        taosCacheDestroyIter(SCacheIter *pIter);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -118,7 +118,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_INS_TABLE_USER_USERS             "user_users"
 #define TSDB_INS_TABLE_LICENCES               "grants"
 #define TSDB_INS_TABLE_VGROUPS                "vgroups"
-#define TSDB_INS_TABLE_TOPICS                 "topics"
 #define TSDB_INS_TABLE_CONSUMERS              "consumers"
 #define TSDB_INS_TABLE_SUBSCRIBES             "subscribes"
 #define TSDB_INS_TABLE_TRANS                  "trans"

include/util/tencode.h

@@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) {
   union {
     uint32_t ui;
     float    f;
-  } v = {.f = val};
+  } v;
+  v.f = val;
 
   return tEncodeU32(pEncoder, v.ui);
 }
@@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) {
   union {
     uint64_t ui;
     double   d;
-  } v = {.d = val};
+  } v;
+  v.d = val;
 
   return tEncodeU64(pEncoder, v.ui);
 }

include/util/tjson.h

@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#ifdef WINDOWS
+#define tjsonGetNumberValue(pJson, pName, val) -1
+#else
 #define tjsonGetNumberValue(pJson, pName, val) \
   ({ \
     uint64_t _tmp = 0; \
@@ -29,6 +32,7 @@ extern "C" {
     val = _tmp; \
     _code; \
   })
+#endif
 
 typedef void SJson;
 

source/client/inc/clientInt.h

@@ -219,6 +219,7 @@ void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
 void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
 int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
 void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;

source/client/src/clientEnv.c

@@ -30,15 +30,15 @@
 #define TSC_VAR_RELEASED    0
 
 SAppInfo appInfo;
-int32_t  clientReqRefPool  = -1;
+int32_t  clientReqRefPool = -1;
 int32_t  clientConnRefPool = -1;
 
-static TdThreadOnce   tscinit = PTHREAD_ONCE_INIT;
-volatile int32_t      tscInitRes = 0;
+static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
+volatile int32_t    tscInitRes = 0;
 
 static void registerRequest(SRequestObj *pRequest) {
   STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
-  
+
   assert(pTscObj != NULL);
 
   // connection has been released already, abort creating request.
@@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) {
   if (pTscObj->pAppInfo) {
     SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary;
 
-    int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1);
-    int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1);
+    int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
+    int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
     tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
              ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
              pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
@@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) {
 static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj *         pTscObj = pRequest->pTscObj;
+  STscObj          *pTscObj = pRequest->pTscObj;
   SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
 
-  int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1);
+  int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
   int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
   tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
            " ms, current:%d, app current:%d",
-           pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst);
+           pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
   releaseTscObj(pTscObj->id);
 }
 
@@ -109,12 +109,12 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
 }
 
 void closeAllRequests(SHashObj *pRequests) {
-  void   *pIter = taosHashIterate(pRequests, NULL);
+  void *pIter = taosHashIterate(pRequests, NULL);
   while (pIter != NULL) {
     int64_t *rid = pIter;
 
-    releaseRequest(*rid);    
-    
+    releaseRequest(*rid);
+
     pIter = taosHashIterate(pRequests, pIter);
   }
 }
@@ -144,7 +144,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
-  
+
   pObj->pAppInfo = pAppInfo;
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
@@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
   return pObj;
 }
 
-STscObj *acquireTscObj(int64_t rid) {
-  return (STscObj *)taosAcquireRef(clientConnRefPool, rid);
-}
+STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); }
 
-int32_t releaseTscObj(int64_t rid) {
-  return taosReleaseRef(clientConnRefPool, rid);
-}
+int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }
 
 void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
   assert(pObj != NULL);
@@ -190,11 +186,11 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
-    
+
   return pRequest;
 }
 
-static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
+void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   taosMemoryFreeClear(pResInfo->pRspMsg);
   taosMemoryFreeClear(pResInfo->length);
   taosMemoryFreeClear(pResInfo->row);
@@ -217,7 +213,7 @@ static void doDestroyRequest(void *p) {
   assert(RID_VALID(pRequest->self));
 
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
-  
+
   taosMemoryFreeClear(pRequest->msgBuf);
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFreeClear(pRequest->pDb);
@@ -244,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) {
   taosRemoveRef(clientReqRefPool, pRequest->self);
 }
 
-SRequestObj *acquireRequest(int64_t rid) {
-  return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid);
-}
-
-int32_t releaseRequest(int64_t rid) {
-  return taosReleaseRef(clientReqRefPool, rid);
-}
+SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
 
+int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
 
 void taos_init_imp(void) {
   // In the APIs of other program language, taos_cleanup is not available yet.
@@ -380,7 +371,7 @@ uint64_t generateRequestId() {
   }
 
   uint64_t id = 0;
-  
+
   while (true) {
     int64_t  ts = taosGetTimestampMs();
     uint64_t pid = taosGetPId();

source/client/src/clientImpl.c

@@ -95,16 +95,16 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
     if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
       return NULL;
     }
-
-    if (port) {
-      epSet.epSet.eps[0].port = port;
-    }
   } else {
     if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
       return NULL;
     }
   }
 
+  if (port) {
+    epSet.epSet.eps[0].port = port;
+  }
+
   char*          key = getClusterKey(user, secretEncrypt, ip, port);
   SAppInstInfo** pInst = NULL;
 
@@ -226,14 +226,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
   pRequest->type = pQuery->msgType;
-  SPlanContext cxt = {.queryId = pRequest->requestId,
-                      .acctId = pRequest->pTscObj->acctId,
-                      .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
-                      .pAstRoot = pQuery->pRoot,
-                      .showRewrite = pQuery->showRewrite};
-  int32_t      code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
-  if (code != 0) {
-    return code;
+  SPlanContext cxt = {
+    .queryId = pRequest->requestId,
+    .acctId = pRequest->pTscObj->acctId,
+    .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
+    .pAstRoot = pQuery->pRoot,
+    .showRewrite = pQuery->showRewrite,
+    .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter
+  };
+  int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
   }
   return code;
 }
@@ -302,8 +305,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
 }
 
 SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
-  SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
-
   if (TSDB_CODE_SUCCESS == code) {
     switch (pQuery->execMode) {
       case QUERY_EXEC_MODE_LOCAL:
@@ -312,12 +313,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
       case QUERY_EXEC_MODE_RPC:
         code = execDdlQuery(pRequest, pQuery);
         break;
-      case QUERY_EXEC_MODE_SCHEDULE:
+      case QUERY_EXEC_MODE_SCHEDULE: {
+        SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
         code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
         if (TSDB_CODE_SUCCESS == code) {
           code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
         }
+        taosArrayDestroy(pNodeList);
         break;
+      }
       case QUERY_EXEC_MODE_EMPTY_RESULT:
         pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
         break;
@@ -326,7 +330,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
     }
   }
 
-  taosArrayDestroy(pNodeList);
   if (!keepQuery) {
     qDestroyQuery(pQuery);
   }
@@ -436,7 +439,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
       return -1;
     }
 
-    taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return terrno;
+    }
+
     mgmtEpSet->numOfEps++;
   }
 

source/client/src/clientMain.c

@@ -135,6 +135,16 @@ void taos_free_result(TAOS_RES *res) {
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
     destroyRequest(pRequest);
+  } else if (TD_RES_TMQ(res)) {
+    SMqRspObj *pRsp = (SMqRspObj *)res;
+    if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
+    if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
+    if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema);
+    if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName);
+    if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
+    if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
+    pRsp->resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&pRsp->resInfo);
   }
 }
 

source/client/src/clientMsgHandler.c

@@ -151,13 +151,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     taosMemoryFreeClear(output.dbVgroup);
 
     tscError("failed to build use db output since %s", terrstr());
-  } else {
+  } else if (output.dbVgroup) {
     struct SCatalog* pCatalog = NULL;
 
     int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (code1 != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
               tstrerror(code1));
+      taosMemoryFreeClear(output.dbVgroup);
     } else {
       catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
     }

source/client/src/tmq.c

@@ -24,16 +24,6 @@
 #include "tqueue.h"
 #include "tref.h"
 
-#if 0
-struct tmq_message_t {
-  SMqPollRsp msg;
-  char*      topic;
-  SArray*    res;  // SArray<SReqResultInfo>
-  int32_t    vgId;
-  int32_t    resIter;
-};
-#endif
-
 typedef struct {
   int8_t  tmqRspType;
   int32_t epoch;
@@ -270,6 +260,16 @@ void tmq_list_destroy(tmq_list_t* list) {
   taosArrayDestroy(container);
 }
 
+int32_t tmq_list_get_size(const tmq_list_t* list) {
+  const SArray* container = &list->container;
+  return taosArrayGetSize(container);
+}
+
+char** tmq_list_to_c_array(const tmq_list_t* list) {
+  const SArray* container = &list->container;
+  return container->pData;
+}
+
 static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
   return sprintf(dst, "%s:%d", topicName, vg);
 }
@@ -397,7 +397,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->commit_cb = conf->commit_cb;
   pTmq->resetOffsetCfg = conf->resetOffset;
 
-  pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
+  pTmq->consumerId = tGenIdPI64();
   pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
   if (pTmq->clientTopics == NULL) {
     taosMemoryFree(pTmq);
@@ -770,105 +770,12 @@ _return:
 }
 #endif
 
-static char* formatTimestamp(char* buf, int64_t val, int precision) {
-  time_t  tt;
-  int32_t ms = 0;
-  if (precision == TSDB_TIME_PRECISION_NANO) {
-    tt = (time_t)(val / 1000000000);
-    ms = val % 1000000000;
-  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
-    tt = (time_t)(val / 1000000);
-    ms = val % 1000000;
-  } else {
-    tt = (time_t)(val / 1000);
-    ms = val % 1000;
-  }
-
-  /* comment out as it make testcases like select_with_tags.sim fail.
-    but in windows, this may cause the call to localtime crash if tt < 0,
-    need to find a better solution.
-    if (tt < 0) {
-      tt = 0;
-    }
-    */
-
-#ifdef WINDOWS
-  if (tt < 0) tt = 0;
-#endif
-  if (tt <= 0 && ms < 0) {
-    tt--;
-    if (precision == TSDB_TIME_PRECISION_NANO) {
-      ms += 1000000000;
-    } else if (precision == TSDB_TIME_PRECISION_MICRO) {
-      ms += 1000000;
-    } else {
-      ms += 1000;
-    }
-  }
-
-  struct tm* ptm = taosLocalTime(&tt, NULL);
-  size_t     pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
-
-  if (precision == TSDB_TIME_PRECISION_NANO) {
-    sprintf(buf + pos, ".%09d", ms);
-  } else if (precision == TSDB_TIME_PRECISION_MICRO) {
-    sprintf(buf + pos, ".%06d", ms);
-  } else {
-    sprintf(buf + pos, ".%03d", ms);
-  }
-
-  return buf;
-}
 #if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return 0;
   SMqPollRsp* pRsp = &tmq_message->msg;
   return pRsp->skipLogNum;
 }
-
-void tmqShowMsg(tmq_message_t* tmq_message) {
-  if (tmq_message == NULL) return;
-
-  static bool noPrintSchema;
-  char        pBuf[128];
-  SMqPollRsp* pRsp = &tmq_message->msg;
-  int32_t     colNum = 2;
-  if (!noPrintSchema) {
-    printf("|");
-    for (int32_t i = 0; i < colNum; i++) {
-      if (i == 0)
-        printf(" %25s |", pRsp->schema->pSchema[i].name);
-      else
-        printf(" %15s |", pRsp->schema->pSchema[i].name);
-    }
-    printf("\n");
-    printf("===============================================\n");
-    noPrintSchema = true;
-  }
-  int32_t sz = taosArrayGetSize(pRsp->pBlockData);
-  for (int32_t i = 0; i < sz; i++) {
-    SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i);
-    int32_t      rows = pDataBlock->info.rows;
-    for (int32_t j = 0; j < rows; j++) {
-      printf("|");
-      for (int32_t k = 0; k < colNum; k++) {
-        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-        switch (pColInfoData->info.type) {
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
-            printf(" %25s |", pBuf);
-            break;
-          case TSDB_DATA_TYPE_INT:
-          case TSDB_DATA_TYPE_UINT:
-            printf(" %15u |", *(uint32_t*)var);
-            break;
-        }
-      }
-      printf("\n");
-    }
-  }
-}
 #endif
 
 int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
@@ -1049,7 +956,6 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     }
     tDeleteSMqCMGetSubEpRsp(&rsp);
   } else {
-    /*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/
     SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
     if (pWrapper == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -1208,7 +1114,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   pRspObj->resIter = -1;
   memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp));
 
-  /*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
   setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
@@ -1355,31 +1260,6 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
   }
 }
 
-#if 0
-tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
-  tmq_message_t* rspMsg = NULL;
-  int64_t        startTime = taosGetTimestampMs();
-
-  int64_t status = atomic_load_64(&tmq->status);
-  tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
-
-  while (1) {
-    rspMsg = tmqSyncPollImpl(tmq, blocking_time);
-    if (rspMsg && rspMsg->consumeRsp.numOfTopics) {
-      return rspMsg;
-    }
-
-    if (blocking_time != 0) {
-      int64_t endTime = taosGetTimestampMs();
-      if (endTime - startTime > blocking_time) {
-        return NULL;
-      }
-    } else
-      return NULL;
-  }
-}
-#endif
-
 TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   SMqRspObj* rspObj;
   int64_t    startTime = taosGetTimestampMs();
@@ -1417,137 +1297,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   }
 }
 
-#if 0
-
-  if (blocking_time <= 0) blocking_time = 1;
-  if (blocking_time > 1000) blocking_time = 1000;
-  /*blocking_time = 1;*/
-
-  if (taosArrayGetSize(tmq->clientTopics) == 0) {
-    tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
-    /*printf("over1\n");*/
-    taosMsleep(blocking_time);
-    return NULL;
-  }
-  SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
-  if (taosArrayGetSize(pTopic->vgs) == 0) {
-    /*printf("over2\n");*/
-    taosMsleep(blocking_time);
-    return NULL;
-  }
-
-  tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
-  int32_t beginVgIdx = pTopic->nextVgIdx;
-  while (1) {
-    pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
-    SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
-    /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
-    SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
-    if (pReq == NULL) {
-      ASSERT(false);
-      taosMsleep(blocking_time);
-      return NULL;
-    }
-
-    SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam));
-    if (param == NULL) {
-      ASSERT(false);
-      taosMsleep(blocking_time);
-      return NULL;
-    }
-    param->tmq = tmq;
-    param->retMsg = &tmq_message;
-    param->pVg = pVg;
-    tsem_init(&param->rspSem, 0, 0);
-
-    SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
-    pRequest->body.requestMsg = (SDataBuf){
-        .pData = pReq,
-        .len = sizeof(SMqConsumeReq),
-        .handle = NULL,
-    };
-
-    SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-    sendInfo->requestObjRefId = 0;
-    sendInfo->param = param;
-    sendInfo->fp = tmqPollCb;
-
-    /*printf("req offset: %ld\n", pReq->offset);*/
-
-    int64_t transporterId = 0;
-    asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-    tmq->pollCnt++;
-
-    tsem_wait(&param->rspSem);
-    tsem_destroy(&param->rspSem);
-    taosMemoryFree(param);
-
-    if (tmq_message == NULL) {
-      if (beginVgIdx == pTopic->nextVgIdx) {
-        taosMsleep(blocking_time);
-      } else {
-        continue;
-      }
-    }
-
-    return tmq_message;
-  }
-
-  /*tsem_wait(&pRequest->body.rspSem);*/
-
-  /*if (body != NULL) {*/
-  /*destroySendMsgInfo(body);*/
-  /*}*/
-
-  /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/
-  /*pRequest->code = terrno;*/
-  /*}*/
-
-  /*return pRequest;*/
-}
-#endif
-
-#if 0
-tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
-  if (tmq_topic_vgroup_list != NULL) {
-    // TODO
-  }
-
-  // TODO: change semaphore to gate
-  for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
-    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-    for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
-      SMqClientVg*   pVg = taosArrayGet(pTopic->vgs, j);
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg);
-
-      SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
-      pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
-      SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam));
-      if (pParam == NULL) {
-        continue;
-      }
-      pParam->tmq = tmq;
-      pParam->pVg = pVg;
-      pParam->async = async;
-      if (!async) tsem_init(&pParam->rspSem, 0, 0);
-
-      SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-      sendInfo->requestObjRefId = 0;
-      sendInfo->param = pParam;
-      sendInfo->fp = tmqCommitCb;
-
-      int64_t transporterId = 0;
-      asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
-
-      if (!async) tsem_wait(&pParam->rspSem);
-    }
-  }
-
-  return 0;
+tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
+  // TODO
+  return TMQ_RESP_ERR__SUCCESS;
 }
-#endif
-
-tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
 
 const char* tmq_err2str(tmq_resp_err_t err) {
   if (err == TMQ_RESP_ERR__SUCCESS) {
@@ -1573,10 +1326,3 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
     return -1;
   }
 }
-
-void tmq_message_destroy(TAOS_RES* res) {
-  if (res == NULL) return;
-  if (TD_RES_TMQ(res)) {
-    SMqRspObj* pRspObj = (SMqRspObj*)res;
-  }
-}

source/client/test/clientTests.cpp

@@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select count(*) from tu");
+  pRes = taos_query(pConn, "select now() from m1");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/CMakeLists.txt

@@ -5,8 +5,8 @@ target_include_directories(
     PUBLIC "${TD_SOURCE_DIR}/include/common"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 IF(${TD_WINDOWS})
-    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32"
-    PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex"
 ENDIF ()
 )
 target_link_libraries(

source/common/src/tdatablock.c

@@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
 
   if (pEp->port == 0) {
     pEp->port = tsServerPort;
-    return -1;
   }
 
   return 0;

source/common/src/tglobal.c

@@ -80,6 +80,9 @@ int32_t  tsTelemInterval = 86400;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
 uint16_t tsTelemPort = 80;
 
+// query
+int32_t tsQueryPolicy = 1;
+
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
  * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@@ -322,6 +325,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 4;
   tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
@@ -515,6 +519,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
   tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
+  tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   return 0;
 }
 

source/common/src/tmsg.c

@@ -114,6 +114,19 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
   return 0;
 }
 
+int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) {
+  if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
+  if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+
 int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pEp->inUse);
@@ -2058,11 +2071,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp)
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  int32_t num = taosArrayGetSize(pRsp->epSetList);
+  int32_t num = taosArrayGetSize(pRsp->addrsList);
   if (tEncodeI32(&encoder, num) < 0) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    SEpSet *epSet = taosArrayGet(pRsp->epSetList, i);
-    if (tEncodeSEpSet(&encoder, epSet) < 0) return -1;
+    SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i);
+    if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1;
   }
   tEndEncode(&encoder);
 
@@ -2078,10 +2091,10 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   if (tStartDecode(&decoder) < 0) return -1;
   int32_t num = 0;
   if (tDecodeI32(&decoder, &num) < 0) return -1;
-  pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet));
-  if (NULL == pRsp->epSetList) return -1;
+  pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
+  if (NULL == pRsp->addrsList) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1;
+    if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
   }
   tEndDecode(&decoder);
 
@@ -2089,7 +2102,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   return 0;
 }
 
-void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
+void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); }
 
 int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
   SCoder encoder = {0};
@@ -2462,7 +2475,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   tEndEncode(&encoder);
@@ -2478,7 +2490,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   tEndDecode(&decoder);
@@ -2736,11 +2747,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
   if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
+  if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
 
   tEndEncode(&encoder);
 
@@ -2762,7 +2773,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
   if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1;
   if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
   if (tDecodeI32(&decoder, &astLen) < 0) return -1;
 
@@ -2776,6 +2786,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
     pReq->ast = taosMemoryCalloc(1, astLen + 1);
     if (pReq->ast == NULL) return -1;
     if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
+  } else {
+    if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
   }
 
   tEndDecode(&decoder);

source/common/src/ttime.c

@@ -412,10 +412,10 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
 int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
   int32_t charLen = varDataLen(inputData);
   char *newColData;
-  if (type == TSDB_DATA_TYPE_BINARY) {
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
     newColData = taosMemoryCalloc(1,  charLen + 1);
     memcpy(newColData, varDataVal(inputData), charLen);
-    bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
+    bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;
@@ -429,7 +429,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
+    bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/common/test/commonTests.cpp

@@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) {
   printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
 
   SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
-  SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0};
+  SBlockOrderInfo order = { true,  TSDB_ORDER_ASC, 0, NULL };
   taosArrayPush(pOrderInfo, &order);
 
   blockDataSort(b, pOrderInfo);

source/dnode/mgmt/exe/dmMain.c

@@ -40,10 +40,12 @@ static void dmSetSignalHandle() {
   taosSetSignal(SIGTERM, dmStopDnode);
   taosSetSignal(SIGHUP, dmStopDnode);
   taosSetSignal(SIGINT, dmStopDnode);
-  taosSetSignal(SIGTSTP, dmStopDnode);
   taosSetSignal(SIGABRT, dmStopDnode);
   taosSetSignal(SIGBREAK, dmStopDnode);
+#ifndef WINDOWS
+  taosSetSignal(SIGTSTP, dmStopDnode);
   taosSetSignal(SIGQUIT, dmStopDnode);
+#endif
 
   if (!tsMultiProcess) {
   } else if (global.ntype == DNODE || global.ntype == NODE_END) {

source/dnode/mgmt/implement/src/dmExec.c

@@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
 
 void dmCloseNode(SMgmtWrapper *pWrapper) {
   dInfo("node:%s, start to close", pWrapper->name);
+  pWrapper->deployed = false;
 
   while (pWrapper->refCount > 0) {
     taosMsleep(10);

source/dnode/mgmt/implement/src/dmHandle.c

@@ -216,6 +216,129 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) {
   dmStopStatusThread(pWrapper->pDnode);
 }
 
+static int32_t dmSpawnUdfd(SDnode *pDnode);
+
+void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
+  dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
+  SDnode *pDnode = process->data;
+  if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pDnode->udfdData.stopCalled)) {
+    dInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
+  } else {
+    dInfo("udfd process restart");
+    dmSpawnUdfd(pDnode);
+  }
+}
+
+static int32_t dmSpawnUdfd(SDnode *pDnode) {
+  dInfo("dnode start spawning udfd");
+  uv_process_options_t options = {0};
+
+  char path[PATH_MAX] = {0};
+  if (tsProcPath == NULL) {
+    path[0] = '.';
+  } else {
+    strncpy(path, tsProcPath, strlen(tsProcPath));
+    taosDirName(path);
+  }
+  strcat(path, "/udfd");
+  char* argsUdfd[] = {path, "-c", configDir, NULL};
+  options.args = argsUdfd;
+  options.file = path;
+
+  options.exit_cb = dmUdfdExit;
+
+  SUdfdData *pData = &pDnode->udfdData;
+  uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
+
+  uv_stdio_container_t child_stdio[3];
+  child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
+  child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
+  child_stdio[1].flags = UV_IGNORE;
+  child_stdio[2].flags = UV_INHERIT_FD;
+  child_stdio[2].data.fd = 2;
+  options.stdio_count = 3;
+  options.stdio = child_stdio;
+
+  options.flags = UV_PROCESS_DETACHED;
+
+  char dnodeIdEnvItem[32] = {0};
+  char thrdPoolSizeEnvItem[32] = {0};
+  snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
+  float numCpuCores = 4;
+  taosGetCpuCores(&numCpuCores);
+  snprintf(thrdPoolSizeEnvItem,32,  "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
+  char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
+  options.env = envUdfd;
+
+  int err = uv_spawn(&pData->loop, &pData->process, &options);
+  pData->process.data = (void*)pDnode;
+
+  if (err != 0) {
+    dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
+  }
+  return err;
+}
+
+static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
+  if (!uv_is_closing(handle)) {
+    uv_close(handle, NULL);
+  }
+}
+
+static void dmUdfdStopAsyncCb(uv_async_t *async) {
+  SDnode *pDnode = async->data;
+  SUdfdData *pData = &pDnode->udfdData;
+  uv_stop(&pData->loop);
+}
+
+static void dmWatchUdfd(void *args) {
+  SDnode *pDnode = args;
+  SUdfdData *pData = &pDnode->udfdData;
+  uv_loop_init(&pData->loop);
+  uv_async_init(&pData->loop, &pData->stopAsync, dmUdfdStopAsyncCb);
+  pData->stopAsync.data = pDnode;
+  int32_t err = dmSpawnUdfd(pDnode);
+  atomic_store_32(&pData->spawnErr, err);
+  uv_barrier_wait(&pData->barrier);
+  uv_run(&pData->loop, UV_RUN_DEFAULT);
+  uv_loop_close(&pData->loop);
+
+  uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
+  uv_run(&pData->loop, UV_RUN_DEFAULT);
+  uv_loop_close(&pData->loop);
+  return;
+}
+
+static int32_t dmStartUdfd(SDnode *pDnode) {
+  SUdfdData *pData = &pDnode->udfdData;
+  if (pData->startCalled) {
+    dInfo("dnode-mgmt start udfd already called");
+    return 0;
+  }
+  pData->startCalled = true;
+  uv_barrier_init(&pData->barrier, 2);
+  uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
+  uv_barrier_wait(&pData->barrier);
+  pData->needCleanUp = true;
+  return pData->spawnErr;
+}
+
+static int32_t dmStopUdfd(SDnode *pDnode) {
+  dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
+        pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
+  SUdfdData *pData = &pDnode->udfdData;
+  if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
+    return 0;
+  }
+  atomic_store_32(&pData->stopCalled, 1);
+  pData->needCleanUp = false;
+  uv_barrier_destroy(&pData->barrier);
+  uv_async_send(&pData->stopAsync);
+  uv_thread_join(&pData->thread);
+
+  return 0;
+}
+
 static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to init");
   SDnode *pDnode = pWrapper->pDnode;
@@ -247,6 +370,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "dnode-transport", "initialized");
 
+//  if (dmStartUdfd(pDnode) != 0) {
+//    dError("failed to start udfd");
+//  }
+
   dInfo("dnode-mgmt is initialized");
   return 0;
 }
@@ -254,6 +381,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
 static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to clean up");
   SDnode *pDnode = pWrapper->pDnode;
+  dmStopUdfd(pDnode);
   dmStopWorker(pDnode);
 
   taosWLockLatch(&pDnode->data.latch);

source/dnode/mgmt/implement/src/dmTransport.c

@@ -59,6 +59,10 @@ static inline int32_t dmBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
   pMsg->clientIp = connInfo.clientIp;
   pMsg->clientPort = connInfo.clientPort;
   memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
+  if ((pRpc->msgType & 1u)) {
+    assert(pRpc->refId != 0);
+  }
+  // assert(pRpc->handle != NULL && pRpc->refId != 0 && pMsg->rpcMsg.refId != 0);
   return 0;
 }
 
@@ -107,7 +111,7 @@ _OVER:
         }
       }
 
-      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId};
       tmsgSendRsp(&rsp);
     }
     dTrace("msg:%p, is freed", pMsg);
@@ -173,6 +177,9 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
   }
 
   dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle);
+  if (isReq) {
+    assert(pMsg->refId != 0);
+  }
   dmProcessRpcMsg(pWrapper, pMsg, pEpSet);
 }
 

source/dnode/mgmt/interface/inc/dmDef.h

@@ -16,6 +16,7 @@
 #ifndef _TD_DM_DEF_H_
 #define _TD_DM_DEF_H_
 
+#include "uv.h"
 #include "dmLog.h"
 
 #include "cJSON.h"
@@ -142,6 +143,19 @@ typedef struct {
   char desc[TSDB_STEP_DESC_LEN];
 } SStartupInfo;
 
+typedef struct SUdfdData {
+  bool          startCalled;
+  bool          needCleanUp;
+  uv_loop_t     loop;
+  uv_thread_t   thread;
+  uv_barrier_t  barrier;
+  uv_process_t  process;
+  int           spawnErr;
+  uv_pipe_t     ctrlPipe;
+  uv_async_t    stopAsync;
+  int32_t        stopCalled;
+} SUdfdData;
+
 typedef struct SDnode {
   EDndProcType  ptype;
   EDndNodeType  ntype;
@@ -150,6 +164,7 @@ typedef struct SDnode {
   SStartupInfo  startup;
   SDnodeTrans   trans;
   SDnodeData    data;
+  SUdfdData     udfdData;
   TdThreadMutex mutex;
   SMgmtWrapper  wrappers[NODE_END];
 } SDnode;

source/dnode/mgmt/interface/src/dmEnv.c

@@ -53,6 +53,7 @@ void dmCleanup() {
   }
 
   monCleanup();
+  syncCleanUp();
   walCleanUp();
   taosStopCacheRefreshWorker();
   dInfo("dnode env is cleaned up");

source/dnode/mgmt/mgmt_bnode/src/bmWorker.c

@@ -17,7 +17,8 @@
 #include "bmInt.h"
 
 static void bmSendErrorRsp(SNodeMsg *pMsg, int32_t code) {
-  SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
+  SRpcMsg rpcRsp = {
+      .handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code, .refId = pMsg->rpcMsg.refId};
   tmsgSendRsp(&rpcRsp);
 
   dTrace("msg:%p, is freed", pMsg);

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -218,9 +218,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
 
   // Requests handled by VNODE
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -105,6 +105,7 @@ void    vmStopWorker(SVnodesMgmt *pMgmt);
 int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 void    vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);  // sync integration
 int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -121,6 +121,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
   pCfg->hashMethod = pCreate->hashMethod;
+
+  // sync integration
+  pCfg->syncCfg.myIndex = pCreate->selfIndex;
+  pCfg->syncCfg.replicaNum = pCreate->replica;
+  memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo));
+  for (int i = 0; i < pCreate->replica; ++i) {
+    (pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port;
+    snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s",
+             (pCreate->replicas)[i].fqdn);
+  }
 }
 
 static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
@@ -172,6 +182,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+  msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
   msgCb.qsizeFp = vmGetQueueSize;
 
   SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -252,7 +263,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -265,11 +276,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);  
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -283,4 +290,17 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
 
   dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+
+  // sync integration
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -73,6 +73,12 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
     return -1;
   }
 
+  // sync integration
+  vnodeSyncSetQ(pImpl, NULL);
+  vnodeSyncSetRpc(pImpl, NULL);
+  int32_t ret = vnodeSyncStart(pImpl);
+  assert(ret == 0);
+
   taosWLockLatch(&pMgmt->latch);
   int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
   taosWUnLockLatch(&pMgmt->latch);
@@ -137,6 +143,7 @@ static void *vmOpenVnodeFunc(void *param) {
     msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+    msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
     msgCb.qsizeFp = vmGetQueueSize;
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
     SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -266,6 +273,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
   // walCleanUp();
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
+
+  // syncCleanUp();
+
   dInfo("vnode-mgmt is cleaned up");
 }
 
@@ -306,6 +316,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "vnode-wal", "initialized");
 
+  // sync integration
+  if (syncInit() != 0) {
+    dError("failed to open sync since %s", terrstr());
+    return -1;
+  }
+
   if (vnodeInit(tsNumOfCommitThreads) != 0) {
     dError("failed to init vnode since %s", terrstr());
     goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -14,8 +14,12 @@
  */
 
 #define _DEFAULT_SOURCE
+
 #include "vmInt.h"
 
+#include "sync.h"
+#include "syncTools.h"
+
 static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
                  .ahandle = pMsg->rpcMsg.ahandle,
@@ -89,7 +93,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 
 static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    version;
+  SRpcMsg    rsp;
 
   SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
   if (pArray == NULL) {
@@ -108,13 +112,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     }
   }
 
+#if 0
+  int64_t version;
+
   vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version);
 
   numOfMsgs = taosArrayGetSize(pArray);
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
-    SRpcMsg * pRpc = &pMsg->rpcMsg;
-    SRpcMsg   rsp;
+    SRpcMsg  *pRpc = &pMsg->rpcMsg;
 
     rsp.pCont = NULL;
     rsp.contLen = 0;
@@ -125,17 +131,37 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
     tmsgSendRsp(&rsp);
+  }
+#else
+  // sync integration response
+  for (int i = 0; i < taosArrayGetSize(pArray); i++) {
+    SNodeMsg *pMsg;
+    SRpcMsg * pRpc;
 
-#if 0
-    if (pRsp != NULL) {
-      pRsp->ahandle = pRpc->ahandle;
-      taosMemoryFree(pRsp);
+    pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    pRpc = &pMsg->rpcMsg;
+
+    rsp.ahandle = pRpc->ahandle;
+    rsp.handle = pRpc->handle;
+    rsp.refId = pRpc->refId;
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
+    if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      rsp.code = TSDB_CODE_SYN_NOT_LEADER;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) {
+      rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) {
+      // ok
+      // send response in applyQ
     } else {
-      if (code != 0 && terrno != 0) code = terrno;
-      vmSendRsp(pVnode->pWrapper, pMsg, code);
+      assert(0);
     }
-#endif
   }
+#endif
 
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
@@ -150,13 +176,43 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SNodeMsg * pMsg = NULL;
+  SRpcMsg    rsp;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
+#if 1
+    // sync integration
+
     taosGetQitem(qall, (void **)&pMsg);
 
-    // todo
-    SRpcMsg *pRsp = NULL;
-    // (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    // init response rpc msg
+    rsp.code = 0;
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    // get original rpc msg
+    assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg);
+    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
+    SRpcMsg originalRpcMsg;
+    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
+
+    // apply data into tsdb
+    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
+      rsp.code = terrno;
+      dTrace("vnodeProcessWriteReq error, code:%d", terrno);
+    }
+
+    syncApplyMsgDestroy(pSyncApplyMsg);
+    rpcFreeCont(originalRpcMsg.pCont);
+
+    // if leader, send response
+    if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+      rsp.ahandle = pMsg->rpcMsg.ahandle;
+      rsp.handle = pMsg->rpcMsg.handle;
+      rsp.refId = pMsg->rpcMsg.refId;
+      tmsgSendRsp(&rsp);
+    }
+#endif
   }
 }
 
@@ -309,6 +365,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
         dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
         taosWriteQitem(pVnode->pMergeQ, pMsg);
         break;
+      case SYNC_QUEUE:  // sync integration
+        dTrace("msg:%p, will be put into vnode-sync queue", pMsg);
+        taosWriteQitem(pVnode->pSyncQ, pMsg);
+        break;
       default:
         code = -1;
         terrno = TSDB_CODE_INVALID_PARA;
@@ -335,6 +395,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
   return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE);
 }
 
+// sync integration
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE);
+}
+
 int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
   int32_t    size = -1;
   SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) {
   }
 
   SRetrieveTableReq retrieveReq = {0};
-  retrieveReq.type = showType;
   strcpy(retrieveReq.db, db);
   strcpy(retrieveReq.tb, tb);
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -392,7 +392,6 @@ typedef struct {
   int16_t        numOfColumns;
   int32_t        rowSize;
   int32_t        numOfRows;
-  int32_t        payloadLen;
   void*          pIter;
   SMnode*        pMnode;
   STableMetaRsp* pMeta;

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -30,6 +30,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
 int32_t  mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
 SEpSet   mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
 int32_t  mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
+int32_t  mndGetGlobalVgroupVersion(int32_t *vgId);
 
 void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

source/dnode/mnode/impl/src/mndDb.c

@@ -1191,6 +1191,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
   char *p = strchr(usedbReq.db, '.');
   if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
     memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
+    //mndGetGlobalVgroupVersion(); TODO
     static int32_t vgVersion = 1;
     if (usedbReq.vgVersion < vgVersion) {
       usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
@@ -1202,16 +1203,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
       mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
       usedbRsp.vgVersion = vgVersion++;
 
-      if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) {
-        terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      } else {
-        code = 0;
-      }
     } else {
       usedbRsp.vgVersion = usedbReq.vgVersion;
-      code = 0;
     }
     usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
+    code = 0;
 
     // no jump, need to construct rsp
   } else {

source/dnode/mnode/impl/src/mndDnode.c

@@ -52,9 +52,9 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq);
 static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp);
 static int32_t mndProcessStatusReq(SNodeMsg *pReq);
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextDnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitDnode(SMnode *pMnode) {
@@ -335,10 +335,13 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
       }
       bool roleChanged = false;
       for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
-        if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
-          roleChanged = true;
+        // sync integration
+        if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
+          if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
+            roleChanged = true;
+          }
+          pVgroup->vnodeGid[vg].role = pVload->syncState;
         }
-        pVgroup->vnodeGid[vg].role = pVload->syncState;
       }
       if (roleChanged) {
         // notify scheduler role has changed
@@ -634,7 +637,7 @@ static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode *pMnode = pReq->pNode;
   int32_t totalRows = 0;
   int32_t numOfRows = 0;
@@ -667,12 +670,12 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
     STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN);
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) buf, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
 
     STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) bufVal, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false);
 
     numOfRows++;
   }
@@ -683,7 +686,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
 static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {}
 
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -698,8 +701,8 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     cols = 0;
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false);
 
     char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]);
@@ -709,19 +712,19 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int16_t id = mndGetVnodesNum(pMnode, pDnode->id);
-    colDataAppend(pColInfo, numOfRows, (const char*) &id, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&id, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
 
     char b1[9] = {0};
-    STR_TO_VARSTR(b1, online? "ready":"offline");
+    STR_TO_VARSTR(b1, online ? "ready" : "offline");
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, b1, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
 
     char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -199,15 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = {
     {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
 };
 
-// TODO put into perf schema
-static const SInfosTableSchema topicSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "sql", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-};
-
 static const SInfosTableSchema consumerSchema[] = {
     {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
@@ -291,7 +282,6 @@ static const SInfosTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
     {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
     {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
-    {TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
     {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
     {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
     {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -41,14 +41,15 @@ static const SPerfsTableSchema queriesSchema[] = {
 
 static const SPerfsTableSchema topicSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    /*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    /*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/
 };
 
 static const SPerfsTableSchema consumerSchema[] = {
     {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -61,6 +62,7 @@ static const SPerfsTableSchema subscribeSchema[] = {
     {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
 };
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -58,7 +58,8 @@ static void      mndCancelGetNextQuery(SMnode *pMnode, void *pIter);
 int32_t mndInitProfile(SMnode *pMnode) {
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
-  int32_t connCheckTime = tsShellActivityTimer * 2;
+  // in ms
+  int32_t connCheckTime = tsShellActivityTimer * 2 * 1000;
   pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn");
   if (pMgmt->cache == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -174,10 +175,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
 }
 
 static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
-  SMnode *    pMnode = pReq->pNode;
-  SUserObj *  pUser = NULL;
-  SDbObj *    pDb = NULL;
-  SConnObj *  pConn = NULL;
+  SMnode     *pMnode = pReq->pNode;
+  SUserObj   *pUser = NULL;
+  SDbObj     *pDb = NULL;
+  SConnObj   *pConn = NULL;
   int32_t     code = -1;
   SConnectReq connReq = {0};
   char        ip[30] = {0};
@@ -464,7 +465,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
   taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
 
   int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp);
-  void *  buf = rpcMallocCont(tlen);
+  void   *buf = rpcMallocCont(tlen);
   tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp);
 
   int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps);
@@ -486,7 +487,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
-  SMnode *      pMnode = pReq->pNode;
+  SMnode       *pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -520,7 +521,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
-  SMnode *      pMnode = pReq->pNode;
+  SMnode       *pMnode = pReq->pNode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@@ -552,11 +553,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
 }
 
 static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode *  pMnode = pReq->pNode;
+  SMnode   *pMnode = pReq->pNode;
   int32_t   numOfRows = 0;
   SConnObj *pConn = NULL;
   int32_t   cols = 0;
-  char *    pWrite;
+  char     *pWrite;
   char      ipStr[TSDB_IPv4ADDR_LEN + 6];
 
   if (pShow->pIter == NULL) {
@@ -611,8 +612,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int
 }
 
 static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode *pMnode = pReq->pNode;
-  int32_t numOfRows = 0;
+  SMnode   *pMnode = pReq->pNode;
+  int32_t   numOfRows = 0;
 #if 0
   SConnObj *pConn = NULL;
   int32_t   cols = 0;

source/dnode/mnode/impl/src/mndQnode.c

@@ -444,8 +444,8 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet));
-  if (NULL == qlistRsp.epSetList) {
+  qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr));
+  if (NULL == qlistRsp.addrsList) {
     mError("failed to alloc epSet while process qnode list req");
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _OVER;
@@ -455,11 +455,13 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
     void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
     if (pIter == NULL) break;
 
-    SEpSet epSet = {.numOfEps = 1};
-    tstrncpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
-    epSet.eps[0].port = pObj->pDnode->port;
+    SQueryNodeAddr nodeAddr = {0};
+    nodeAddr.nodeId = QNODE_HANDLE;
+    nodeAddr.epSet.numOfEps = 1;
+    tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
+    nodeAddr.epSet.eps[0].port = pObj->pDnode->port;
 
-    (void)taosArrayPush(qlistRsp.epSetList, &epSet);
+    (void)taosArrayPush(qlistRsp.addrsList, &nodeAddr);
 
     numOfRows++;
     sdbRelease(pSdb, pObj);

source/dnode/mnode/impl/src/mndShow.c

@@ -18,7 +18,7 @@
 
 #define SHOW_STEP_SIZE 100
 
-static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq);
+static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq);
 static void      mndFreeShowObj(SShowObj *pShow);
 static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId);
 static void      mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
@@ -28,7 +28,7 @@ static int32_t   mndProcessRetrieveSysTableReq(SNodeMsg *pReq);
 int32_t mndInitShow(SMnode *pMnode) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
 
-  pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show");
+  pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
   if (pMgmt->cache == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("failed to alloc show cache since %s", terrstr());
@@ -47,18 +47,78 @@ void mndCleanupShow(SMnode *pMnode) {
   }
 }
 
-static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) {
+static int32_t convertToRetrieveType(char* name, int32_t len) {
+  int32_t type = -1;
+
+  if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_DNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_MNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) {
+    type = TSDB_MGMT_TABLE_MODULE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_QNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_BNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_SNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {
+    type = TSDB_MGMT_TABLE_CLUSTER;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) {
+    type = TSDB_MGMT_TABLE_DB;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
+    type = TSDB_MGMT_TABLE_FUNC;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
+    //    type = TSDB_MGMT_TABLE_INDEX;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
+    type = TSDB_MGMT_TABLE_STB;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) {
+    type = TSDB_MGMT_TABLE_STREAMS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
+    type = TSDB_MGMT_TABLE_TABLE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) {
+    //    type = TSDB_MGMT_TABLE_DIST;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) {
+    type = TSDB_MGMT_TABLE_USER;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) {
+    type = TSDB_MGMT_TABLE_GRANTS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
+    type = TSDB_MGMT_TABLE_VGROUP;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONSUMERS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {
+    type = TSDB_MGMT_TABLE_SUBSCRIBES;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) {
+    type = TSDB_MGMT_TABLE_TRANS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) {
+    type = TSDB_MGMT_TABLE_SMAS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONFIGS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONNS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) {
+    type = TSDB_MGMT_TABLE_QUERIES;
+  }  else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_VNODES;
+  } else {
+//    ASSERT(0);
+  }
+
+  return type;
+}
+
+static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
 
   int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1);
   if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1);
 
-  int32_t  size = sizeof(SShowObj) + pReq->payloadLen;
+  int32_t  size = sizeof(SShowObj);
+
   SShowObj showObj = {0};
-  showObj.id = showId;
+  showObj.id     = showId;
   showObj.pMnode = pMnode;
-  showObj.type = pReq->type;
-  showObj.payloadLen = pReq->payloadLen;
+  showObj.type   = convertToRetrieveType(pReq->tb, tListLen(pReq->tb));
   memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN);
 
   int32_t   keepTime = tsShellActivityTimer * 6 * 1000;
@@ -127,10 +187,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
   }
 
   if (retrieveReq.showId == 0) {
-    SShowReq req = {0};
-    req.type = retrieveReq.type;
-    strncpy(req.db, retrieveReq.db, tListLen(req.db));
-
     STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1);
     if (pMeta == NULL) {
       terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
@@ -138,7 +194,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
       return -1;
     }
 
-    pShow = mndCreateShowObj(pMnode, &req);
+    pShow = mndCreateShowObj(pMnode, &retrieveReq);
     if (pShow == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       mError("failed to process show-meta req since %s", terrstr());

source/dnode/mnode/impl/src/mndStream.c

@@ -59,7 +59,7 @@ int32_t mndInitStream(SMnode *pMnode) {
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
 
   //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream);
-  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);
+  /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/
 
   return sdbSetTable(pMnode->pSdb, table);
 }
@@ -247,7 +247,8 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64
   return code;
 }
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans) {
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark,
+                            STrans *pTrans) {
   SNode *pAst = NULL;
 
   if (nodesStringToNode(ast, &pAst) < 0) {

source/dnode/mnode/impl/src/mndTopic.c

@@ -35,7 +35,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj
 static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq);
 static int32_t mndProcessDropTopicReq(SNodeMsg *pReq);
 static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp);
-static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
+static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
 
 int32_t mndInitTopic(SMnode *pMnode) {
@@ -51,7 +51,7 @@ int32_t mndInitTopic(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq);
   mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp);
 
-  //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic);
 
   return sdbSetTable(pMnode->pSdb, table);
@@ -511,56 +511,40 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo
   return 0;
 }
 
-static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
   SMnode      *pMnode = pReq->pNode;
   SSdb        *pSdb = pMnode->pSdb;
   int32_t      numOfRows = 0;
   SMqTopicObj *pTopic = NULL;
-  int32_t      cols = 0;
-  char        *pWrite;
-  char         prefix[TSDB_DB_FNAME_LEN] = {0};
 
-  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
-  if (pDb == NULL) return 0;
-
-  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
-  strcat(prefix, TS_PATH_DELIMITER);
-  int32_t prefixLen = (int32_t)strlen(prefix);
-
-  while (numOfRows < rows) {
+  while (numOfRows < rowsCapacity) {
     pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
     if (pShow->pIter == NULL) break;
 
-    if (pTopic->dbUid != pDb->uid) {
-      if (strncmp(pTopic->name, prefix, prefixLen) != 0) {
-        mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid);
-      }
+    int32_t cols = 0;
 
-      sdbRelease(pSdb, pTopic);
-      continue;
-    }
+    char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN);
+    varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE]));
 
-    cols = 0;
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
 
-    char topicName[TSDB_TOPIC_NAME_LEN] = {0};
-    tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN);
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, topicName);
-    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int64_t *)pWrite = pTopic->createTime;
-    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
+    varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
+    colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pTopic->sql, pShow->bytes[cols]);
-    cols++;
+    taosMemoryFree(sql);
 
     numOfRows++;
     sdbRelease(pSdb, pTopic);
   }
 
-  mndReleaseDb(pMnode, pDb);
   pShow->numOfRows += numOfRows;
   return numOfRows;
 }

source/dnode/mnode/impl/test/db/db.cpp

@@ -89,6 +89,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
     void*   pReq = rpcMallocCont(contLen);
     tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
 
+    taosMsleep(1000); // Wait for the vnode to become the leader
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);

source/dnode/mnode/impl/test/show/show.cpp

@@ -36,7 +36,7 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
+  ASSERT_NE(pRsp->code, 0);
 }
 
 TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
@@ -50,7 +50,7 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
+  ASSERT_NE(pRsp->code, 0);
 }
 
 TEST_F(MndTestShow, 03_ShowMsg_Conn) {

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -258,6 +258,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateDbReq(dbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(1000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -304,6 +304,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -438,6 +439,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -497,6 +499,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -538,6 +541,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -602,6 +606,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -655,6 +660,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -715,6 +721,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {
@@ -775,6 +782,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    taosMsleep(2000); // Wait for the vnode to become the leader
   }
 
   {

source/dnode/vnode/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(
     "src/vnd/vnodeStateMgr.c"
     "src/vnd/vnodeModule.c"
     "src/vnd/vnodeSvr.c"
+    "src/vnd/vnodeSync.c"
 
     # meta
     "src/meta/metaOpen.c"

source/dnode/vnode/inc/vnode.h

@@ -21,6 +21,7 @@
 #include "tqueue.h"
 #include "trpc.h"
 
+#include "sync.h"
 #include "tarray.h"
 #include "tfs.h"
 #include "wal.h"
@@ -60,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int     vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
 
+int64_t vnodeGetSyncHandle(SVnode *pVnode);
+void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
+
 // meta
 typedef struct SMeta      SMeta;  // todo: remove
 typedef struct SMTbCursor SMTbCursor;
@@ -109,7 +113,8 @@ int     tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
 int     tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList);
 int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReadHandle *pHandle);
-int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows);
+int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows,
+                            int16_t *pNumOfCols);
 
 // need to reposition
 
@@ -146,6 +151,7 @@ struct SVnodeCfg {
   bool     isWeak;
   STsdbCfg tsdbCfg;
   SWalCfg  walCfg;
+  SSyncCfg syncCfg;  // sync integration
   uint32_t hashBegin;
   uint32_t hashEnd;
   int8_t   hashMethod;
@@ -165,6 +171,11 @@ typedef struct {
   uint64_t uid;
 } STableKeyInfo;
 
+// sync integration
+void    vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void    vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+int32_t vnodeSyncStart(SVnode *pVnode);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tq.h

@@ -18,8 +18,10 @@
 
 #include "executor.h"
 #include "os.h"
+#include "tcache.h"
 #include "thash.h"
 #include "tmsg.h"
+#include "trpc.h"
 #include "ttimer.h"
 #include "wal.h"
 
@@ -81,29 +83,18 @@ typedef struct STqOffsetStore STqOffsetStore;
 
 struct STqReadHandle {
   int64_t           ver;
-  int64_t           tbUid;
   SHashObj*         tbIdHash;
   const SSubmitReq* pMsg;
   SSubmitBlk*       pBlock;
   SSubmitMsgIter    msgIter;
   SSubmitBlkIter    blkIter;
   SMeta*            pVnodeMeta;
-  SArray*           pColIdList;  // SArray<int32_t>
+  SArray*           pColIdList;  // SArray<int16_t>
   int32_t           sver;
   SSchemaWrapper*   pSchemaWrapper;
   STSchema*         pSchema;
 };
 
-typedef struct {
-  int8_t type;
-  int8_t reserved[7];
-  union {
-    void*   data;
-    int64_t wmTs;
-    int64_t checkpointId;
-  };
-} STqStreamToken;
-
 typedef struct {
   int16_t ver;
   int16_t action;
@@ -152,27 +143,40 @@ typedef struct {
 } STqMetaStore;
 
 typedef struct {
-  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
-  int64_t consumerId;
-  int32_t epoch;
-  char*   qmsg;
+  int64_t  consumerId;
+  int32_t  epoch;
+  int32_t  skipLogNum;
+  int64_t  reqOffset;
+  SRWLatch lock;
+  SRpcMsg* handle;
+} STqPushHandle;
+
+typedef struct {
+  char          subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int64_t       consumerId;
+  int32_t       epoch;
+  int8_t        subType;
+  int8_t        withTbName;
+  int8_t        withSchema;
+  int8_t        withTag;
+  int8_t        withTagSchema;
+  char*         qmsg;
+  STqPushHandle pushHandle;
   // SRWLatch        lock;
-  SWalReadHandle* pReadHandle;
-  // number should be identical to fetch thread num
-  qTaskInfo_t task[4];
+  SWalReadHandle* pWalReader;
+  // task number should be the same with fetch thread
+  STqReadHandle* pExecReader[5];
+  qTaskInfo_t    task[5];
 } STqExec;
 
 struct STQ {
-  // the collection of groups
-  // the handle of meta kvstore
-  bool          writeTrigger;
-  char*         path;
-  STqMetaStore* tqMeta;
-  SHashObj*     tqMetaNew;  // subKey -> tqExec
-  SHashObj*     pStreamTasks;
-  SVnode*       pVnode;
-  SWal*         pWal;
-  SMeta*        pVnodeMeta;
+  char* path;
+  // STqMetaStore* tqMeta;
+  SHashObj* pushMgr;  // consumerId -> STqExec*
+  SHashObj* execs;    // subKey -> STqExec
+  SHashObj* pStreamTasks;
+  SVnode*   pVnode;
+  SWal*     pWal;
 };
 
 typedef struct {
@@ -213,20 +217,6 @@ typedef struct {
   SArray* topics;  // SArray<STqTopic>
 } STqConsumer;
 
-enum {
-  TQ_PUSHER_TYPE__CLIENT = 1,
-  TQ_PUSHER_TYPE__STREAM,
-};
-
-typedef struct {
-  int8_t   type;
-  int8_t   reserved[3];
-  int32_t  ttl;
-  int64_t  consumerId;
-  SRpcMsg* pMsg;
-  // SMqPollRsp* rsp;
-} STqClientPusher;
-
 typedef struct {
   int8_t      type;
   int8_t      nodeType;
@@ -236,10 +226,6 @@ typedef struct {
   // TODO sync function
 } STqStreamPusher;
 
-typedef struct {
-  SHashObj* pHash;  // <id, STqPush*>
-} STqPushMgr;
-
 typedef struct {
   int8_t inited;
   tmr_h  timer;
@@ -252,17 +238,14 @@ int  tqInit();
 void tqCleanUp();
 
 // open in each vnode
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, SMemAllocatorFactory* allocFac);
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
 void tqClose(STQ*);
 // required by vnode
-int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
+int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int tqCommit(STQ*);
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
-// int32_t tqProcessRebReq(STQ* pTq, char* msg);
-// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
 int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@@ -303,16 +286,6 @@ int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t
 int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
 int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
 
-// tqPush
-int32_t tqPushMgrInit();
-void    tqPushMgrCleanUp();
-
-STqPushMgr* tqPushMgrOpen();
-void        tqPushMgrClose(STqPushMgr* pushMgr);
-
-STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
-STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -68,13 +68,13 @@ typedef struct {
 } SStreamSinkInfo;
 
 typedef struct {
-  SVnode*   pVnode;
-  SHashObj* pHash;  // streamId -> SStreamSinkInfo
+  SVnode   *pVnode;
+  SHashObj *pHash;  // streamId -> SStreamSinkInfo
 } SSink;
 
 // SVState
 struct SVState {
-  int64_t processed;
+  // int64_t processed;
   int64_t committed;
   int64_t applied;
 };
@@ -85,25 +85,26 @@ struct SVnodeInfo {
 };
 
 struct SVnode {
-  char*      path;
+  char      *path;
   SVnodeCfg  config;
   SVState    state;
-  STfs*      pTfs;
+  STfs      *pTfs;
   SMsgCb     msgCb;
-  SVBufPool* pBufPool;
-  SMeta*     pMeta;
-  STsdb*     pTsdb;
-  SWal*      pWal;
-  STQ*       pTq;
-  SSink*     pSink;
+  SVBufPool *pBufPool;
+  SMeta     *pMeta;
+  STsdb     *pTsdb;
+  SWal      *pWal;
+  STQ       *pTq;
+  SSink     *pSink;
+  int64_t    sync;  // sync integration
   tsem_t     canCommit;
-  SQHandle*  pQuery;
+  SQHandle  *pQuery;
 };
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
 // sma
-void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
+void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
 
 #include "vnd.h"
 
@@ -113,6 +114,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 #include "tq.h"
 
+#include "vnodeSync.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeSync.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_SYNC_H_
+#define _TD_VNODE_SYNC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
+int32_t vnodeSyncStart(SVnode *pVnode);
+void    vnodeSyncClose(SVnode *pVnode);
+
+void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+
+int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
+int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
+
+void    vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
+
+SSyncFSM *syncVnodeMakeFsm();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SYNC_H_*/

source/dnode/vnode/src/meta/metaIdx.c

@@ -23,6 +23,9 @@ struct SMetaIdx {
   SIndex *pIdx;
 #endif
   /* data */
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 };
 
 int metaOpenIdx(SMeta *pMeta) {

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -45,10 +45,12 @@ struct SMetaDB {
 #endif
 };
 
-typedef struct __attribute__((__packed__)) {
+#pragma pack(push,1)
+typedef struct {
   tb_uid_t uid;
   int32_t  sver;
 } SSchemaDbKey;
+#pragma pack(pop)
 
 typedef struct {
   char    *name;

source/dnode/vnode/src/tq/tq.c

@@ -15,11 +15,14 @@
 
 #include "vnodeInt.h"
 
-int32_t tqInit() { return tqPushMgrInit(); }
+int32_t tqInit() {
+  //
+  return 0;
+}
 
-void tqCleanUp() { tqPushMgrCleanUp(); }
+void tqCleanUp() {}
 
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMemAllocatorFactory* allocFac) {
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
   STQ* pTq = taosMemoryMalloc(sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -28,18 +31,21 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe
   pTq->path = strdup(path);
   pTq->pVnode = pVnode;
   pTq->pWal = pWal;
-  pTq->pVnodeMeta = pVnodeMeta;
+#if 0
   pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
                             (FTqDelete)taosMemoryFree, 0);
   if (pTq->tqMeta == NULL) {
     taosMemoryFree(pTq);
     return NULL;
   }
+#endif
 
-  pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
+  pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
 
   pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
 
+  pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
+
   return pTq;
 }
 
@@ -51,8 +57,139 @@ void tqClose(STQ* pTq) {
   // TODO
 }
 
-int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) {
+int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
   if (msgType != TDMT_VND_SUBMIT) return 0;
+  void*       pIter = NULL;
+  STqExec*    pExec = NULL;
+  SSubmitReq* pReq = (SSubmitReq*)msg;
+  int32_t     workerId = 4;
+  int64_t     fetchOffset = ver;
+
+  while (1) {
+    pIter = taosHashIterate(pTq->pushMgr, pIter);
+    if (pIter == NULL) break;
+    pExec = *(STqExec**)pIter;
+
+    taosWLockLatch(&pExec->pushHandle.lock);
+
+    SRpcMsg* pMsg = atomic_load_ptr(&pExec->pushHandle.handle);
+    ASSERT(pMsg);
+
+    SMqDataBlkRsp rsp = {0};
+    rsp.reqOffset = pExec->pushHandle.reqOffset;
+    rsp.blockData = taosArrayInit(0, sizeof(int32_t));
+    rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
+
+    if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
+      qTaskInfo_t task = pExec->task[workerId];
+      ASSERT(task);
+      qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+      while (1) {
+        SSDataBlock* pDataBlock = NULL;
+        uint64_t     ts = 0;
+        if (qExecTask(task, &pDataBlock, &ts) < 0) {
+          ASSERT(0);
+        }
+        if (pDataBlock == NULL) break;
+
+        ASSERT(pDataBlock->info.rows != 0);
+        ASSERT(pDataBlock->info.numOfCols != 0);
+
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        pRetrieve->useconds = ts;
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
+      }
+    } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
+      STqReadHandle* pReader = pExec->pExecReader[workerId];
+      tqReadHandleSetMsg(pReader, pReq, 0);
+      while (tqNextDataBlock(pReader)) {
+        SSDataBlock block = {0};
+        if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+                                &block.info.numOfCols) < 0) {
+          ASSERT(0);
+        }
+        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);
+        void*              buf = taosMemoryCalloc(1, dataStrLen);
+        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+        /*pRetrieve->useconds = 0;*/
+        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+        pRetrieve->compressed = 0;
+        pRetrieve->completed = 1;
+        pRetrieve->numOfRows = htonl(block.info.rows);
+
+        // TODO enable compress
+        int32_t actualLen = 0;
+        blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false);
+        actualLen += sizeof(SRetrieveTableRsp);
+        ASSERT(actualLen <= dataStrLen);
+        taosArrayPush(rsp.blockDataLen, &actualLen);
+        taosArrayPush(rsp.blockData, &buf);
+        rsp.blockNum++;
+      }
+    } else {
+      ASSERT(0);
+    }
+
+    if (rsp.blockNum == 0) {
+      taosWUnLockLatch(&pExec->pushHandle.lock);
+      continue;
+    }
+
+    ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
+    ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
+
+    rsp.rspOffset = fetchOffset;
+
+    int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
+    void*   buf = rpcMallocCont(tlen);
+    if (buf == NULL) {
+      pMsg->code = -1;
+      return -1;
+    }
+
+    ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
+    ((SMqRspHead*)buf)->epoch = pExec->pushHandle.epoch;
+    ((SMqRspHead*)buf)->consumerId = pExec->pushHandle.consumerId;
+
+    void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
+    tEncodeSMqDataBlkRsp(&abuf, &rsp);
+    pMsg->pCont = buf;
+    pMsg->contLen = tlen;
+    pMsg->code = 0;
+    tmsgSendRsp(pMsg);
+
+    atomic_store_ptr(&pExec->pushHandle.handle, NULL);
+    taosWUnLockLatch(&pExec->pushHandle.lock);
+
+    vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+           TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
+           rsp.reqOffset, rsp.rspOffset);
+
+    // TODO destroy
+    taosArrayDestroy(rsp.blockData);
+    taosArrayDestroy(rsp.blockDataLen);
+  }
+
+  return 0;
+}
+
+int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
+  if (msgType != TDMT_VND_SUBMIT) return 0;
+
   void* data = taosMemoryMalloc(msgLen);
   if (data == NULL) {
     return -1;
@@ -60,7 +197,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
   memcpy(data, msg, msgLen);
 
   if (msgType == TDMT_VND_SUBMIT) {
-    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, version) != 0) {
+    if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
       return -1;
     }
   }
@@ -70,6 +207,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
       .pCont = data,
       .contLen = msgLen,
   };
+
   tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
 
 #if 0
@@ -104,7 +242,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
   return 0;
 }
 
-int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); }
+int tqCommit(STQ* pTq) {
+  // do nothing
+  /*return tqStorePersist(pTq->tqMeta);*/
+  return 0;
+}
 
 int32_t tqGetTopicHandleSize(const STqTopic* pTopic) {
   return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) +
@@ -219,10 +361,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
     }
     for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
       pTopic->buffer.output[j].status = 0;
-      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
+      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
       SReadHandle    handle = {
              .reader = pReadHandle,
-             .meta = pTq->pVnodeMeta,
+             .meta = pTq->pVnode->pMeta,
       };
       pTopic->buffer.output[j].pReadHandle = pReadHandle;
       pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
@@ -235,9 +377,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReqV2* pReq = pMsg->pCont;
   int64_t       consumerId = pReq->consumerId;
+  int64_t       waitTime = pReq->blockingTime;
   int32_t       reqEpoch = pReq->epoch;
   int64_t       fetchOffset;
 
+  // get offset to fetch message
   if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
     fetchOffset = walGetFirstVer(pTq->pWal);
   } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
@@ -249,7 +393,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
-  STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey));
+  STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey));
   ASSERT(pExec);
 
   int32_t consumerEpoch = atomic_load_32(&pExec->epoch);
@@ -259,8 +403,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
-  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
   rsp.blockData = taosArrayInit(0, sizeof(void*));
+  rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
 
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
@@ -271,12 +415,34 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     }
 
     SWalReadHead* pHead;
-    if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) {
+    if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
+
+#if 0
+      // add to pushMgr
+      taosWLockLatch(&pExec->pushHandle.lock);
+
+      pExec->pushHandle.consumerId = consumerId;
+      pExec->pushHandle.epoch = reqEpoch;
+      pExec->pushHandle.reqOffset = rsp.reqOffset;
+      pExec->pushHandle.skipLogNum = rsp.skipLogNum;
+      pExec->pushHandle.handle = pMsg;
+
+      taosWUnLockLatch(&pExec->pushHandle.lock);
+
+      // TODO add timer
+
+      // TODO: the pointer will always be valid?
+      taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*));
+      taosArrayDestroy(rsp.blockData);
+      taosArrayDestroy(rsp.blockDataLen);
+      return 0;
+#endif
+
       break;
     }
 
@@ -285,41 +451,73 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
     if (pHead->msgType == TDMT_VND_SUBMIT) {
       SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
-      qTaskInfo_t task = pExec->task[workerId];
-      ASSERT(task);
-      qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
-      while (1) {
-        SSDataBlock* pDataBlock = NULL;
-        uint64_t     ts = 0;
-        if (qExecTask(task, &pDataBlock, &ts) < 0) {
-          ASSERT(0);
+      if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
+        qTaskInfo_t task = pExec->task[workerId];
+        ASSERT(task);
+        qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+        while (1) {
+          SSDataBlock* pDataBlock = NULL;
+          uint64_t     ts = 0;
+          if (qExecTask(task, &pDataBlock, &ts) < 0) {
+            ASSERT(0);
+          }
+          if (pDataBlock == NULL) break;
+
+          ASSERT(pDataBlock->info.rows != 0);
+          ASSERT(pDataBlock->info.numOfCols != 0);
+
+          int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
+          void*              buf = taosMemoryCalloc(1, dataStrLen);
+          SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+          pRetrieve->useconds = ts;
+          pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+          pRetrieve->compressed = 0;
+          pRetrieve->completed = 1;
+          pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
+
+          // TODO enable compress
+          int32_t actualLen = 0;
+          blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
+          actualLen += sizeof(SRetrieveTableRsp);
+          ASSERT(actualLen <= dataStrLen);
+          taosArrayPush(rsp.blockDataLen, &actualLen);
+          taosArrayPush(rsp.blockData, &buf);
+          rsp.blockNum++;
         }
-        if (pDataBlock == NULL) break;
-
-        ASSERT(pDataBlock->info.rows != 0);
-        ASSERT(pDataBlock->info.numOfCols != 0);
-
-        int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
-        void*              buf = taosMemoryCalloc(1, dataStrLen);
-        SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
-        pRetrieve->useconds = ts;
-        pRetrieve->precision = TSDB_DEFAULT_PRECISION;
-        pRetrieve->compressed = 0;
-        pRetrieve->completed = 1;
-        pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
-
-        // TODO enable compress
-        int32_t actualLen = 0;
-        blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
-        actualLen += sizeof(SRetrieveTableRsp);
-        ASSERT(actualLen <= dataStrLen);
-        taosArrayPush(rsp.blockDataLen, &actualLen);
-        taosArrayPush(rsp.blockData, &buf);
-        rsp.blockNum++;
+      } else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
+        STqReadHandle* pReader = pExec->pExecReader[workerId];
+        tqReadHandleSetMsg(pReader, pCont, 0);
+        while (tqNextDataBlock(pReader)) {
+          SSDataBlock block = {0};
+          if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
+                                  &block.info.numOfCols) < 0) {
+            ASSERT(0);
+          }
+          int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);
+          void*              buf = taosMemoryCalloc(1, dataStrLen);
+          SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
+          /*pRetrieve->useconds = 0;*/
+          pRetrieve->precision = TSDB_DEFAULT_PRECISION;
+          pRetrieve->compressed = 0;
+          pRetrieve->completed = 1;
+          pRetrieve->numOfRows = htonl(block.info.rows);
+
+          // TODO enable compress
+          int32_t actualLen = 0;
+          blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false);
+          actualLen += sizeof(SRetrieveTableRsp);
+          ASSERT(actualLen <= dataStrLen);
+          taosArrayPush(rsp.blockDataLen, &actualLen);
+          taosArrayPush(rsp.blockData, &buf);
+          rsp.blockNum++;
+        }
+      } else {
+        ASSERT(0);
       }
     }
 
-    // TODO batch optimization
+    // TODO batch optimization:
+    // TODO continue scan until meeting batch requirement
     if (rsp.blockNum != 0) break;
     rsp.skipLogNum++;
     fetchOffset++;
@@ -572,10 +770,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
 // TODO: persist meta into tdb
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
-  SMqRebVgReq req;
+  SMqRebVgReq req = {0};
   tDecodeSMqRebVgReq(msg, &req);
   // todo lock
-  STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey));
+  STqExec* pExec = taosHashGet(pTq->execs, req.subKey, strlen(req.subKey));
   if (pExec == NULL) {
     ASSERT(req.oldConsumerId == -1);
     ASSERT(req.newConsumerId != -1);
@@ -586,19 +784,27 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
     memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
     pExec->consumerId = req.newConsumerId;
     pExec->epoch = -1;
+
+    pExec->subType = req.subType;
+    pExec->withTbName = req.withTbName;
+    pExec->withSchema = req.withSchema;
+    pExec->withTag = req.withTag;
+    pExec->withTagSchema = req.withTagSchema;
+
     pExec->qmsg = req.qmsg;
     req.qmsg = NULL;
-    pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal);
-    for (int32_t i = 0; i < 4; i++) {
-      STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
-      SReadHandle    handle = {
-             .reader = pReadHandle,
-             .meta = pTq->pVnodeMeta,
+
+    pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
+    for (int32_t i = 0; i < 5; i++) {
+      pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
+      SReadHandle handle = {
+          .reader = pExec->pExecReader[i],
+          .meta = pTq->pVnode->pMeta,
       };
       pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
       ASSERT(pExec->task[i]);
     }
-    taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
+    taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
     return 0;
   } else {
     /*if (req.newConsumerId != -1) {*/
@@ -627,12 +833,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
     return -1;
   }
   for (int32_t i = 0; i < parallel; i++) {
-    STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
+    STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
     SReadHandle    handle = {
-           .reader = pReadHandle,
-           .meta = pTq->pVnodeMeta,
+           .reader = pStreamReader,
+           .meta = pTq->pVnode->pMeta,
     };
-    pTask->exec.runners[i].inputHandle = pReadHandle;
+    pTask->exec.runners[i].inputHandle = pStreamReader;
     pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
     ASSERT(pTask->exec.runners[i].executor);
   }

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   }
   strcpy(pMeta->dirPath, path);
 
-  char name[pathLen + 10];
+  char *name = taosMemoryMalloc(pathLen + 10) ;
 
   strcpy(name, path);
   if (!taosDirExist(name) && taosMkDir(name) != 0) {
@@ -99,6 +99,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s ", name, terrstr());
     // free memory
+    taosMemoryFree(name);
     return NULL;
   }
 
@@ -106,6 +107,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList));
   if (pMeta->unpersistHead == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
+    taosMemoryFree(name);
     return NULL;
   }
   pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
@@ -116,8 +118,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s", name, terrstr());
+    taosMemoryFree(name);
     return NULL;
   }
+  taosMemoryFree(name);
 
   pMeta->pFile = pFile;
 

source/dnode/vnode/src/tq/tqPush.c

@@ -12,73 +12,3 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
-#include "vnodeInt.h"
-
-int32_t tqPushMgrInit() {
-  //
-  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
-  if (old == 1) return 0;
-
-  tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
-  return 0;
-}
-
-void tqPushMgrCleanUp() {
-  int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
-  if (old == 0) return;
-  taosTmrStop(tqPushMgmt.timer);
-  taosTmrCleanUp(tqPushMgmt.timer);
-}
-
-STqPushMgr* tqPushMgrOpen() {
-  STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr));
-  if (mgr == NULL) {
-    return NULL;
-  }
-  mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-  return mgr;
-}
-
-void tqPushMgrClose(STqPushMgr* pushMgr) {
-  taosHashCleanup(pushMgr->pHash);
-  taosMemoryFree(pushMgr);
-}
-
-STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
-  STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher));
-  if (clientPusher == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
-  clientPusher->pMsg = pMsg;
-  clientPusher->consumerId = consumerId;
-  clientPusher->ttl = ttl;
-  if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(clientPusher);
-    // TODO send rsp back
-    return NULL;
-  }
-  return clientPusher;
-}
-
-STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
-  STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher));
-  if (streamPusher == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  streamPusher->type = TQ_PUSHER_TYPE__STREAM;
-  streamPusher->nodeType = 0;
-  streamPusher->streamId = streamId;
-  /*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/
-
-  if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(streamPusher);
-    return NULL;
-  }
-  return streamPusher;
-}