Merge remote-tracking branch 'origin/3.0' into feature/check

cmake/cmake.define

 cmake_minimum_required(VERSION 3.16)
 
+set(CMAKE_VERBOSE_MAKEFILE ON)
+
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
@@ -43,11 +45,14 @@ ENDIF ()
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
     SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
-    SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(COMMON_FLAGS "/W3 /D_WIN32")
 
-    IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
-      SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
-    ENDIF ()
+    # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
+    #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
+    # ENDIF ()
+
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
 
 ELSE ()
     IF (${SANITIZER} MATCHES "true")

cmake/cmake.options

@@ -25,9 +25,28 @@ IF(${TD_WINDOWS})
             ON
           )
 
-ENDIF ()
+    MESSAGE("build msvcregex Win32")
+    option(
+            BUILD_MSVCREGEX
+            "If build msvcregex on Windows"
+            ON
+          )
 
-IF(${TD_LINUX} MATCHES TRUE)
+    MESSAGE("build wcwidth Win32")
+    option(
+            BUILD_WCWIDTH
+            "If build wcwidth on Windows"
+            ON
+          )
+
+    MESSAGE("build wingetopt Win32")
+    option(
+            BUILD_WINGETOPT 
+                "If build wingetopt on Windows"
+            ON
+        )
+
+ENDIF ()
 
 option(
     BUILD_TEST 
@@ -35,8 +54,6 @@ option(
     ON
 )
 
-ENDIF ()
-
 option(
     BUILD_WITH_LEVELDB 
     "If build with leveldb" 

cmake/msvcregex_CMakeLists.txt.in

+
+# msvcregex
+ExternalProject_Add(msvcregex
+        GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/pthread_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(pthread
         GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
         GIT_TAG v3.0.3.1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/wcwidth_CMakeLists.txt.in

+
+# wcwidth
+ExternalProject_Add(wcwidth
+        GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/wingetopt_CMakeLists.txt.in

+
+# wingetopt
+ExternalProject_Add(wingetopt
+        GIT_REPOSITORY https://github.com/alex85k/wingetopt.git
+        GIT_TAG master
+        SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

contrib/CMakeLists.txt

@@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
     cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
-# gnu regex
-if(${BUILD_GNUREGEX})
-    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+# iconv
+if(${BUILD_WITH_ICONV})
+    cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# msvc regex
+if(${BUILD_MSVCREGEX})
+    cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wcwidth
+if(${BUILD_WCWIDTH})
+    cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
@@ -99,8 +114,27 @@ if(${BUILD_TEST})
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
+    )
+    if(${TD_WINDOWS})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
+        )
+    endif(${TD_WINDOWS})
+    if(${TD_LINUX})
+        target_include_directories(
+            gtest
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
         )
+    endif(${TD_LINUX})
+    if(${TD_DARWIN})
+        target_include_directories(
+            gtest
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
+        )
+    endif(${TD_DARWIN})
+    
+    
 endif(${BUILD_TEST})
 
 # cJson
@@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT})
     add_subdirectory(nuraft)
 endif(${BUILD_WITH_NURAFT})
 
+# pthread
+if(${BUILD_PTHREAD})
+    set(CMAKE_BUILD_TYPE release)
+    add_definitions(-DPTW32_STATIC_LIB) 
+    add_subdirectory(pthread)
+    set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
+    add_library(pthread STATIC IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+endif()
+
+# iconv
+if(${BUILD_WITH_ICONV})
+    add_subdirectory(iconv)
+endif(${BUILD_WITH_ICONV})
+
+# wingetopt
+if(${BUILD_WINGETOPT})
+    add_subdirectory(wingetopt)
+endif(${BUILD_WINGETOPT})
+
+# msvcregex
+if(${BUILD_MSVCREGEX})
+    add_library(msvcregex STATIC "")
+    target_sources(msvcregex
+        PRIVATE "msvcregex/regex.c"
+    )
+    target_include_directories(msvcregex
+        PRIVATE "msvcregex"
+    )
+    target_link_libraries(msvcregex 
+        INTERFACE Shell32
+    )
+    SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
+endif(${BUILD_MSVCREGEX})
+
+# msvcregex
+if(${BUILD_WCWIDTH})
+    add_library(wcwidth STATIC "")
+    target_sources(wcwidth
+        PRIVATE "wcwidth/wcwidth.c"
+    )
+    target_include_directories(wcwidth
+        PRIVATE "wcwidth"
+    )
+    SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
+endif(${BUILD_WCWIDTH})
+
 # CRAFT
 if(${BUILD_WITH_CRAFT})
     add_library(craft STATIC IMPORTED GLOBAL)
@@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
     target_link_libraries(sqlite 
         INTERFACE m 
         INTERFACE pthread 
+    )
+    if(NOT TD_WINDOWS)
+        target_link_libraries(sqlite 
             INTERFACE dl
         )
+    endif(NOT TD_WINDOWS)
 endif(${BUILD_WITH_SQLITE})
 
 # pthread

contrib/test/craft/CMakeLists.txt

 add_executable(simulate_vnode "simulate_vnode.c")
 target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
+if(${BUILD_WINGETOPT})
+    target_link_libraries(simulate_vnode PUBLIC wingetopt)
+    target_include_directories(
+        simulate_vnode
+        PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
+    )
+endif()
\ No newline at end of file

contrib/test/tdev/src/main.c

@@ -6,43 +6,39 @@
 #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
 #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
 
-#define tPutA(buf, val)              \
-  ({                                 \
-    memcpy(buf, &val, sizeof(val));  \
-    POINTER_SHIFT(buf, sizeof(val)); \
-  })
+static inline void tPutA(void **buf, uint64_t val) {
+  memcpy(buf, &val, sizeof(val));
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutB(buf, val)                         \
-  ({                                            \
-    ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
-    ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
-    ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
-    ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
-    ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
-    ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
-    ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;  \
-    ((uint8_t *)buf)[0] = (val)&0xff;           \
-    POINTER_SHIFT(buf, sizeof(val));            \
-  })
+static inline void tPutB(void **buf, uint64_t val) {
+  ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
+  ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
+  ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
+  ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
+  ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
+  ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
+  ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
+  ((uint8_t *)buf)[0] = (val)&0xff;
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
-#define tPutC(buf, val)                \
-  ({                                   \
-    if (buf) {                         \
-      ((uint64_t *)buf)[0] = (val);    \
-      POINTER_SHIFT(buf, sizeof(val)); \
-    }                                  \
-    NULL;                                 \
-  })
+static inline void tPutC(void **buf, uint64_t val) {
+  if (buf) {
+    ((uint64_t *)buf)[0] = (val);
+    POINTER_SHIFT(buf, sizeof(val));
+  }
+  *buf = NULL;
+}
 
-#define tPutD(buf, val)                        \
-  ({                                           \
-    uint64_t tmp = val;                        \
-    for (size_t i = 0; i < sizeof(val); i++) { \
-      ((uint8_t *)buf)[i] = tmp & 0xff;        \
-      tmp >>= 8;                               \
-    }                                          \
-    POINTER_SHIFT(buf, sizeof(val));           \
-  })
+static inline void tPutD(void **buf, uint64_t val) {
+  uint64_t tmp = val;
+  for (size_t i = 0; i < sizeof(val); i++) {
+    ((uint8_t *)buf)[i] = tmp & 0xff;
+    tmp >>= 8;
+  }
+  *buf = POINTER_SHIFT(buf, sizeof(val));
+}
 
 static inline void tPutE(void **buf, uint64_t val) {
   if (buf) {
@@ -61,7 +57,7 @@ static void func(T t) {
   switch (t) {
     case A:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutA(pBuf, val);
+        tPutA(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -69,7 +65,7 @@ static void func(T t) {
       break;
     case B:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutB(pBuf, val);
+        tPutB(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -77,7 +73,7 @@ static void func(T t) {
       break;
     case C:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutC(pBuf, val);
+        tPutC(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }
@@ -85,7 +81,7 @@ static void func(T t) {
       break;
     case D:
       for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
-        pBuf = tPutD(pBuf, val);
+        tPutD(pBuf, val);
         if (POINTER_DISTANCE(buf, pBuf) == 1024) {
           pBuf = buf;
         }

example/CMakeLists.txt

@@ -39,6 +39,7 @@ target_include_directories(tstream
 
 target_include_directories(demoapi
     PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PUBLIC "${TD_SOURCE_DIR}/include/os"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

example/src/demoapi.c

@@ -4,10 +4,12 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
+// #include <unistd.h>
 #include <inttypes.h>
+#ifndef WINDOWS
 #include <argp.h>
-
+#endif
+#include "osSleep.h"
 #include "taos.h"
 
 #define debugPrint(fmt, ...) \
@@ -32,6 +34,7 @@
 int64_t g_num_of_tb = 2;
 int64_t g_num_of_rec = 3;
 
+#ifndef WINDOWS
 static struct argp_option options[] = {
     {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
     {"records", 'n', "NUMBER", 0,
@@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
 }
 
 static struct argp argp = {options, parse_opt, "", ""};
-
+#endif
 static void prepare_data(TAOS* taos) {
     TAOS_RES *res;
     res = taos_query(taos, "drop database if exists test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
 
     res = taos_query(taos, "create database test;");
     taos_free_result(res);
-    usleep(100000);
+    taosMsleep(100);
     taos_select_db(taos, "test");
 
     char command[1024] = {0};
@@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) {
     taos_free_result(res);
 
     for (int64_t i = 0; i < g_num_of_tb; i ++) {
+        // sprintf(command, "create table t%"PRId64" using meters "
+        //         "tags(%"PRId64", '%s', '%s', '%s');",
+        //         i, i, (i%2)?"beijing":"shanghai",
+        //         (i%2)?"朝阳区":"黄浦区",
+        //         (i%2)?"长安街":"中山路");
         sprintf(command, "create table t%"PRId64" using meters "
-                "tags(%"PRId64", '%s', '%s', '%s');",
-                i, i, (i%2)?"beijing":"shanghai",
-                (i%2)?"朝阳区":"黄浦区",
-                (i%2)?"长安街":"中山路");
-/*        sprintf(command, "create table t%"PRId64" using meters "
                 "tags(%"PRId64", '%s', '%s', '%s');",
                 i, i,
                 (i%2)?"beijing":"shanghai",
                 (i%2)?"chaoyang":"huangpu",
                 (i%2?"changan street":"jianguo rd"));
-                */
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             okPrint("t%" PRId64 " created\n", i);
@@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) {
                     "'%c%d', '%s%c%d', '%c%d')",
                     i, 1650000000000+j, (float)j, j,
                     'a'+(int)j%25, rand(),
-                    "涛思", 'z' - (int)j%25, rand(),
+                    // "涛思", 'z' - (int)j%25, rand(),
+                    "TAOS", 'z' - (int)j%25, rand(),
                     'b' - (int)j%25, rand()
                     );
             res = taos_query(taos,  command);
@@ -176,7 +179,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
     }
 
     if (block) {
-        warnPrint("%s", "call taos_fetch_block()\n");
+        warnPrint("%s", "call taos_fetch_block(), don't call taos_fetch_lengths()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
             int *lengths = taos_fetch_lengths(res);
@@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %"PRId64"\n",
                                             f, c,
-                                            *(int64_t*)(row[f]+c*sizeof(int64_t)));
+                                            *(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
                                 }
                                 break;
 
@@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %d\n",
                                             f, c,
-                                            *(int32_t*)(row[f]+c*sizeof(int32_t)));
+                                            *(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
                                 }
                                 break;
 
@@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                                     printf("col%d, row: %"PRId64", "
                                             "value: %f\n",
                                             f, c,
-                                            *(float*)(row[f]+c*sizeof(float)));
+                                            *(float*)((char*)(row[f])+c*sizeof(float)));
                                 }
                                 break;
 
@@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
                     if (offsets) {
                         for (int c = 0; c < rows; c++) {
                             if (offsets[c] != -1) {
-                                int length = *(int16_t*)(row[f] + offsets[c]);
+                                int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
                                 char *buf = calloc(1, length + 1);
-                                strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
+                                strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
                                 printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
                                         c, f, offsets[c], length, buf);
                                 free(buf);
@@ -314,8 +317,9 @@ int main(int argc, char *argv[]) {
     const char* host = "127.0.0.1";
     const char* user = "root";
     const char* passwd = "taosdata";
-
+#ifndef WINDOWS
     argp_parse(&argp, argc, argv, 0, 0, NULL);
+#endif
     TAOS* taos = taos_connect(host, user, passwd, "", 0);
     if (taos == NULL) {
         printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));

example/src/tmq.c

@@ -22,6 +22,7 @@
 static int  running = 1;
 static void msg_process(TAOS_RES* msg) {
   char buf[1024];
+  memset(buf, 0, 1024);
   printf("topic: %s\n", tmq_get_topic_name(msg));
   printf("vg:%d\n", tmq_get_vgroup_id(msg));
   while (1) {
@@ -100,8 +101,7 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*const char* sql = "select * from tu1";*/
-  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
   pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
@@ -140,7 +140,7 @@ int32_t create_topic() {
   return 0;
 }
 
-void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) {
   printf("commit %d\n", resp);
 }
 
@@ -162,7 +162,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   tmq_conf_set(conf, "td.connect.db", "abc1");
   tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
-  tmq_t* tmq = tmq_consumer_new1(conf, NULL, 0);
+  tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   return tmq;
 }
 
@@ -188,7 +188,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       cnt++;
       /*printf("get data\n");*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
       /*} else {*/
       /*break;*/
     }
@@ -218,9 +218,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000);
     if (tmqmessage) {
       msg_process(tmqmessage);
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
 
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
+      /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }
 
@@ -248,7 +248,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
       batchCnt++;
       /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/
       /*msg_process(tmqmessage);*/
-      tmq_message_destroy(tmqmessage);
+      taos_free_result(tmqmessage);
     } else {
       break;
     }

include/client/taos.h

@@ -92,38 +92,14 @@ typedef struct taosField {
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
-typedef struct TAOS_BIND {
+typedef struct TAOS_BIND_v2 {
   int       buffer_type;
   void     *buffer;
-  uintptr_t  buffer_length;  // unused
-  uintptr_t *length;
-  int       *is_null;
-
-  int  is_unsigned;  // unused
-  int *error;        // unused
-  union {
-    int64_t        ts;
-    int8_t         b;
-    int8_t         v1;
-    int16_t        v2;
-    int32_t        v4;
-    int64_t        v8;
-    float          f4;
-    double         f8;
-    unsigned char *bin;
-    char          *nchar;
-  } u;
-  unsigned int allocated;
-} TAOS_BIND;
-
-typedef struct TAOS_MULTI_BIND {
-  int       buffer_type;
-  void     *buffer;
-  uintptr_t buffer_length;
+  int32_t   buffer_length;
   int32_t  *length;
   char     *is_null;
   int       num;
-} TAOS_MULTI_BIND;
+} TAOS_BIND_v2;
 
 typedef enum {
   SET_CONF_RET_SUCC = 0,
@@ -154,22 +130,23 @@ const char *taos_data_type(int type);
 
 DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
 DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+DLL_EXPORT int         taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags);
 DLL_EXPORT int         taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
 DLL_EXPORT int         taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
 
 DLL_EXPORT int         taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
 DLL_EXPORT int         taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
 DLL_EXPORT int         taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int         taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int         taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind);
+DLL_EXPORT int         taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx);
 DLL_EXPORT int         taos_stmt_add_batch(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char       *taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES   *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_RES   *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@@ -239,19 +216,20 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
 
 typedef struct tmq_conf_t tmq_conf_t;
 typedef struct tmq_list_t tmq_list_t;
-// typedef struct tmq_message_t tmq_message_t;
 
-typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param));
+typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
 
 DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
+DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
-#if 1
+#if 0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 #endif
 
-DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
@@ -285,24 +263,19 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
 
-#if 0
-// temporary used function for demo only
-void    tmqShowMsg(tmq_message_t *tmq_message);
-int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
-#endif
-
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 
 DLL_EXPORT char   *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
+// TODO
+#if 0
+DLL_EXPORT char   *tmq_get_block_table_name(TAOS_RES *res);
+#endif
+
 #if 0
-DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
 DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
-DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
-DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
 #endif
-DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);

include/common/taosdef.h

@@ -78,13 +78,6 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
-typedef enum {
-  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
-  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
-} ETsdbBSmaType;
-
-#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
-
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tdatablock.h

@@ -54,12 +54,34 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet);
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
+#define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1)
+#define colDataSetNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] = -1)
+
+#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
+
+#define colDataGetVarData(p1_, r_) ((p1_)->pData + (p1_)->varmeta.offset[(r_)])
+
+#define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes))
+// SColumnInfoData, rowNumber
+#define colDataGetData(p1_, r_)                                        \
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ?  colDataGetVarData(p1_, r_)  \
+                                        :  colDataGetNumData(p1_, r_))
+
 static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) {
+  if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){
+    if(colDataIsNull_var(pColumnInfoData, row)){
+      return true;
+    }
+    char *data = colDataGetVarData(pColumnInfoData, row);
+    return (*data == TSDB_DATA_TYPE_NULL);
+  } 
+  
   if (!pColumnInfoData->hasNull) {
     return false;
   }
-  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
+
+  if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) {
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -86,7 +108,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    return pColumnInfoData->varmeta.offset[row] == -1;
+    return colDataIsNull_var(pColumnInfoData, row);
   } else {
     if (pColumnInfoData->nullbitmap == NULL) {
       return false;
@@ -96,17 +118,10 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
   }
 }
 
-#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT)
-
-// SColumnInfoData, rowNumber
-#define colDataGetData(p1_, r_)                                                        \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
-                                        : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
-
 static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uint32_t currentRow) {
   // There is a placehold for each NULL value of binary or nchar type.
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    pColumnInfoData->varmeta.offset[currentRow] = -1;  // it is a null value of VAR type.
+    colDataSetNull_var(pColumnInfoData, currentRow);  // it is a null value of VAR type.
   } else {
     colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow);
   }
@@ -117,7 +132,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
 static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
     for (int32_t i = start; i < start + nRows; ++i) {
-      pColumnInfoData->varmeta.offset[i] = -1;  // it is a null value of VAR type.
+      colDataSetNull_var(pColumnInfoData,i);  // it is a null value of VAR type.
     }
   } else {
     for (int32_t i = start; i < start + nRows; ++i) {
@@ -265,3 +280,4 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
 #endif
 
 #endif /*_TD_COMMON_EP_H_*/
+

include/common/tdataformat.h

@@ -502,7 +502,7 @@ typedef struct {
 
 #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define kvRowLen(r)            (*(TDRowLenT *)(r))
+#define kvRowLen(r)            (*(uint16_t *)(r))
 #define kvRowNCols(r)          (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
 #define kvRowSetLen(r, len)    kvRowLen(r) = (len)
 #define kvRowSetNCols(r, n)    kvRowNCols(r) = (n)
@@ -608,7 +608,7 @@ void    tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void    tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow  tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, int8_t type, const void *value) {
+static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
@@ -621,7 +621,6 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co
 
   pBuilder->nCols++;
 
-  int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type];
   if (tlen > pBuilder->alloc - pBuilder->size) {
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;

include/common/tglobal.h

@@ -90,6 +90,9 @@ extern bool    tsRetrieveBlockingModel;  // retrieve threads will be blocked
 extern bool    tsKeepOriginalColumnName;
 extern bool    tsDeadLockKillQuery;
 
+// query client
+extern int32_t tsQueryPolicy;
+
 // client
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;

include/common/tmsg.h

@@ -324,13 +324,22 @@ typedef struct SEpSet {
 int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp);
 int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
 int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
-void*   taosDecodeSEpSet(void* buf, SEpSet* pEp);
+void*   taosDecodeSEpSet(const void* buf, SEpSet* pEp);
+
+typedef struct {
+  SEpSet epSet;
+} SMEpSet;
+
+int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq);
+int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq);
 
 typedef struct {
   int8_t  connType;
   int32_t pid;
   char    app[TSDB_APP_NAME_LEN];
   char    db[TSDB_DB_NAME_LEN];
+  char    user[TSDB_USER_LEN];
+  char    passwd[TSDB_PASSWORD_LEN];
   int64_t startTime;
 } SConnectReq;
 
@@ -630,13 +639,19 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
 
 typedef struct {
-  SArray* epSetList;  // SArray<SEpSet>
+  SArray* addrsList;  // SArray<SQueryNodeAddr>
 } SQnodeListRsp;
 
 int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
 void    tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
 
+typedef struct SQueryNodeAddr {
+  int32_t nodeId;  // vgId or qnodeId
+  SEpSet  epSet;
+} SQueryNodeAddr;
+
+
 typedef struct {
   SArray* pArray;  // Array of SUseDbRsp
 } SUseDbBatchRsp;
@@ -680,6 +695,7 @@ int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq);
 
 typedef struct {
   int32_t numOfFuncs;
+  bool    ignoreCodeComment;
   SArray* pFuncNames;
 } SRetrieveFuncReq;
 
@@ -708,6 +724,7 @@ typedef struct {
 
 int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
 int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
+void tFreeSFuncInfo(SFuncInfo *pInfo);
 void    tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
 
 typedef struct {
@@ -980,7 +997,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
 void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
-  int32_t type;
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
   int64_t showId;
@@ -1273,8 +1289,11 @@ typedef struct {
 } SMVCreateStreamRsp, SMSCreateStreamRsp;
 
 typedef struct {
-  char   name[TSDB_TOPIC_FNAME_LEN];
+  char    name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
   int8_t  igExists;
+  int8_t  withTbName;
+  int8_t  withSchema;
+  int8_t  withTag;
   char*   sql;
   char*   ast;
   char    subscribeDbName[TSDB_DB_NAME_LEN];
@@ -1291,10 +1310,14 @@ typedef struct {
 int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
 int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
 
+typedef struct {
+  int64_t consumerId;
+} SMqConsumerLostMsg;
+
 typedef struct {
   int32_t topicNum;
   int64_t consumerId;
-  char*   consumerGroup;
+  char    cgroup[TSDB_CGROUP_LEN];
   SArray* topicNames;  // SArray<char*>
 } SCMSubscribeReq;
 
@@ -1302,7 +1325,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pReq->topicNum);
   tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeString(buf, pReq->consumerGroup);
+  tlen += taosEncodeString(buf, pReq->cgroup);
 
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
@@ -1313,7 +1336,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
 static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
   buf = taosDecodeFixedI32(buf, &pReq->topicNum);
   buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeString(buf, &pReq->consumerGroup);
+  buf = taosDecodeStringTo(buf, pReq->cgroup);
   pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
   for (int32_t i = 0; i < pReq->topicNum; i++) {
     char* name;
@@ -1389,7 +1412,7 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
 }
 
 typedef struct {
-  const char* key;
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
   SArray* lostConsumers;     // SArray<int64_t>
   SArray* removedConsumers;  // SArray<int64_t>
   SArray* newConsumers;      // SArray<int64_t>
@@ -1400,7 +1423,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) {
   if (pRebSub == NULL) {
     goto _err;
   }
-  pRebSub->key = strdup(key);
+  strcpy(pRebSub->key, key);
   pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t));
   if (pRebSub->lostConsumers == NULL) {
     goto _err;
@@ -1425,6 +1448,7 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
+  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 
@@ -1466,8 +1490,12 @@ typedef struct {
 typedef struct {
   float      xFilesFactor;
   int32_t    delay;
-  int8_t     nFuncIds;
+  int32_t    qmsg1Len;
+  int32_t    qmsg2Len;
   func_id_t* pFuncIds;
+  char*      qmsg1;  // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1
+  char*      qmsg2;  // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2
+  int8_t     nFuncIds;
 } SRSmaParam;
 
 typedef struct SVCreateTbReq {
@@ -1927,6 +1955,64 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
   return buf;
 }
 
+enum {
+  TOPIC_SUB_TYPE__DB = 1,
+  TOPIC_SUB_TYPE__TABLE,
+};
+
+typedef struct {
+  int64_t leftForVer;
+  int32_t vgId;
+  int64_t oldConsumerId;
+  int64_t newConsumerId;
+  char    subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int8_t  subType;
+  int8_t  withTbName;
+  int8_t  withSchema;
+  int8_t  withTag;
+  int8_t  withTagSchema;
+  char*   qmsg;
+} SMqRebVgReq;
+
+static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
+  tlen += taosEncodeFixedI32(buf, pReq->vgId);
+  tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
+  tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
+  tlen += taosEncodeString(buf, pReq->subKey);
+  tlen += taosEncodeFixedI8(buf, pReq->subType);
+  tlen += taosEncodeFixedI8(buf, pReq->withTbName);
+  tlen += taosEncodeFixedI8(buf, pReq->withSchema);
+  tlen += taosEncodeFixedI8(buf, pReq->withTag);
+  tlen += taosEncodeFixedI8(buf, pReq->withTagSchema);
+  if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
+    tlen += taosEncodeString(buf, pReq->qmsg);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) {
+  buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
+  buf = taosDecodeFixedI32(buf, &pReq->vgId);
+  buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
+  buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
+  buf = taosDecodeStringTo(buf, pReq->subKey);
+  buf = taosDecodeFixedI8(buf, &pReq->subType);
+  buf = taosDecodeFixedI8(buf, &pReq->withTbName);
+  buf = taosDecodeFixedI8(buf, &pReq->withSchema);
+  buf = taosDecodeFixedI8(buf, &pReq->withTag);
+  buf = taosDecodeFixedI8(buf, &pReq->withTagSchema);
+  if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
+    buf = taosDecodeString(buf, &pReq->qmsg);
+  }
+  return (void*)buf;
+}
+
+typedef struct {
+  int8_t reserved;
+} SMqRebVgRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2259,9 +2345,10 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
   }
 }
 
-static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) {
+static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
   tdDestroyTSmaWrapper(pSW);
-  taosMemoryFreeClear(pSW);
+  taosMemoryFree(pSW);
+  return NULL;
 }
 
 static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
@@ -2397,6 +2484,7 @@ typedef struct {
   int64_t consumerId;
 } SMqRspHead;
 
+#if 0
 typedef struct {
   SMsgHead head;
 
@@ -2410,6 +2498,17 @@ typedef struct {
   uint64_t reqId;
   char     topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
+#endif
+
+typedef struct {
+  SMsgHead head;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int32_t  epoch;
+  uint64_t reqId;
+  int64_t  consumerId;
+  int64_t  blockingTime;
+  int64_t  currentOffset;
+} SMqPollReqV2;
 
 typedef struct {
   int32_t vgId;
@@ -2483,13 +2582,81 @@ static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRs
   return (void*)buf;
 }
 
+typedef struct {
+  SMqRspHead head;
+  int64_t    reqOffset;
+  int64_t    rspOffset;
+  int32_t    skipLogNum;
+  int32_t    blockNum;
+  int8_t     withTbName;
+  int8_t     withSchema;
+  int8_t     withTag;
+  int8_t     withTagSchema;
+  SArray*    blockDataLen;    // SArray<int32_t>
+  SArray*    blockData;       // SArray<SRetrieveTableRsp*>
+  SArray*    blockTbName;     // SArray<char*>
+  SArray*    blockSchema;     // SArray<SSchemaWrapper>
+  SArray*    blockTags;       // SArray<kvrow>
+  SArray*    blockTagSchema;  // SArray<kvrow>
+} SMqDataBlkRsp;
+
+static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
+  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
+  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
+  tlen += taosEncodeFixedI32(buf, pRsp->blockNum);
+  if (pRsp->blockNum != 0) {
+    tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
+    tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTag);
+    tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
+      void*   data = taosArrayGetP(pRsp->blockData, i);
+      tlen += taosEncodeFixedI32(buf, bLen);
+      tlen += taosEncodeBinary(buf, data, bLen);
+    }
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) {
+  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
+  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
+  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
+  buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
+  pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  if (pRsp->blockNum != 0) {
+    buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
+    buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTag);
+    buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
+
+    for (int32_t i = 0; i < pRsp->blockNum; i++) {
+      int32_t bLen = 0;
+      void*   data = NULL;
+      buf = taosDecodeFixedI32(buf, &bLen);
+      buf = taosDecodeBinary(buf, &data, bLen);
+      taosArrayPush(pRsp->blockDataLen, &bLen);
+      taosArrayPush(pRsp->blockData, &data);
+    }
+  }
+  return (void*)buf;
+}
+
 typedef struct {
   SMqRspHead head;
   char       cgroup[TSDB_CGROUP_LEN];
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
+static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
+  // taosMemoryFree(pSubTopicEp->schema.pSchema);
+  taosArrayDestroy(pSubTopicEp->vgs);
+}
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;
@@ -2569,6 +2736,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
   }
   return buf;
 }
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -147,6 +147,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
@@ -175,10 +176,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@@ -204,6 +202,21 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
+  // sync integration
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

include/common/trow.h

@@ -684,6 +684,43 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief The invoker is responsible for memory alloc/dealloc.
+ *
+ * @param pBuilder
+ * @param pBuf Output buffer of STSRow
+ */
+static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) {
+  pBuilder->pBuf = (STSRow *)pBuf;
+  if (!pBuilder->pBuf) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0);
+
+  uint32_t len = 0;
+  switch (pBuilder->rowType) {
+    case TD_ROW_TP:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen);
+#endif
+      break;
+    case TD_ROW_KV:
+#ifdef TD_SUPPORT_BITMAP
+      pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
+#endif
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
 /**
  * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数
  *
@@ -1322,7 +1359,7 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
   printf(">>>");
   for (int i = 0; i < pSchema->numOfCols; ++i) {
     STColumn *stCol = pSchema->columns + i;
-    SCellVal  sVal = {.valType = 255, .val = NULL};
+    SCellVal  sVal = { 255, NULL};
     if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
       break;
     }

include/common/ttokendef.h

@@ -156,87 +156,96 @@
 #define TK_INTERVAL                       138
 #define TK_TOPIC                          139
 #define TK_AS                             140
-#define TK_DESC                           141
-#define TK_DESCRIBE                       142
-#define TK_RESET                          143
-#define TK_QUERY                          144
-#define TK_EXPLAIN                        145
-#define TK_ANALYZE                        146
-#define TK_VERBOSE                        147
-#define TK_NK_BOOL                        148
-#define TK_RATIO                          149
-#define TK_COMPACT                        150
-#define TK_VNODES                         151
-#define TK_IN                             152
-#define TK_OUTPUTTYPE                     153
-#define TK_AGGREGATE                      154
-#define TK_BUFSIZE                        155
-#define TK_STREAM                         156
-#define TK_INTO                           157
-#define TK_TRIGGER                        158
-#define TK_AT_ONCE                        159
-#define TK_WINDOW_CLOSE                   160
-#define TK_WATERMARK                      161
-#define TK_KILL                           162
-#define TK_CONNECTION                     163
-#define TK_MERGE                          164
-#define TK_VGROUP                         165
-#define TK_REDISTRIBUTE                   166
-#define TK_SPLIT                          167
-#define TK_SYNCDB                         168
-#define TK_NULL                           169
-#define TK_NK_QUESTION                    170
-#define TK_NK_ARROW                       171
-#define TK_ROWTS                          172
-#define TK_TBNAME                         173
-#define TK_QSTARTTS                       174
-#define TK_QENDTS                         175
-#define TK_WSTARTTS                       176
-#define TK_WENDTS                         177
-#define TK_WDURATION                      178
-#define TK_CAST                           179
-#define TK_NOW                            180
-#define TK_TODAY                          181
-#define TK_TIMEZONE                       182
-#define TK_COUNT                          183
-#define TK_FIRST                          184
-#define TK_LAST                           185
-#define TK_LAST_ROW                       186
-#define TK_BETWEEN                        187
-#define TK_IS                             188
-#define TK_NK_LT                          189
-#define TK_NK_GT                          190
-#define TK_NK_LE                          191
-#define TK_NK_GE                          192
-#define TK_NK_NE                          193
-#define TK_MATCH                          194
-#define TK_NMATCH                         195
-#define TK_CONTAINS                       196
-#define TK_JOIN                           197
-#define TK_INNER                          198
-#define TK_SELECT                         199
-#define TK_DISTINCT                       200
-#define TK_WHERE                          201
-#define TK_PARTITION                      202
-#define TK_BY                             203
-#define TK_SESSION                        204
-#define TK_STATE_WINDOW                   205
-#define TK_SLIDING                        206
-#define TK_FILL                           207
-#define TK_VALUE                          208
-#define TK_NONE                           209
-#define TK_PREV                           210
-#define TK_LINEAR                         211
-#define TK_NEXT                           212
-#define TK_GROUP                          213
-#define TK_HAVING                         214
-#define TK_ORDER                          215
-#define TK_SLIMIT                         216
-#define TK_SOFFSET                        217
-#define TK_LIMIT                          218
-#define TK_OFFSET                         219
-#define TK_ASC                            220
-#define TK_NULLS                          221
+#define TK_WITH                           141
+#define TK_SCHEMA                         142
+#define TK_DESC                           143
+#define TK_DESCRIBE                       144
+#define TK_RESET                          145
+#define TK_QUERY                          146
+#define TK_EXPLAIN                        147
+#define TK_ANALYZE                        148
+#define TK_VERBOSE                        149
+#define TK_NK_BOOL                        150
+#define TK_RATIO                          151
+#define TK_COMPACT                        152
+#define TK_VNODES                         153
+#define TK_IN                             154
+#define TK_OUTPUTTYPE                     155
+#define TK_AGGREGATE                      156
+#define TK_BUFSIZE                        157
+#define TK_STREAM                         158
+#define TK_INTO                           159
+#define TK_TRIGGER                        160
+#define TK_AT_ONCE                        161
+#define TK_WINDOW_CLOSE                   162
+#define TK_WATERMARK                      163
+#define TK_KILL                           164
+#define TK_CONNECTION                     165
+#define TK_MERGE                          166
+#define TK_VGROUP                         167
+#define TK_REDISTRIBUTE                   168
+#define TK_SPLIT                          169
+#define TK_SYNCDB                         170
+#define TK_NULL                           171
+#define TK_NK_QUESTION                    172
+#define TK_NK_ARROW                       173
+#define TK_ROWTS                          174
+#define TK_TBNAME                         175
+#define TK_QSTARTTS                       176
+#define TK_QENDTS                         177
+#define TK_WSTARTTS                       178
+#define TK_WENDTS                         179
+#define TK_WDURATION                      180
+#define TK_CAST                           181
+#define TK_NOW                            182
+#define TK_TODAY                          183
+#define TK_TIMEZONE                       184
+#define TK_COUNT                          185
+#define TK_FIRST                          186
+#define TK_LAST                           187
+#define TK_LAST_ROW                       188
+#define TK_BETWEEN                        189
+#define TK_IS                             190
+#define TK_NK_LT                          191
+#define TK_NK_GT                          192
+#define TK_NK_LE                          193
+#define TK_NK_GE                          194
+#define TK_NK_NE                          195
+#define TK_MATCH                          196
+#define TK_NMATCH                         197
+#define TK_CONTAINS                       198
+#define TK_JOIN                           199
+#define TK_INNER                          200
+#define TK_SELECT                         201
+#define TK_DISTINCT                       202
+#define TK_WHERE                          203
+#define TK_PARTITION                      204
+#define TK_BY                             205
+#define TK_SESSION                        206
+#define TK_STATE_WINDOW                   207
+#define TK_SLIDING                        208
+#define TK_FILL                           209
+#define TK_VALUE                          210
+#define TK_NONE                           211
+#define TK_PREV                           212
+#define TK_LINEAR                         213
+#define TK_NEXT                           214
+#define TK_GROUP                          215
+#define TK_HAVING                         216
+#define TK_ORDER                          217
+#define TK_SLIMIT                         218
+#define TK_SOFFSET                        219
+#define TK_LIMIT                          220
+#define TK_OFFSET                         221
+#define TK_ASC                            222
+#define TK_NULLS                          223
+#define TK_ID                             224
+#define TK_NK_BITNOT                      225
+#define TK_INSERT                         226
+#define TK_VALUES                         227
+#define TK_IMPORT                         228
+#define TK_NK_SEMI                        229
+#define TK_FILE                           230
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -244,13 +253,6 @@
 #define TK_NK_HEX                         303   // hex number  0x123
 #define TK_NK_OCT                         304   // oct number
 #define TK_NK_BIN                         305   // bin format data 0b111
-#define TK_NK_FILE                        306
-
-#define TK_NK_BITNOT                      501
-#define TK_INSERT                         502
-#define TK_VALUES                         507
-#define TK_IMPORT                         509
-#define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535
 

include/common/ttypes.h

@@ -49,7 +49,7 @@ typedef struct {
 #define varDataCopy(dst, v)    memcpy((dst), (void *)(v), varDataTLen(v))
 #define varDataLenByData(v)    (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
-#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
+#define IS_VAR_DATA_TYPE(t)    (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
 
 #define varDataNetLen(v)  (htons(((VarDataLenT *)(v))[0]))
 #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
@@ -252,7 +252,7 @@ typedef struct tDataTypeDescriptor {
                      int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 
-extern tDataTypeDescriptor tDataTypes[15];
+extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
 
 bool isValidDataType(int32_t type);
 

include/dnode/bnode/bnode.h

@@ -26,6 +26,9 @@ extern "C" {
 typedef struct SBnode SBnode;
 
 typedef struct {
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
 } SBnodeLoad;
 
 typedef struct {

include/dnode/mnode/sdb/sdb.h

@@ -92,7 +92,13 @@ extern "C" {
 typedef struct SMnode  SMnode;
 typedef struct SSdbRaw SSdbRaw;
 typedef struct SSdbRow SSdbRow;
-typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType;
+
+typedef enum {
+  SDB_KEY_BINARY = 1,
+  SDB_KEY_INT32 = 2,
+  SDB_KEY_INT64 = 3,
+} EKeyType;
+
 typedef enum {
   SDB_STATUS_INIT = 0,
   SDB_STATUS_CREATING = 1,

include/libs/catalog/catalog.h

@@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const
  */
 int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
-
 int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
 
 int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);
@@ -224,6 +223,8 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
 int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
 
+int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo);
+
 
 /**
  * Destroy catalog and relase all resources

include/libs/function/function.h

@@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
 typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
-typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
+typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
@@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData {
   SColumnInfoData  *pPTS;           // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;
+  uint64_t          uid;            // table uid
 } SInputColumnInfoData;
 
 // sql function runtime context
@@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx {
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
   int32_t          numOfParams;
-  SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  SFunctParam     *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t         *ptsList;       // corresponding timestamp array list
   SColumnInfoData *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
   int32_t          offset;

include/libs/function/functionMgt.h

@@ -119,12 +119,19 @@ typedef enum EFunctionType {
 struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
+struct SCatalog;
+
+typedef struct SFmGetFuncInfoParam {
+  struct SCatalog* pCtg;
+  void *pRpc;
+  const SEpSet* pMgmtEps;
+} SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
 

include/libs/nodes/cmdnodes.h

@@ -241,12 +241,20 @@ typedef struct SDropComponentNodeStmt {
   int32_t dnodeId;
 } SDropComponentNodeStmt;
 
+typedef struct STopicOptions {
+  ENodeType type;
+  bool withTable;
+  bool withSchema;
+  bool withTag;
+} STopicOptions;
+
 typedef struct SCreateTopicStmt {
   ENodeType type;
   char topicName[TSDB_TABLE_NAME_LEN];
   char subscribeDbName[TSDB_DB_NAME_LEN];
   bool ignoreExists;
   SNode* pQuery;
+  STopicOptions* pOptions;
 } SCreateTopicStmt;
 
 typedef struct SDropTopicStmt {

include/libs/nodes/nodes.h

@@ -48,6 +48,9 @@ extern "C" {
     (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \
     cell1 = cell1->pNext, cell2 = cell2->pNext)
 
+#define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode)
+#define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode)
+
 #define FOREACH_FOR_REWRITE(node, list)	\
   for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext)
 
@@ -81,6 +84,7 @@ typedef enum ENodeType {
   QUERY_NODE_INDEX_OPTIONS,
   QUERY_NODE_EXPLAIN_OPTIONS,
   QUERY_NODE_STREAM_OPTIONS,
+  QUERY_NODE_TOPIC_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,

include/libs/nodes/plannodes.h

@@ -155,7 +155,6 @@ typedef struct SLogicSubplan {
 
 typedef struct SQueryLogicPlan {
   ENodeType type;
-  int32_t totalLevel;
   SNodeList* pTopSubplans;
 } SQueryLogicPlan;
 

include/libs/nodes/querynodes.h

@@ -229,28 +229,30 @@ typedef struct SFillNode {
 typedef struct SSelectStmt {
   ENodeType type; // QUERY_NODE_SELECT_STMT
   bool isDistinct;
-  SNodeList* pProjectionList; // SNode
+  SNodeList* pProjectionList;
   SNode* pFromTable;
   SNode* pWhere;
-  SNodeList* pPartitionByList; // SNode
+  SNodeList* pPartitionByList;
   SNode* pWindow;
   SNodeList* pGroupByList; // SGroupingSetNode
   SNode* pHaving;
   SNodeList* pOrderByList; // SOrderByExprNode
-  SNode* pLimit;
-  SNode* pSlimit;
+  SLimitNode* pLimit;
+  SLimitNode* pSlimit;
   char stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t precision;
   bool isEmptyResult;
 } SSelectStmt;
 
 typedef enum ESetOperatorType {
-  SET_OP_TYPE_UNION_ALL = 1
+  SET_OP_TYPE_UNION_ALL = 1,
+  SET_OP_TYPE_UNION
 } ESetOperatorType;
 
 typedef struct SSetOperator {
   ENodeType type; // QUERY_NODE_SET_OPERATOR
   ESetOperatorType opType;
+  SNodeList* pProjectionList;
   SNode* pLeft;
   SNode* pRight;
   SNodeList* pOrderByList; // SOrderByExprNode

include/libs/parser/parser.h

@@ -21,6 +21,15 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "query.h"
+
+typedef struct SStmtCallback {
+  TAOS_STMT* pStmt;
+  int32_t (*getTbNameFn)(TAOS_STMT*, char**);
+  int32_t (*setBindInfoFn)(TAOS_STMT*, STableMeta*, void*);
+  int32_t (*setExecInfoFn)(TAOS_STMT*, SHashObj*, SHashObj*);
+  int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
+} SStmtCallback;
 
 typedef struct SParseContext {
   uint64_t         requestId;
@@ -34,6 +43,7 @@ typedef struct SParseContext {
   char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
   int32_t          msgLen;         // max length of the msg
   struct SCatalog *pCatalog;
+  SStmtCallback   *pStmtCb;
 } SParseContext;
 
 typedef struct SCmdMsgInfo {
@@ -66,11 +76,27 @@ typedef struct SQuery {
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
+bool isInsertSql(const char* pStr, size_t length);
 
 void qDestroyQuery(SQuery* pQueryNode);
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
 
+int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
+int32_t qResetStmtDataBlock(void* block, bool keepBuf);
+int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
+void    qFreeStmtDataBlock(void* pDataBlock);
+int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
+void    qDestroyStmtDataBlock(void* pBlock);
+int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
+int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
+int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen);
+void destroyBoundColumnInfo(void* pBoundInfo);
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
+
+
 #ifdef __cplusplus
 }
 #endif

include/libs/planner/planner.h

@@ -30,10 +30,13 @@ typedef struct SPlanContext {
   SNode* pAstRoot;
   bool topicQuery;
   bool streamQuery;
+  bool rSmaQuery;
   bool showRewrite;
   int8_t triggerType;
   int64_t watermark;
   bool isStmtQuery;
+  void* pTransporter;
+  struct SCatalog* pCatalog;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.
@@ -45,7 +48,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-typedef TAOS_MULTI_BIND TAOS_BIND_v2; // todo remove
 int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams);
 
 // Convert to subplan to string for the scheduler to send to the executor

include/libs/qcom/query.h

@@ -51,6 +51,10 @@ typedef struct STableComInfo {
 
 typedef struct SIndexMeta {
 
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
+
 } SIndexMeta;
 
 
@@ -129,12 +133,6 @@ typedef struct SMsgSendInfo {
   SDataBuf             msgInfo;
 } SMsgSendInfo;
 
-typedef struct SQueryNodeAddr {
-  int32_t nodeId;  // vgId or qnodeId
-  SEpSet  epSet;
-} SQueryNodeAddr;
-
-
 typedef struct SQueryNodeStat {
   int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;

include/libs/scalar/scalar.h

@@ -76,6 +76,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 /* Time related functions */
 int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
-#include "taosdef.h"
-#include "trpc.h"
-#include "wal.h"
+//#include <tdatablock.h>
+#include "cJSON.h"
+#include "tdef.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  uint16_t nodePort;
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
-typedef struct SSyncFSM {
-  void* data;
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                     ESyncState state);
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
 
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
+typedef struct SSyncFSM {
+  void* data;
 
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
 
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN   0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
-typedef struct SStateMgr {
-  void* data;
-
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+struct SWal;
+typedef struct SWal SWal;
 
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
+struct SEpSet;
+typedef struct SEpSet SEpSet;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,27 +140,35 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
-struct SSyncNode;
-typedef struct SSyncNode SSyncNode;
-
 int32_t     syncInit();
 void        syncCleanUp();
-
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
 void        syncStop(int64_t rid);
 int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
 
-// propose with sequence number, to implement linearizable semantics
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
+typedef enum {
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
 
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
 extern int32_t sDebugFlag;
 
-const char *syncStr(ESyncState state);
+//-----------------------------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+struct SSyncBuffer;
+typedef struct SSyncBuffer SSyncBuffer;
+//-----------------------------------------
+
+const char* syncStr(ESyncState state);
 
 #ifdef __cplusplus
 }

include/libs/sync/syncTools.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -38,11 +38,11 @@ typedef struct SRpcConnInfo {
 
 typedef struct SRpcMsg {
   tmsg_t  msgType;
-  void *  pCont;
+  void   *pCont;
   int     contLen;
   int32_t code;
-  void *  handle;         // rpc handle returned to app
-  void *  ahandle;        // app handle set by client
+  void   *handle;         // rpc handle returned to app
+  void   *ahandle;        // app handle set by client
   int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int     persistHandle;  // persist handle or not
 
@@ -60,10 +60,11 @@ typedef struct {
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
 typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
-  char *   label;         // for debug purpose
+  char    *label;         // for debug purpose
   int      numOfThreads;  // number of threads to handle connections
   int      sessions;      // number of sessions allowed
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
@@ -80,7 +81,10 @@ typedef struct SRpcInit {
   RpcCfp cfp;
 
   // call back to retrieve the client auth info, for server app only
-  RpcAfp afp;;
+  RpcAfp afp;
+
+  // user defined retry func
+  RpcRfp rfp;
 
   void *parent;
 } SRpcInit;
@@ -99,17 +103,17 @@ typedef struct {
 } SRpcBrokenlinkVal;
 
 typedef struct {
-  SHashObj *        args;
+  SHashObj         *args;
   SRpcBrokenlinkVal brokenVal;
 } SRpcCtx;
 
 int32_t rpcInit();
 void    rpcCleanup();
-void *  rpcOpen(const SRpcInit *pRpc);
+void   *rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
-void *  rpcMallocCont(int contLen);
+void   *rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
-void *  rpcReallocCont(void *ptr, int contLen);
+void   *rpcReallocCont(void *ptr, int contLen);
 
 // Because taosd supports multi-process mode
 // These functions should not be used on the server side

include/os/os.h

@@ -22,7 +22,6 @@ extern "C" {
 
 #include <assert.h>
 #include <ctype.h>
-#include <semaphore.h>
 
 #include <regex.h>
 
@@ -51,6 +50,15 @@ extern "C" {
 #include <sys/prctl.h>
 #include <argp.h>
 #endif
+#else
+
+#include <time.h>
+#ifndef TD_USE_WINSOCK
+#include <winsock2.h>
+#else
+#include <winsock.h>
+#endif
+
 
 #endif
 

include/os/osAtomic.h

@@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, int32_t val);
+void* atomic_add_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, int32_t val);
+void* atomic_fetch_add_ptr(void *ptr, void *val);
 int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
+void* atomic_sub_fetch_ptr(void *ptr, void *val);
 int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
+void* atomic_fetch_sub_ptr(void *ptr, void *val);
 int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);

include/os/osDef.h

@@ -191,7 +191,7 @@ extern "C" {
   #define threadlocal __declspec( thread )
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define PRIzu "ld"  
 #else
   #define PRIzu "zu"  

include/os/osDir.h

@@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr;
 
 
 void    taosRemoveDir(const char *dirname);
-bool    taosDirExist(char *dirname);
+bool    taosDirExist(const char *dirname);
 int32_t taosMkDir(const char *dirname);
 int32_t taosMulMkDir(const char *dirname);
 void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
 int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
-int32_t taosRealPath(char *dirname, int32_t maxlen);
+int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
 bool    taosIsDir(const char *dirname);
 char*   taosDirName(char *dirname);
 char*   taosDirEntryBaseName(char *dirname);
@@ -50,7 +50,7 @@ TdDirPtr      taosOpenDir(const char *dirname);
 TdDirEntryPtr taosReadDir(TdDirPtr pDir);
 bool          taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
 char*         taosGetDirEntryName(TdDirEntryPtr pDirEntry);
-int32_t       taosCloseDir(TdDirPtr pDir);
+int32_t       taosCloseDir(TdDirPtr *ppDir);
 
 #ifdef __cplusplus
 }

include/os/osEnv.h

@@ -34,6 +34,7 @@ extern int64_t         tsOpenMax;
 extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
+extern char*           tsProcPath;
 
 extern char configDir[];
 extern char tsDataDir[];

include/os/osFile.h

@@ -93,7 +93,6 @@ void    taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix,
 
 int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
 
-void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
 bool taosValidFile(TdFilePtr pFile);
 
 int32_t taosGetErrorFile(TdFilePtr pFile);

include/os/osLz4.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   int32_t BUILDIN_CLZL(uint64_t val);
   int32_t BUILDIN_CLZ(uint32_t val);
   int32_t BUILDIN_CTZL(uint64_t val);

include/os/osMath.h

@@ -23,7 +23,7 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define TSWAP(a, b, c) \
     do {                \
@@ -33,6 +33,7 @@ extern "C" {
     } while (0)
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
+  #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 

include/os/osSemaphore.h

@@ -22,21 +22,28 @@ extern "C" {
 
 #include <semaphore.h>
 
-#if defined (_TD_DARWIN_64)
-  typedef struct tsem_s *tsem_t;
-  int tsem_init(tsem_t *sem, int pshared, unsigned int value);
-  int tsem_wait(tsem_t *sem);
-  int tsem_post(tsem_t *sem);
-  int tsem_destroy(tsem_t *sem);
+#if defined(_TD_DARWIN_64)
+
+typedef struct tsem_s *tsem_t;
+
+int tsem_init(tsem_t *sem, int pshared, unsigned int value);
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+int tsem_post(tsem_t *sem);
+int tsem_destroy(tsem_t *sem);
+
 #else
-  #define tsem_t sem_t
-  #define tsem_init sem_init
-  int tsem_wait(tsem_t* sem);
-  #define tsem_post sem_post
-  #define tsem_destroy sem_destroy
+
+#define tsem_t       sem_t
+#define tsem_init    sem_init
+int tsem_wait(tsem_t *sem);
+int tsem_timewait(tsem_t *sim, int64_t nanosecs);
+#define tsem_post    sem_post
+#define tsem_destroy sem_destroy
+
 #endif
 
-#if defined (_TD_DARWIN_64)
+#if defined(_TD_DARWIN_64)
 //  #define TdThreadRwlock TdThreadMutex
 //  #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
 //  #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
@@ -44,20 +51,20 @@ extern "C" {
 //  #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
 //  #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
 
-  #define TdThreadSpinlock TdThreadMutex
-  #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
-  #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
-  #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
-  #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
+#define TdThreadSpinlock               TdThreadMutex
+#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
+#define taosThreadSpinDestroy(lock)    taosThreadMutexDestroy(lock)
+#define taosThreadSpinLock(lock)       taosThreadMutexLock(lock)
+#define taosThreadSpinUnlock(lock)     taosThreadMutexUnlock(lock)
 #endif
 
 bool    taosCheckPthreadValid(TdThread thread);
 int64_t taosGetSelfPthreadId();
 int64_t taosGetPthreadId(TdThread thread);
-void    taosResetPthread(TdThread* thread);
+void    taosResetPthread(TdThread *thread);
 bool    taosComparePthread(TdThread first, TdThread second);
 int32_t taosGetPId();
-int32_t taosGetAppName(char* name, int32_t* len);
+int32_t taosGetAppName(char *name, int32_t *len);
 
 #ifdef __cplusplus
 }

include/os/osSocket.h

@@ -31,10 +31,31 @@
 #endif
 
 #if defined(WINDOWS)
-  #include "winsock2.h"
-  #include <WS2tcpip.h>
-  #include <winbase.h>
-  #include <Winsock2.h>
+  #if BYTE_ORDER == LITTLE_ENDIAN
+    #include <stdlib.h>
+    #define htobe16(x) _byteswap_ushort(x)
+    #define htole16(x) (x)
+    #define be16toh(x) _byteswap_ushort(x)
+    #define le16toh(x) (x)
+                      
+    #define htobe32(x) _byteswap_ulong(x)
+    #define htole32(x) (x)
+    #define be32toh(x) _byteswap_ulong(x)
+    #define le32toh(x) (x)
+                      
+    #define htobe64(x) _byteswap_uint64(x)
+    #define htole64(x) (x)
+    #define be64toh(x) _byteswap_uint64(x)
+    #define le64toh(x) (x)
+  #else
+    #error byte order not supported
+  #endif
+
+  #define __BYTE_ORDER    BYTE_ORDER
+  #define __BIG_ENDIAN    BIG_ENDIAN
+  #define __LITTLE_ENDIAN LITTLE_ENDIAN
+  #define __PDP_ENDIAN    PDP_ENDIAN
+
 #else
   #include <netinet/in.h>
   #include <sys/socket.h>
@@ -52,14 +73,10 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  #define htobe64 htonll
-#endif
-
-#if defined(WINDOWS)
+  typedef int socklen_t;
   #define TAOS_EPOLL_WAIT_TIME 100
   typedef SOCKET eventfd_t;
   #define eventfd(a, b) -1
-  typedef SOCKET EpollFd;
   #define EpollClose(pollFd) epoll_close(pollFd)
   #ifndef EPOLLWAKEUP
     #define EPOLLWAKEUP (1u << 29)
@@ -79,31 +96,29 @@ extern "C" {
 #if defined(_TD_DARWIN_64)
 //  #define htobe64 htonll
 
-#	include <libkern/OSByteOrder.h>
+#include <libkern/OSByteOrder.h>
 
-#	define htobe16(x) OSSwapHostToBigInt16(x)
-#	define htole16(x) OSSwapHostToLittleInt16(x)
-#	define be16toh(x) OSSwapBigToHostInt16(x)
-#	define le16toh(x) OSSwapLittleToHostInt16(x)
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
 
-#	define htobe32(x) OSSwapHostToBigInt32(x)
-#	define htole32(x) OSSwapHostToLittleInt32(x)
-#	define be32toh(x) OSSwapBigToHostInt32(x)
-#	define le32toh(x) OSSwapLittleToHostInt32(x)
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
 
-#	define htobe64(x) OSSwapHostToBigInt64(x)
-#	define htole64(x) OSSwapHostToLittleInt64(x)
-#	define be64toh(x) OSSwapBigToHostInt64(x)
-#	define le64toh(x) OSSwapLittleToHostInt64(x)
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
 
-#	define __BYTE_ORDER    BYTE_ORDER
-#	define __BIG_ENDIAN    BIG_ENDIAN
-#	define __LITTLE_ENDIAN LITTLE_ENDIAN
-#	define __PDP_ENDIAN    PDP_ENDIAN
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-#define TAOS_EPOLL_WAIT_TIME 500
-
 typedef int32_t SocketFd;
 typedef SocketFd  EpollFd;
 
@@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr;
 int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
+void taosWinSocketInit();
 
 TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
-TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
+TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
 
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
+int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen);
 
 void     taosBlockSIGPIPE();
 uint32_t taosGetIpv4FromFqdn(const char *);

include/os/osString.h

@@ -39,7 +39,7 @@ typedef int32_t TdUcs4;
     #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
   #define tstrdup(str) _strdup(str)
 #else
   #define tstrdup(str) strdup(str)

include/os/osThread.h

@@ -22,9 +22,12 @@
 extern "C" {
 #endif
 
+#ifndef WINDOWS
 #ifndef __USE_XOPEN2K
+#define TD_USE_SPINLOCK_AS_MUTEX
 typedef pthread_mutex_t pthread_spinlock_t;
 #endif
+#endif
 
 typedef pthread_t TdThread;
 typedef pthread_spinlock_t TdThreadSpinlock;
@@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey;
 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop pthread_cleanup_pop
 
+
+#ifdef WINDOWS
+#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
+#else
+#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#endif
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
 int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
 void taosThreadTestCancel(void);
-int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
-int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
+void taosThreadClear(TdThread *thread);
 
 #ifdef __cplusplus
 }

include/os/osTime.h

@@ -20,8 +20,6 @@
 extern "C" {
 #endif
 
-#include <time.h>
-
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -34,7 +32,7 @@ extern "C" {
     #define mktime MKTIME_FUNC_TAOS_FORBID
 #endif
 
-#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+#ifdef WINDOWS
 
   #define CLOCK_REALTIME 	0
 
@@ -58,6 +56,8 @@ extern "C" {
 
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
+int32_t taosClockGetTime(int clock_id, struct timespec *pTS);
+
 //@return timestamp in second
 int32_t taosGetTimestampSec();
 
@@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() {
 //@return timestamp in nanosecond
 static FORCE_INLINE int64_t taosGetTimestampNs() {
   struct timespec systemTime = {0};
-  clock_gettime(CLOCK_REALTIME, &systemTime);
+  taosClockGetTime(CLOCK_REALTIME, &systemTime);
   return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
 }
 

include/util/talgo.h

@@ -27,6 +27,11 @@ extern "C" {
 typedef int32_t (*__compar_fn_t)(const void *, const void *);
 #endif
 
+typedef void *(*FCopy)(void *);
+typedef void (*FDelete)(void *);
+typedef int32_t (*FEncode)(void **buf, const void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst);
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4
@@ -77,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
  * @return
  */
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                    __ext_compar_fn_t compar, char* buf, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -93,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @return
  */
 void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                  bool maxroot);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -132,6 +132,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TSC_INVALID_JSON_TYPE         TAOS_DEF_ERROR_CODE(0, 0x0222)
 #define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE        TAOS_DEF_ERROR_CODE(0, 0x0223)
 #define TSDB_CODE_TSC_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0X0224)
+#define TSDB_CODE_TSC_STMT_API_ERROR            TAOS_DEF_ERROR_CODE(0, 0X0225)
+#define TSDB_CODE_TSC_STMT_TBNAME_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0226)
+#define TSDB_CODE_TSC_STMT_CLAUSE_ERROR         TAOS_DEF_ERROR_CODE(0, 0X0227)
 
 // mnode-common
 #define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
@@ -279,6 +282,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -407,6 +411,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+// sync integration
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)
@@ -609,6 +617,12 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INTER_SLIDING_UNIT        TAOS_DEF_ERROR_CODE(0, 0x2630)
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG     TAOS_DEF_ERROR_CODE(0, 0x2631)
 #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
+#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG         TAOS_DEF_ERROR_CODE(0, 0x2633)
+#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL      TAOS_DEF_ERROR_CODE(0, 0x2634)
+#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL   TAOS_DEF_ERROR_CODE(0, 0x2635)
+#define TSDB_CODE_PAR_INVALID_DAYS_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2636)
+#define TSDB_CODE_PAR_OFFSET_LESS_ZERO          TAOS_DEF_ERROR_CODE(0, 0x2637)
+#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY  TAOS_DEF_ERROR_CODE(0, 0x2638)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tarray.h

@@ -199,6 +199,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize);
  */
 SArray* taosArrayDup(const SArray* pSrc);
 
+/**
+ * deep copy a new array
+ * @param pSrc
+ */
+SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
+
+
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -212,19 +219,9 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
-
-/**
- * destroy array list
- * @param pArray
- */
 void* taosArrayDestroy(SArray* pArray);
-
-/**
- *
- * @param pArray
- * @param fp
- */
-void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
+void  taosArrayDestroyP(SArray* pArray, FDelete fp);
+void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
 
 /**
  * sort the array
@@ -272,6 +269,9 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t
 
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
+int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
+void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tcache.h

@@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
 /**
  * initialize the cache object
  * @param keyType              key type
- * @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
+ * @param refreshTimeInMs      refresh operation interval time, the maximum survival time when one element is expired
  *                             and not referenced by other objects
  * @param extendLifespan       auto extend lifespan, if accessed
  * @param fn                   free resource callback function
  * @return
  */
-SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
+SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
                          const char *cacheName);
 
 /**
@@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
  * @param pCacheObj
  * @return
  */
-size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
+size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj);
 
 /**
  *  move all data node into trash, clear node in trash can if it is not referenced by any clients
@@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
  */
 void taosStopCacheRefreshWorker();
 
-SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
-bool taosCacheIterNext(SCacheIter* pIter);
-void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
-void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
-void taosCacheDestroyIter(SCacheIter* pIter);
+SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj);
+bool        taosCacheIterNext(SCacheIter *pIter);
+void       *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen);
+void       *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen);
+void        taosCacheDestroyIter(SCacheIter *pIter);
 
 #ifdef __cplusplus
 }

include/util/tcompare.h

@@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
+int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);

include/util/tdef.h

@@ -116,7 +116,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_INS_TABLE_USER_USERS             "user_users"
 #define TSDB_INS_TABLE_LICENCES               "grants"
 #define TSDB_INS_TABLE_VGROUPS                "vgroups"
-#define TSDB_INS_TABLE_TOPICS                 "topics"
 #define TSDB_INS_TABLE_CONSUMERS              "consumers"
 #define TSDB_INS_TABLE_SUBSCRIBES             "subscribes"
 #define TSDB_INS_TABLE_TRANS                  "trans"
@@ -271,6 +270,8 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TAGS           128
 #define TSDB_MAX_TAG_CONDITIONS 1024
 
+#define TSDB_MAX_JSON_TAG_LEN   16384
+
 #define TSDB_AUTH_LEN          16
 #define TSDB_PASSWORD_LEN      32
 #define TSDB_USET_PASSWORD_LEN 129

include/util/tencode.h

@@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) {
   union {
     uint32_t ui;
     float    f;
-  } v = {.f = val};
+  } v;
+  v.f = val;
 
   return tEncodeU32(pEncoder, v.ui);
 }
@@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) {
   union {
     uint64_t ui;
     double   d;
-  } v = {.d = val};
+  } v;
+  v.d = val;
 
   return tEncodeU64(pEncoder, v.ui);
 }

include/util/tjson.h

@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#ifdef WINDOWS
+#define tjsonGetNumberValue(pJson, pName, val) -1
+#else
 #define tjsonGetNumberValue(pJson, pName, val) \
   ({ \
     uint64_t _tmp = 0; \
@@ -29,6 +32,7 @@ extern "C" {
     val = _tmp; \
     _code; \
   })
+#endif
 
 typedef void SJson;
 
@@ -76,6 +80,7 @@ char* tjsonToString(const SJson* pJson);
 char* tjsonToUnformattedString(const SJson* pJson);
 
 SJson* tjsonParse(const char* pStr);
+bool   tjsonValidateJson(const char* pJson);
 
 #ifdef __cplusplus
 }

include/util/tlog.h

@@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
+extern int32_t fnDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

include/util/tutil.h

@@ -26,8 +26,6 @@ extern "C" {
 #endif
 
 int32_t strdequote(char *src);
-int32_t strndequote(char *dst, const char *z, int32_t len);
-int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
 char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);

packaging/install.sh

@@ -30,6 +30,15 @@ bin_dir="/usr/local/taos/bin"
 
 service_config_dir="/etc/systemd/system"
 
+#taos-tools para
+demoName="taosdemo"
+benchmarkName="taosBenchmark"
+dumpName="taosdump"
+emailName="taosdata.com"
+taosName="taos"
+toolsName="taostools"
+
+
 # Color setting
 RED='\033[0;31m'
 GREEN='\033[1;32m'
@@ -230,8 +239,20 @@ function install_header() {
 
 # temp install taosBenchmark
 function install_taosTools() {
-    cd ${script_dir}/taos-tools/
-    tar xvf taosTools-1.4.1-Linux-x64.tar.gz && cd  taosTools-1.4.1/ && ./install-taostools.sh
+    ${csudo} rm -f ${bin_link_dir}/${benchmarkName}    || :
+    ${csudo} rm -f ${bin_link_dir}/${dumpName}         || :
+    ${csudo} rm -f ${bin_link_dir}/rm${toolsName}      || :
+
+    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${dumpName} ${install_main_dir}/bin/${dumpName}
+    ${csudo} /usr/bin/install -c -m 755 ${script_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${benchmarkName}
+    ${csudo} ln -sf ${install_main_dir}/bin/${benchmarkName} ${install_main_dir}/bin/${demoName}
+    #Make link
+    [[ -x ${install_main_dir}/bin/${benchmarkName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName}        || :
+    [[ -x ${install_main_dir}/bin/${demoName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${demoName} ${bin_link_dir}/${demoName}                  || :
+    [[ -x ${install_main_dir}/bin/${dumpName} ]] && \
+        ${csudo} ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName}                  || :
 }
 
 function add_newHostname_to_hosts() {

packaging/release.sh

@@ -39,7 +39,7 @@ cd ${compile_dir}
 
 echo "compile_dir: ${compile_dir}"
 
-cmake ..
+cmake .. -DBUILD_TOOLS=true
 make -j32
 
 release_dir="${top_dir}/release"
@@ -55,7 +55,6 @@ mkdir -p ${install_dir}
 mkdir -p ${install_dir}/bin
 mkdir -p ${install_dir}/lib
 mkdir -p ${install_dir}/inc
-mkdir -p ${install_dir}/taos-tools
 
 install_files="${script_dir}/install.sh"
 chmod a+x ${script_dir}/install.sh || :
@@ -64,13 +63,14 @@ cp ${install_files} ${install_dir}
 header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h"
 cp ${header_files} ${install_dir}/inc
  
-bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos  ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh"
-cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
+bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos  ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh ${compile_dir}/build/bin/taosBenchmark   ${compile_dir}/build/bin/taosdump"
+cp -rf ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
+
+cp -rf ${compile_dir}/source/client/libtaos.so  ${install_dir}/lib/
+cp -rf ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/
+cp -rf ${compile_dir}/build/lib/libavro* ${install_dir}/lib/ > /dev/null || echo -e "failed to copy avro libraries"
+cp -rf ${compile_dir}/build/lib/pkgconfig  ${install_dir}/lib/ > /dev/null || echo -e "failed to copy pkgconfig directory"
 
-cp ${compile_dir}/build/lib/libtaos.so  ${install_dir}/lib/
-cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/
-taostoolfile="${top_dir}/tools/taosTools-1.4.1-Linux-x64.tar.gz"
-cp ${taostoolfile} ${install_dir}/taos-tools 
 
 #cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
 #cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/

source/client/inc/clientInt.h

@@ -188,10 +188,12 @@ typedef struct SRequestSendRecvBody {
 
 typedef struct {
   int8_t         resType;
-  char*   topic;
-  SArray* res;  // SArray<SReqResultInfo>
-  int32_t resIter;
+  char           topic[TSDB_TOPIC_FNAME_LEN];
   int32_t        vgId;
+  SSchemaWrapper schema;
+  int32_t        resIter;
+  SMqDataBlkRsp  rsp;
+  SReqResultInfo resInfo;
 } SMqRspObj;
 
 typedef struct SRequestObj {
@@ -203,7 +205,8 @@ typedef struct SRequestObj {
   char*                sqlstr;  // sql string
   int32_t              sqlLen;
   int64_t              self;
-  char*                msgBuf;  // error msg buffer
+  char*                msgBuf;
+  int32_t              msgBufLen;
   int32_t              code;
   SArray*              dbList;
   SArray*              tableList;
@@ -211,16 +214,25 @@ typedef struct SRequestObj {
   SRequestSendRecvBody body;
 } SRequestObj;
 
+void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
+void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
+void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
+void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
+
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  int32_t    resIter = msg->resIter == -1 ? 0 : msg->resIter;
-  return (SReqResultInfo*)taosArrayGet(msg->res, resIter);
+  return (SReqResultInfo*)&msg->resInfo;
 }
 
-static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res) {
+static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
   SMqRspObj* msg = (SMqRspObj*)res;
-  if (++msg->resIter < taosArrayGetSize(msg->res)) {
-    return (SReqResultInfo*)taosArrayGet(msg->res, msg->resIter);
+  msg->resIter++;
+  if (msg->resIter < msg->rsp.blockNum) {
+    SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
+    setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
+    return &msg->resInfo;
   }
   return NULL;
 }
@@ -242,14 +254,14 @@ int   taos_init();
 
 void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
 void     destroyTscObj(void* pObj);
-STscObj *acquireTscObj(int64_t rid);
+STscObj* acquireTscObj(int64_t rid);
 int32_t  releaseTscObj(int64_t rid);
 
 uint64_t generateRequestId();
 
 void*        createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
 void         destroyRequest(SRequestObj* pRequest);
-SRequestObj *acquireRequest(int64_t rid);
+SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
 
 char* getDbOfConnection(STscObj* pObj);
@@ -268,17 +280,12 @@ void initMsgHandleFp();
 TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                             uint16_t port, int connType);
 
-int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery);
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
+
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
-void*   doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
-void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
-void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
-void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
-
 // --- heartbeat
 // global, called by mgmt
 int  hbMgrInit();
@@ -290,7 +297,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
 void       appHbMgrCleanup(void);
 
 // conn level
-int  hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
+int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);
 void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
 
 int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
@@ -298,6 +305,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 // --- mq
 void hbMgrInitMqHbRspHandle();
 
+SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
+
 #ifdef __cplusplus
 }
 #endif

source/client/inc/clientStmt.h

@@ -19,6 +19,9 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "catalog.h"
+
+typedef void STableDataBlocks;
 
 typedef enum {
   STMT_TYPE_INSERT = 1,
@@ -26,17 +29,65 @@ typedef enum {
   STMT_TYPE_QUERY,
 } STMT_TYPE;
 
-typedef struct STscStmt {
+typedef enum {
+  STMT_INIT = 1,
+  STMT_PREPARE,
+  STMT_SETTBNAME,
+  STMT_SETTAGS,
+  STMT_FETCH_TAG_FIELDS,
+  STMT_FETCH_COL_FIELDS,
+  STMT_BIND,
+  STMT_BIND_COL,
+  STMT_ADD_BATCH,
+  STMT_EXECUTE,
+} STMT_STATUS;
+
+typedef struct SStmtTableCache {
+  STableDataBlocks* pDataBlock;
+  void*             boundTags;
+} SStmtTableCache;
+
+typedef struct SStmtBindInfo {
+  bool         needParse;
+  uint64_t     tbUid;
+  uint64_t     tbSuid;
+  int32_t      sBindRowNum;
+  int32_t      sBindLastIdx;
+  int8_t       tbType;
+  void*        boundTags;  
+  char*        tbName;
+  SName        sname;
+} SStmtBindInfo;
+
+typedef struct SStmtExecInfo {
+  int32_t      affectedRows;
+  SRequestObj* pRequest;
+  SHashObj*    pVgHash;
+  SHashObj*    pBlockHash;
+} SStmtExecInfo;
+
+typedef struct SStmtSQLInfo {
   STMT_TYPE    type;
-  //int16_t  last;
-  //STscObj* taos;
-  //SSqlObj* pSql;
-  //SMultiTbStmt mtb;
-  //SNormalStmt normal;
+  STMT_STATUS  status;
+  bool         autoCreate;
+  uint64_t     runTimes;
+  SHashObj*    pTableCache;   //SHash<SStmtTableCache>
+  SQuery*      pQuery;
+  char*        sqlStr;
+  int32_t      sqlLen;
+} SStmtSQLInfo;
 
-  //int numOfRows;
+typedef struct STscStmt {
+  STscObj*      taos;
+  SCatalog*     pCatalog;
+  int32_t       affectedRows;
+
+  SStmtSQLInfo  sql;
+  SStmtExecInfo exec;
+  SStmtBindInfo bInfo;
 } STscStmt;
 
+
 #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
 #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
 #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
@@ -44,16 +95,16 @@ typedef struct STscStmt {
 TAOS_STMT *stmtInit(TAOS *taos);
 int stmtClose(TAOS_STMT *stmt);
 int stmtExec(TAOS_STMT *stmt);
-char *stmtErrstr(TAOS_STMT *stmt);
+const char *stmtErrstr(TAOS_STMT *stmt);
 int stmtAffectedRows(TAOS_STMT *stmt);
-int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind);
 int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags);
+int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags);
 int stmtIsInsert(TAOS_STMT *stmt, int *insert);
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums);
 int stmtAddBatch(TAOS_STMT *stmt);
 TAOS_RES *stmtUseResult(TAOS_STMT *stmt);
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx);
 
 
 #ifdef __cplusplus

source/client/src/clientEnv.c

@@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) {
   if (pTscObj->pAppInfo) {
     SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary;
 
-    int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1);
-    int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1);
+    int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
+    int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
     tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
              ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
              pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
@@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) {
 static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj *         pTscObj = pRequest->pTscObj;
+  STscObj          *pTscObj = pRequest->pTscObj;
   SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
 
-  int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1);
+  int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
   int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
   tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
            " ms, current:%d, app current:%d",
-           pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst);
+           pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
   releaseTscObj(pTscObj->id);
 }
 
@@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
   return pObj;
 }
 
-STscObj *acquireTscObj(int64_t rid) {
-  return (STscObj *)taosAcquireRef(clientConnRefPool, rid);
-}
+STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); }
 
-int32_t releaseTscObj(int64_t rid) {
-  return taosReleaseRef(clientConnRefPool, rid);
-}
+int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }
 
 void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
   assert(pObj != NULL);
@@ -186,6 +182,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
   pRequest->pTscObj = pObj;
   pRequest->body.fp = fp;  // not used it yet
   pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
+  pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
@@ -193,7 +190,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
   return pRequest;
 }
 
-static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
+void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   taosMemoryFreeClear(pResInfo->pRspMsg);
   taosMemoryFreeClear(pResInfo->length);
   taosMemoryFreeClear(pResInfo->row);
@@ -243,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) {
   taosRemoveRef(clientReqRefPool, pRequest->self);
 }
 
-SRequestObj *acquireRequest(int64_t rid) {
-  return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid);
-}
-
-int32_t releaseRequest(int64_t rid) {
-  return taosReleaseRef(clientReqRefPool, rid);
-}
+SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
 
+int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
 
 void taos_init_imp(void) {
   // In the APIs of other program language, taos_cleanup is not available yet.

source/client/src/clientImpl.c

@@ -95,16 +95,16 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
     if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
       return NULL;
     }
-
-    if (port) {
-      epSet.epSet.eps[0].port = port;
-    }
   } else {
     if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
       return NULL;
     }
   }
 
+  if (port) {
+    epSet.epSet.eps[0].port = port;
+  }
+
   char*          key = getClusterKey(user, secretEncrypt, ip, port);
   SAppInstInfo** pInst = NULL;
 
@@ -146,7 +146,8 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   (*pRequest)->sqlstr[sqlLen] = 0;
   (*pRequest)->sqlLen = sqlLen;
 
-  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) {
+  if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
+                  sizeof((*pRequest)->self))) {
     destroyRequest(*pRequest);
     *pRequest = NULL;
     tscError("put request to request hash failed");
@@ -157,7 +158,7 @@ int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
+int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb) {
   STscObj* pTscObj = pRequest->pTscObj;
 
   SParseContext cxt = {
@@ -170,6 +171,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
       .pMsg = pRequest->msgBuf,
       .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
       .pTransporter = pTscObj->pAppInfo->pTransporter,
+      .pStmtCb = pStmtCb,
   };
 
   cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
@@ -224,14 +226,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
   pRequest->type = pQuery->msgType;
-  SPlanContext cxt = {.queryId = pRequest->requestId,
+  SPlanContext cxt = {
+    .queryId = pRequest->requestId,
     .acctId = pRequest->pTscObj->acctId,
     .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
     .pAstRoot = pQuery->pRoot,
-                      .showRewrite = pQuery->showRewrite};
-  int32_t      code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
-  if (code != 0) {
-    return code;
+    .showRewrite = pQuery->showRewrite,
+    .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter
+  };
+  int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
   }
   return code;
 }
@@ -262,7 +267,8 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
 }
 
 void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) {
-  if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO && precision != TSDB_TIME_PRECISION_NANO) {
+  if (precision != TSDB_TIME_PRECISION_MILLI && precision != TSDB_TIME_PRECISION_MICRO &&
+      precision != TSDB_TIME_PRECISION_NANO) {
     return;
   }
 
@@ -298,16 +304,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
   return pRequest->code;
 }
 
-SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
-  SRequestObj* pRequest = NULL;
-  SQuery*      pQuery = NULL;
-  SArray*      pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
-
-  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = parseSql(pRequest, false, &pQuery);
-  }
-
+SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     switch (pQuery->execMode) {
       case QUERY_EXEC_MODE_LOCAL:
@@ -316,12 +313,15 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
       case QUERY_EXEC_MODE_RPC:
         code = execDdlQuery(pRequest, pQuery);
         break;
-      case QUERY_EXEC_MODE_SCHEDULE:
+      case QUERY_EXEC_MODE_SCHEDULE: {
+        SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
         code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
         if (TSDB_CODE_SUCCESS == code) {
           code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
         }
+        taosArrayDestroy(pNodeList);
         break;
+      }
       case QUERY_EXEC_MODE_EMPTY_RESULT:
         pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
         break;
@@ -330,8 +330,10 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
     }
   }
 
-  taosArrayDestroy(pNodeList);
+  if (!keepQuery) {
     qDestroyQuery(pQuery);
+  }
+
   if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
     pRequest->code = terrno;
   }
@@ -339,6 +341,18 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   return pRequest;
 }
 
+SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
+  SRequestObj* pRequest = NULL;
+  SQuery*      pQuery = NULL;
+
+  int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = parseSql(pRequest, false, &pQuery, NULL);
+  }
+
+  return launchQueryImpl(pRequest, pQuery, code, false);
+}
+
 int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
   SCatalog* pCatalog = NULL;
   int32_t   code = 0;
@@ -383,7 +397,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   int32_t      code = 0;
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
-    pRequest = execQueryImpl(pTscObj, sql, sqlLen);
+    pRequest = launchQuery(pTscObj, sql, sqlLen);
     if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
     }
@@ -425,7 +439,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
       return -1;
     }
 
-    taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return terrno;
+    }
+
     mgmtEpSet->numOfEps++;
   }
 
@@ -512,6 +531,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest, int8_t connType) {
   connectReq.pid = htonl(appInfo.pid);
   connectReq.startTime = htobe64(appInfo.startTime);
   tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app));
+  tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user));
+  tstrncpy(connectReq.passwd, pObj->pass, sizeof(connectReq.passwd));
 
   int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
   void*   pReq = taosMemoryMalloc(contLen);
@@ -715,6 +736,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
 
           int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p));
           ASSERT(len <= bytes);
+          ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i]));
 
           varDataSetLen(p, len);
           pCol->offset[j] = (p - pResultInfo->convertBuf[i]);
@@ -725,6 +747,80 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
       pResultInfo->row[i] = pResultInfo->pCol[i].pData;
     }
+
+    if (type == TSDB_DATA_TYPE_JSON) {
+      char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
+      if (p == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+
+      pResultInfo->convertBuf[i] = p;
+      int32_t        len = 0;
+      SResultColumn* pCol = &pResultInfo->pCol[i];
+      for (int32_t j = 0; j < numOfRows; ++j) {
+        if (pCol->offset[j] != -1) {
+          char* pStart = pCol->offset[j] + pCol->pData;
+
+          int32_t jsonInnerType = *pStart;
+          char*   jsonInnerData = pStart + CHAR_BYTES;
+          char    dst[TSDB_MAX_JSON_TAG_LEN] = {0};
+          if (jsonInnerType == TSDB_DATA_TYPE_NULL) {
+            sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) {
+            int32_t length =
+                taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst));
+
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                       varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length);
+          } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) {  // value -> "value"
+            *(char*)varDataVal(dst) = '\"';
+            int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
+                                           varDataVal(dst) + CHAR_BYTES);
+            if (length <= 0) {
+              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+                       varDataVal(jsonInnerData));
+              length = 0;
+            }
+            varDataSetLen(dst, length + CHAR_BYTES * 2);
+            *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"';
+          } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) {
+            double jsonVd = *(double*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%.9lf", jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) {
+            int64_t jsonVd = *(int64_t*)(jsonInnerData);
+            sprintf(varDataVal(dst), "%" PRId64, jsonVd);
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) {
+            sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false");
+            varDataSetLen(dst, strlen(varDataVal(dst)));
+          } else {
+            ASSERT(0);
+          }
+
+          if (len + varDataTLen(dst) > colLength[i]) {
+            p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst));
+            if (p == NULL) {
+              return TSDB_CODE_OUT_OF_MEMORY;
+            }
+
+            pResultInfo->convertBuf[i] = p;
+          }
+          p = pResultInfo->convertBuf[i] + len;
+          memcpy(p, dst, varDataTLen(dst));
+          pCol->offset[j] = len;
+          len += varDataTLen(dst);
+        }
+      }
+
+      pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/client/src/clientMain.c

@@ -14,12 +14,12 @@
  */
 
 #include "catalog.h"
-#include "scheduler.h"
 #include "clientInt.h"
-#include "clientStmt.h"
 #include "clientLog.h"
+#include "clientStmt.h"
 #include "os.h"
 #include "query.h"
+#include "scheduler.h"
 #include "tglobal.h"
 #include "tmsg.h"
 #include "tref.h"
@@ -128,9 +128,23 @@ const char *taos_errstr(TAOS_RES *res) {
 }
 
 void taos_free_result(TAOS_RES *res) {
+  if (NULL == res) {
+    return;
+  }
+  
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
     destroyRequest(pRequest);
+  } else if (TD_RES_TMQ(res)) {
+    SMqRspObj *pRsp = (SMqRspObj *)res;
+    if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
+    if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
+    if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema);
+    if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName);
+    if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
+    if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
+    pRsp->resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&pRsp->resInfo);
   }
 }
 
@@ -178,24 +192,23 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
 
   } else if (TD_RES_TMQ(res)) {
     SMqRspObj      *msg = ((SMqRspObj *)res);
-    if (msg->resIter == -1) msg->resIter++;
-    SReqResultInfo *pResultInfo = taosArrayGet(msg->res, msg->resIter);
+    SReqResultInfo *pResultInfo;
+    if (msg->resIter == -1) {
+      pResultInfo = tmqGetNextResInfo(res, true);
+    } else {
+      pResultInfo = tmqGetCurResInfo(res);
+    }
     if (pResultInfo->current < pResultInfo->numOfRows) {
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
     } else {
-      msg->resIter++;
-      if (msg->resIter < taosArrayGetSize(msg->res)) {
-        pResultInfo = taosArrayGet(msg->res, msg->resIter);
+      pResultInfo = tmqGetNextResInfo(res, true);
+      if (pResultInfo == NULL) return NULL;
       doSetOneRowPtr(pResultInfo);
       pResultInfo->current += 1;
       return pResultInfo->row;
-      } else {
-        return NULL;
-      }
     }
-
   } else {
     // assert to avoid un-initialization error
     ASSERT(0);
@@ -455,7 +468,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
     (*numOfRows) = pResultInfo->numOfRows;
     return pRequest->code;
   } else if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, true);
     if (pResultInfo == NULL) return -1;
 
     pResultInfo->current = pResultInfo->numOfRows;
@@ -474,7 +487,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
   }
 
   if (TD_RES_TMQ(res)) {
-    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res);
+    SReqResultInfo *pResultInfo = tmqGetNextResInfo(res, false);
     if (pResultInfo == NULL) {
       (*numOfRows) = 0;
       return 0;
@@ -580,84 +593,115 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) {
   return stmtInit(taos);
 }
 
-int taos_stmt_close(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
+int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
+  if (stmt == NULL || sql == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtClose(stmt);
+  return stmtPrepare(stmt, sql, length);
 }
 
-int taos_stmt_execute(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
+int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) {
+  if (stmt == NULL || name == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtExec(stmt);
+  int32_t code = stmtSetTbName(stmt, name);
+  if (code) {
+    return code;
+  }
+
+  if (tags) {
+    return stmtSetTbTags(stmt, tags);
+  }
+
+  return TSDB_CODE_SUCCESS;
 }
 
-char *taos_stmt_errstr(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
+int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
+  if (stmt == NULL || name == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
-    return NULL;
+    return terrno;
   }
 
-  return stmtErrstr(stmt);
+  return stmtSetTbName(stmt, name);
 }
 
-int taos_stmt_affected_rows(TAOS_STMT *stmt) {
-  if (stmt == NULL) {
+int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
+  return taos_stmt_set_tbname(stmt, name);
+}
+
+int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
+  if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
-    return 0;
+    return terrno;
   }
 
-  return stmtAffectedRows(stmt);
+  if (bind->num > 1) {
+    tscError("invalid bind number %d for %s", bind->num, __FUNCTION__);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  
+  return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) {
+int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtBind(stmt, bind);
+  if (bind->num <= 0 || bind->num > INT16_MAX) {
+    tscError("invalid bind num %d", bind->num);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+
+  return stmtBindBatch(stmt, bind, -1);
 }
 
-int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
-  if (stmt == NULL || sql == NULL) {
+int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) {
+  if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtPrepare(stmt, sql, length);
+  if (colIdx < 0) {
+    tscError("invalid bind column idx %d", colIdx);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  
+  return stmtBindBatch(stmt, bind, colIdx);
 }
 
-int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
-  if (stmt == NULL || name == NULL || tags == NULL) {
+int taos_stmt_add_batch(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtSetTbNameTags(stmt, name, tags);
+  return stmtAddBatch(stmt);
 }
 
-int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
-  if (stmt == NULL || name == NULL) {
+int taos_stmt_execute(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtSetTbNameTags(stmt, name, NULL);
+  return stmtExec(stmt);
 }
 
 int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
@@ -680,40 +724,42 @@ int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
   return stmtGetParamNum(stmt, nums);
 }
 
-int taos_stmt_add_batch(TAOS_STMT *stmt) {
+TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
   if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
-    return terrno;
+    return NULL;
   }
 
-  return stmtAddBatch(stmt);
+  return stmtUseResult(stmt);
 }
 
-TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
+char *taos_stmt_errstr(TAOS_STMT *stmt) {
+  return (char *)stmtErrstr(stmt);
+}
+
+int taos_stmt_affected_rows(TAOS_STMT *stmt) {
   if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
-    return NULL;
+    return 0;
   }
 
-  return stmtUseResult(stmt);
+  return stmtAffectedRows(stmt);
 }
 
-int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
-  if (stmt == NULL || bind == NULL) {
+int taos_stmt_close(TAOS_STMT *stmt) {
+  if (stmt == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);
     terrno = TSDB_CODE_INVALID_PARA;
     return terrno;
   }
 
-  return stmtBindBatch(stmt, bind);
+  return stmtClose(stmt);
 }
 
-
 TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
   // TODO
   return NULL;
 }
 
-

source/client/src/clientMsgHandler.c

@@ -151,13 +151,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     taosMemoryFreeClear(output.dbVgroup);
 
     tscError("failed to build use db output since %s", terrstr());
-  } else {
+  } else if (output.dbVgroup) {
     struct SCatalog* pCatalog = NULL;
 
     int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
     if (code1 != TSDB_CODE_SUCCESS) {
       tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
               tstrerror(code1));
+      taosMemoryFreeClear(output.dbVgroup);
     } else {
       catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
     }

source/client/src/clientStmt.c

@@ -4,86 +4,540 @@
 #include "clientStmt.h"
 #include "tdef.h"
 
-TAOS_STMT *stmtInit(TAOS *taos) {
-  STscObj* pObj = (STscObj*)taos;
-  STscStmt* pStmt = NULL;
+int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
+  switch (newStatus) {
+    case STMT_SETTBNAME:
+      break;
+    default:
+      break;
+  }
 
-#if 0
-  pStmt = taosMemoryCalloc(1, sizeof(STscStmt));
-  if (pStmt == NULL) {
+  //STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+
+  pStmt->sql.status = newStatus;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->sql.type = STMT_TYPE_MULTI_INSERT;
+  
+  if (NULL == pStmt->bInfo.tbName) {
+    tscError("no table name set");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_TBNAME_ERROR);
+  }
+
+  *tbName = pStmt->bInfo.tbName;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->bInfo.tbUid = pTableMeta->uid;
+  pStmt->bInfo.tbSuid = pTableMeta->suid;
+  pStmt->bInfo.tbType = pTableMeta->tableType;
+  pStmt->bInfo.boundTags = tags;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtSetExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  pStmt->exec.pVgHash = pVgHash;
+  pStmt->exec.pBlockHash = pBlockHash;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHash) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  *pVgHash = pStmt->exec.pVgHash;
+  *pBlockHash = pStmt->exec.pBlockHash;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCacheBlock(STscStmt *pStmt) {
+  if (pStmt->sql.type != STMT_TYPE_MULTI_INSERT) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  uint64_t uid;
+  if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) {
+    uid = pStmt->bInfo.tbSuid;
+  } else {
+    ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType);
+    uid = pStmt->bInfo.tbUid;
+  }
+
+  if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid));
+  STableDataBlocks* pDst = NULL;
+  
+  STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc));
+
+  SStmtTableCache cache = {
+    .pDataBlock = pDst,
+    .boundTags = pStmt->bInfo.boundTags,
+  };
+
+  if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pStmt->bInfo.boundTags = NULL;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtParseSql(STscStmt* pStmt) {
+  SStmtCallback stmtCb = {
+    .pStmt = pStmt, 
+    .getTbNameFn = stmtGetTbName, 
+    .setBindInfoFn = stmtSetBindInfo,
+    .setExecInfoFn = stmtSetExecInfo,
+    .getExecInfoFn = stmtGetExecInfo,
+  };
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+  
+  STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb));
+
+  pStmt->bInfo.needParse = false;
+  
+  switch (nodeType(pStmt->sql.pQuery->pRoot)) {
+    case QUERY_NODE_VNODE_MODIF_STMT:
+      if (0 == pStmt->sql.type) {
+        pStmt->sql.type = STMT_TYPE_INSERT;
+      }
+      break;
+    case QUERY_NODE_SELECT_STMT:
+      pStmt->sql.type = STMT_TYPE_QUERY;
+      break;
+    default:
+      tscError("not supported stmt type %d", nodeType(pStmt->sql.pQuery->pRoot));
+      STMT_ERR_RET(TSDB_CODE_TSC_STMT_CLAUSE_ERROR);
+  }
+
+  STMT_ERR_RET(stmtCacheBlock(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanBindInfo(STscStmt* pStmt) {
+  pStmt->bInfo.tbUid = 0;
+  pStmt->bInfo.tbSuid = 0;
+  pStmt->bInfo.tbType = 0;
+  pStmt->bInfo.needParse = true;
+
+  taosMemoryFreeClear(pStmt->bInfo.tbName);
+  destroyBoundColumnInfo(pStmt->bInfo.boundTags);
+  taosMemoryFreeClear(pStmt->bInfo.boundTags);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
+  taos_free_result(pStmt->exec.pRequest);
+  pStmt->exec.pRequest = NULL;
+
+  void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
+  while (pIter) {
+    STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter;    
+    uint64_t *key = taosHashGetKey(pIter, NULL);
+    
+    if (keepTable && (*key == pStmt->bInfo.tbUid)) {
+      STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true));
+      
+      pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter);
+      continue;
+    }
+
+    qFreeStmtDataBlock(pBlocks);
+    taosHashRemove(pStmt->exec.pBlockHash, key, sizeof(*key));
+
+    pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter);
+  }
+
+  if (keepTable) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  taosHashCleanup(pStmt->exec.pBlockHash);
+  pStmt->exec.pBlockHash = NULL;
+
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
+  taosMemoryFree(pStmt->sql.sqlStr);
+  qDestroyQuery(pStmt->sql.pQuery);
+
+  void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
+  while (pIter) {
+    SStmtTableCache* pCache = (SStmtTableCache*)pIter;    
+
+    qDestroyStmtDataBlock(pCache->pDataBlock);
+    destroyBoundColumnInfo(pCache->boundTags);
+    
+    pIter = taosHashIterate(pStmt->sql.pTableCache, pIter);
+  }
+  taosHashCleanup(pStmt->sql.pTableCache);
+  pStmt->sql.pTableCache = NULL;
+
+  memset(&pStmt->sql, 0, sizeof(pStmt->sql));
+
+  STMT_ERR_RET(stmtCleanExecInfo(pStmt, false));
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtGetFromCache(STscStmt* pStmt) {
+  pStmt->bInfo.needParse = true;
+
+  if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (NULL == pStmt->pCatalog) {
+    STMT_ERR_RET(catalogGetHandle(pStmt->taos->pAppInfo->clusterId, &pStmt->pCatalog));
+  }
+
+  STableMeta *pTableMeta = NULL;
+  SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
+  STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+
+  if (pTableMeta->uid == pStmt->bInfo.tbUid) {
+    pStmt->bInfo.needParse = false;
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) {
+    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+    if (NULL == pCache) {
+      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid);
+      STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
+    }
+    
+    pStmt->bInfo.needParse = false;
+    
+    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.boundTags = pCache->boundTags;
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  if (pCache) {
+    pStmt->bInfo.needParse = false;
+
+    pStmt->bInfo.tbUid = pTableMeta->uid;
+    pStmt->bInfo.tbSuid = pTableMeta->suid;
+    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.boundTags = pCache->boundTags;
+
+    STableDataBlocks* pNewBlock = NULL;
+    STMT_ERR_RET(qRebuildStmtDataBlock(&pNewBlock, pCache->pDataBlock));
+
+    if (taosHashPut(pStmt->exec.pBlockHash, &pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid), &pNewBlock, POINTER_BYTES)) {
+      STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+    
+    return TSDB_CODE_SUCCESS;
+  }
+
+  STMT_ERR_RET(stmtCleanBindInfo(pStmt));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t stmtResetStmt(STscStmt* pStmt) {
+  STMT_ERR_RET(stmtCleanSQLInfo(pStmt));
+
+  pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (NULL == pStmt->sql.pTableCache) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for statement");
-    return NULL;
+    STMT_ERR_RET(terrno);
   }
-  pStmt->taos = pObj;
 
-  SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
+  pStmt->sql.status = STMT_INIT;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+TAOS_STMT *stmtInit(TAOS *taos) {
+  STscObj* pObj = (STscObj*)taos;
+  STscStmt* pStmt = NULL;
 
-  if (pSql == NULL) {
-    free(pStmt);
+  pStmt = taosMemoryCalloc(1, sizeof(STscStmt));
+  if (NULL == pStmt) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to allocate memory for statement");
     return NULL;
   }
 
-  if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
-    free(pSql);
-    free(pStmt);
+  pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+  if (NULL == pStmt->sql.pTableCache) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
-    tscError("failed to malloc payload buffer");
+    taosMemoryFree(pStmt);
     return NULL;
   }
 
-  tsem_init(&pSql->rspSem, 0, 0);
-  pSql->signature   = pSql;
-  pSql->pTscObj     = pObj;
-  pSql->maxRetry    = TSDB_MAX_REPLICA;
-  pStmt->pSql       = pSql;
-  pStmt->last       = STMT_INIT;
-  pStmt->numOfRows  = 0;
-  registerSqlObj(pSql);
-#endif
+  pStmt->taos = pObj;
+  pStmt->bInfo.needParse = true;
+  pStmt->sql.status = STMT_INIT;
   
   return pStmt;
 }
 
-int stmtClose(TAOS_STMT *stmt) {
+int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (pStmt->sql.status >= STMT_PREPARE) {
+    STMT_ERR_RET(stmtResetStmt(pStmt));
+  }
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_PREPARE));
+
+  if (length <= 0) {
+    length = strlen(sql);
+  }
+  
+  pStmt->sql.sqlStr = strndup(sql, length);
+  pStmt->sql.sqlLen = length;
+
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtExec(TAOS_STMT *stmt) {
+
+int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+  
+  STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
+    
+  STMT_ERR_RET(stmtGetFromCache(pStmt));
+
+  if (pStmt->bInfo.needParse) {
+    taosMemoryFree(pStmt->bInfo.tbName);
+    pStmt->bInfo.tbName = strdup(tbName);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
-char *stmtErrstr(TAOS_STMT *stmt) {
-  return NULL;
+int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+  
+  STMT_ERR_RET(qBindStmtTagsValue(*pDataBlock, pStmt->bInfo.boundTags, pStmt->bInfo.tbSuid, &pStmt->bInfo.sname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen));
+
+  return TSDB_CODE_SUCCESS;
 }
 
-int stmtAffectedRows(TAOS_STMT *stmt) {
+
+int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    tscError("invalid operation to get query tag fileds");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields));
+
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtBind(TAOS_STMT *stmt, TAOS_BIND *bind) {
+int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS));
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  if (STMT_TYPE_QUERY == pStmt->sql.type) {
+    tscError("invalid operation to get query column fileds");
+    STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields));
+
   return TSDB_CODE_SUCCESS;  
 }
 
-int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
+int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
+
+  if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
+    pStmt->bInfo.needParse = false;
+  }
+
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+
+  if (pStmt->bInfo.needParse) {
+    STMT_ERR_RET(stmtParseSql(pStmt));
+  }
+
+  STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
+  if (NULL == pDataBlock) {
+    tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
+    STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+  }
+
+  if (colIdx < 0) {
+    qBindStmtColsValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen);
+  } else {
+    if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) {
+      tscError("bind column index not in sequence");
+      STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+    }
+
+    pStmt->bInfo.sBindLastIdx = colIdx;
+    
+    if (0 == colIdx) {
+      pStmt->bInfo.sBindRowNum = bind->num;
+    }
+    
+    qBindStmtSingleColValue(*pDataBlock, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, colIdx, pStmt->bInfo.sBindRowNum);
+  }
+  
   return TSDB_CODE_SUCCESS;
 }
 
-int stmtSetTbNameTags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
+
+int stmtAddBatch(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
+
+  STMT_ERR_RET(stmtCacheBlock(pStmt));
+  
   return TSDB_CODE_SUCCESS;
 }
 
+int stmtExec(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+  int32_t code = 0;
+
+  STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
+
+  STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
+
+  launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
+
+  STMT_ERR_JRET(pStmt->exec.pRequest->code);
+
+  pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest);
+  pStmt->affectedRows += pStmt->exec.affectedRows;
+
+_return:
+
+  stmtCleanExecInfo(pStmt, (code ? false : true));
+  
+  ++pStmt->sql.runTimes;
+  
+  STMT_RET(code);
+}
+
+
+int stmtClose(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  STMT_RET(stmtCleanSQLInfo(pStmt));
+}
+
+const char *stmtErrstr(TAOS_STMT *stmt) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (stmt == NULL) {
+    return (char*) tstrerror(terrno);
+  }
+
+  if (pStmt->exec.pRequest) {
+    pStmt->exec.pRequest->code = terrno;
+  }
+
+  return taos_errstr(pStmt->exec.pRequest);
+}
+
+int stmtAffectedRows(TAOS_STMT *stmt) {
+  return ((STscStmt*)stmt)->affectedRows;
+}
+
 int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
+  STscStmt* pStmt = (STscStmt*)stmt;
+
+  if (pStmt->sql.type) {
+    *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
+  } else {
+    *insert = isInsertSql(pStmt->sql.sqlStr, 0);
+  }
+  
   return TSDB_CODE_SUCCESS;
 }
 
 int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
-  return TSDB_CODE_SUCCESS;
-}
+  STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
   
-int stmtAddBatch(TAOS_STMT *stmt) {
   return TSDB_CODE_SUCCESS;
 }
 
@@ -91,9 +545,5 @@ TAOS_RES *stmtUseResult(TAOS_STMT *stmt) {
   return NULL;
 }
 
-int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
-  return TSDB_CODE_SUCCESS;
-}
-
 
 

source/client/test/clientTests.cpp

@@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select count(*) from tu");
+  pRes = taos_query(pConn, "select now() from m1");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/client/test/tmqTest.cpp

@@ -108,7 +108,7 @@ TEST(testCase, tmq_subscribe_ctb_Test) {
 
   while (1) {
     tmq_message_t* msg = tmq_consumer_poll(tmq, 1000);
-    tmq_message_destroy(msg);
+    taos_free_result(msg);
     //printf("get msg\n");
     //if (msg == NULL) break;
   }
@@ -141,7 +141,7 @@ TEST(testCase, tmq_subscribe_stb_Test) {
       tmq_commit(tmq, NULL, 0);
     }
     //tmq_commit(tmq, NULL, 0);
-    tmq_message_destroy(msg);
+    taos_free_result(msg);
     //printf("get msg\n");
   }
 }

source/common/CMakeLists.txt

@@ -5,8 +5,8 @@ target_include_directories(
     PUBLIC "${TD_SOURCE_DIR}/include/common"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 IF(${TD_WINDOWS})
-    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32"
-    PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex"
 ENDIF ()
 )
 target_link_libraries(

source/common/src/tdatablock.c

@@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
 
   if (pEp->port == 0) {
     pEp->port = tsServerPort;
-    return -1;
   }
 
   return 0;
@@ -113,14 +112,27 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
 
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if(type == TSDB_DATA_TYPE_JSON) {
+      if(*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData+CHAR_BYTES);
+      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
-    if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) {
+    if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
       if (newSize == 0) {
         newSize = 8;
       }
 
-      while (newSize < pAttr->length + varDataTLen(pData)) {
+      while (newSize < pAttr->length + dataLen) {
         newSize = newSize * 1.5;
       }
 
@@ -136,8 +148,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
     uint32_t len = pColumnInfoData->varmeta.length;
     pColumnInfoData->varmeta.offset[currentRow] = len;
 
-    memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData));
-    pColumnInfoData->varmeta.length += varDataTLen(pData);
+    memcpy(pColumnInfoData->pData + len, pData, dataLen);
+    pColumnInfoData->varmeta.length += dataLen;
   } else {
     memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes);
   }

source/common/src/tdataformat.c

@@ -110,7 +110,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
-    int8_t      sma = TSDB_BSMA_TYPE_NONE;
+    int8_t      sma = 0;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);

source/common/src/tglobal.c

@@ -76,6 +76,9 @@ int32_t  tsTelemInterval = 86400;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
 uint16_t tsTelemPort = 80;
 
+// query
+int32_t tsQueryPolicy = 1;
+
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,
  * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@@ -277,6 +280,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "fnDebugFlag", fnDebugFlag, 0, 255, 0) != 0) return -1;
   return 0;
 }
 
@@ -300,6 +304,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 4;
   tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
@@ -452,6 +457,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
   tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
   fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
+  fnDebugFlag = cfgGetItem(pCfg, "fnDebugFlag")->i32;
 }
 
 static int32_t taosSetClientCfg(SConfig *pCfg) {
@@ -484,6 +490,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
   tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
+  tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   return 0;
 }
 

source/common/src/tmsg.c

@@ -114,6 +114,19 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
   return 0;
 }
 
+int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) {
+  if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
+  if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1;
+  return 0;
+}
+
+
 int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI8(buf, pEp->inUse);
@@ -125,14 +138,14 @@ int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
   return tlen;
 }
 
-void *taosDecodeSEpSet(void *buf, SEpSet *pEp) {
+void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) {
   buf = taosDecodeFixedI8(buf, &pEp->inUse);
   buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
     buf = taosDecodeFixedU16(buf, &pEp->eps[i].port);
     buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn);
   }
-  return buf;
+  return (void *)buf;
 }
 
 static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) {
@@ -434,6 +447,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
+        tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
+        if (param->qmsg1Len > 0) {
+          tlen += taosEncodeString(buf, param->qmsg1);
+        }
+
+        tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
+        if (param->qmsg2Len > 0) {
+          tlen += taosEncodeString(buf, param->qmsg2);
+        }
       }
       break;
     case TD_CHILD_TABLE:
@@ -496,18 +518,25 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
       if (pReq->rollup) {
-        pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
+        pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
-          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
+          param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
-        } else {
-          param->pFuncIds = NULL;
+        }
+        buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
+        if (param->qmsg1Len > 0) {
+          buf = taosDecodeString(buf, &param->qmsg1);
+        }
+
+        buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
+        if (param->qmsg2Len > 0) {
+          buf = taosDecodeString(buf, &param->qmsg2);
         }
       } else {
         pReq->stbCfg.pRSmaParam = NULL;
@@ -828,6 +857,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) {
   taosArrayDestroy(pReq->pFields);
   pReq->pFields = NULL;
 }
+int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
 
 int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
   SCoder encoder = {0};
@@ -1624,6 +1674,7 @@ int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->ignoreCodeComment) < 0) return -1;
 
   if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1;
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
@@ -1644,6 +1695,7 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
+  if (tDecodeI8(&decoder, (int8_t *)&pReq->ignoreCodeComment) < 0) return -1;
 
   pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN);
   if (pReq->pFuncNames == NULL) return -1;
@@ -1681,9 +1733,13 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
     if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1;
     if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
     if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
+    if (pInfo->codeSize) {
       if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
+    }
+    if (pInfo->commentSize) {
       if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
     }
+  }
 
   tEndEncode(&encoder);
 
@@ -1713,15 +1769,23 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
     if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1;
     if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1;
     if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1;
+    if (fInfo.codeSize) {
       fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize);
-    fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
-    if (fInfo.pCode == NULL || fInfo.pComment == NULL) {
+      if (fInfo.pCode == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         return -1;
       }
-
       if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1;
+    }
+    if (fInfo.commentSize) {
+      fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
+      if (fInfo.pComment == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
       if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1;
+    }
+
     taosArrayPush(pRsp->pFuncInfos, &fInfo);
   }
   tEndDecode(&decoder);
@@ -1730,12 +1794,20 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
   return 0;
 }
 
+void tFreeSFuncInfo(SFuncInfo *pInfo) {
+  if (NULL == pInfo) {
+    return;
+  }
+
+  taosMemoryFree(pInfo->pCode);
+  taosMemoryFree(pInfo->pComment);
+}
+
 void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) {
   int32_t size = taosArrayGetSize(pRsp->pFuncInfos);
   for (int32_t i = 0; i < size; ++i) {
     SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i);
-    taosMemoryFree(pInfo->pCode);
-    taosMemoryFree(pInfo->pComment);
+    tFreeSFuncInfo(pInfo);
   }
   taosArrayDestroy(pRsp->pFuncInfos);
 }
@@ -1999,11 +2071,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp)
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  int32_t num = taosArrayGetSize(pRsp->epSetList);
+  int32_t num = taosArrayGetSize(pRsp->addrsList);
   if (tEncodeI32(&encoder, num) < 0) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    SEpSet *epSet = taosArrayGet(pRsp->epSetList, i);
-    if (tEncodeSEpSet(&encoder, epSet) < 0) return -1;
+    SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i);
+    if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1;
   }
   tEndEncode(&encoder);
 
@@ -2019,10 +2091,10 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   if (tStartDecode(&decoder) < 0) return -1;
   int32_t num = 0;
   if (tDecodeI32(&decoder, &num) < 0) return -1;
-  pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet));
-  if (NULL == pRsp->epSetList) return -1;
+  pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
+  if (NULL == pRsp->addrsList) return -1;
   for (int32_t i = 0; i < num; ++i) {
-    if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1;
+    if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
   }
   tEndDecode(&decoder);
 
@@ -2030,7 +2102,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
   return 0;
 }
 
-void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
+void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); }
 
 int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
   SCoder encoder = {0};
@@ -2403,7 +2475,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   tEndEncode(&encoder);
@@ -2419,7 +2490,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   tEndDecode(&decoder);
@@ -2674,10 +2744,14 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
+  if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
 
   tEndEncode(&encoder);
 
@@ -2696,6 +2770,9 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
   if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
   if (tDecodeI32(&decoder, &astLen) < 0) return -1;
 
@@ -2709,6 +2786,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
     pReq->ast = taosMemoryCalloc(1, astLen + 1);
     if (pReq->ast == NULL) return -1;
     if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
+  } else {
+    if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
   }
 
   tEndDecode(&decoder);
@@ -2756,6 +2835,8 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tEncodeI32(&encoder, pReq->pid) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->app) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+  if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1;
   tEndEncode(&encoder);
 
@@ -2773,6 +2854,8 @@ int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->passwd) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1;
   tEndDecode(&decoder);
 
@@ -3032,7 +3115,6 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
   return 0;
 }
 
-
 int32_t tSerializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3085,7 +3167,6 @@ int32_t tDeserializeSAlterVnodeReq(void *buf, int32_t bufLen, SAlterVnodeReq *pR
   return 0;
 }
 
-
 int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3599,8 +3680,6 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 

source/common/src/trow.c

@@ -519,8 +519,9 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
     SKvRowIdx *pIdx = tdKvRowColIdxAt(pRow, rcol);
     int16_t    colIdx = -1;
     if (pIdx) {
-      colIdx = POINTER_DISTANCE(pRow->data, pIdx) / sizeof(SKvRowIdx);
+      colIdx = POINTER_DISTANCE(pIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx);
     }
+    TASSERT(colIdx >= 0);
     SCellVal sVal = {0};
     if (pIdx->colId == pDataCol->colId) {
       if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {

source/common/src/ttime.c

@@ -412,10 +412,10 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
 int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
   int32_t charLen = varDataLen(inputData);
   char *newColData;
-  if (type == TSDB_DATA_TYPE_BINARY) {
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
     newColData = taosMemoryCalloc(1,  charLen + 1);
     memcpy(newColData, varDataVal(inputData), charLen);
-    bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
+    bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;
@@ -429,7 +429,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
       return TSDB_CODE_FAILED;
     }
     newColData[len] = 0;
-    bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
+    bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
     if (ret != TSDB_CODE_SUCCESS) {
       taosMemoryFree(newColData);
       return ret;

source/common/src/ttypes.c

@@ -379,7 +379,7 @@ static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *
   *maxIndex = 0;
 }
 
-tDataTypeDescriptor tDataTypes[15] = {
+tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
     {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
     {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
     {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint,
@@ -402,6 +402,7 @@ tDataTypeDescriptor tDataTypes[15] = {
     {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
     {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint,
      getStatics_u64},
+    {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
 };
 
 char tTokenTypeSwitcher[13] = {

source/common/src/tvariant.c

@@ -118,7 +118,7 @@ void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type) {
     }
     case TSDB_DATA_TYPE_BINARY: {
       pVar->pz = strndup(z, n);
-      pVar->nLen = strRmquote(pVar->pz, n);
+      //pVar->nLen = strRmquote(pVar->pz, n);
       break;
     }
     case TSDB_DATA_TYPE_TIMESTAMP: {

source/common/test/commonTests.cpp

@@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) {
   printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
 
   SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
-  SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0};
+  SBlockOrderInfo order = { true,  TSDB_ORDER_ASC, 0, NULL };
   taosArrayPush(pOrderInfo, &order);
 
   blockDataSort(b, pOrderInfo);

source/dnode/mgmt/exe/dmMain.c

@@ -40,10 +40,12 @@ static void dmSetSignalHandle() {
   taosSetSignal(SIGTERM, dmStopDnode);
   taosSetSignal(SIGHUP, dmStopDnode);
   taosSetSignal(SIGINT, dmStopDnode);
-  taosSetSignal(SIGTSTP, dmStopDnode);
   taosSetSignal(SIGABRT, dmStopDnode);
   taosSetSignal(SIGBREAK, dmStopDnode);
+#ifndef WINDOWS
+  taosSetSignal(SIGTSTP, dmStopDnode);
   taosSetSignal(SIGQUIT, dmStopDnode);
+#endif
 
   if (!tsMultiProcess) {
   } else if (global.ntype == DNODE || global.ntype == NODE_END) {

source/dnode/mgmt/implement/src/dmExec.c

@@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
 
 void dmCloseNode(SMgmtWrapper *pWrapper) {
   dInfo("node:%s, start to close", pWrapper->name);
+  pWrapper->deployed = false;
 
   while (pWrapper->refCount > 0) {
     taosMsleep(10);

source/dnode/mgmt/implement/src/dmTransport.c

@@ -119,10 +119,10 @@ _OVER:
 }
 
 static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  SDnodeTrans  *pTrans = &pDnode->trans;
+  SDnodeTrans * pTrans = &pDnode->trans;
   tmsg_t        msgType = pMsg->msgType;
   bool          isReq = msgType & 1u;
-  SMsgHandle   *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
+  SMsgHandle *  pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)];
   SMgmtWrapper *pWrapper = pHandle->pNdWrapper;
 
   if (msgType == TDMT_DND_SERVER_STATUS) {
@@ -449,7 +449,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s
   SAuthReq authReq = {0};
   tstrncpy(authReq.user, user, TSDB_USER_LEN);
   int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void   *pReq = rpcMallocCont(contLen);
+  void *  pReq = rpcMallocCont(contLen);
   tSerializeSAuthReq(pReq, contLen, &authReq);
 
   SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};

source/dnode/mgmt/interface/inc/dmDef.h

@@ -16,6 +16,7 @@
 #ifndef _TD_DM_DEF_H_
 #define _TD_DM_DEF_H_
 
+#include "uv.h"
 #include "dmLog.h"
 
 #include "cJSON.h"
@@ -142,6 +143,19 @@ typedef struct {
   char desc[TSDB_STEP_DESC_LEN];
 } SStartupInfo;
 
+typedef struct SUdfdData {
+  bool          startCalled;
+  bool          needCleanUp;
+  uv_loop_t     loop;
+  uv_thread_t   thread;
+  uv_barrier_t  barrier;
+  uv_process_t  process;
+  int           spawnErr;
+  uv_pipe_t     ctrlPipe;
+  uv_async_t    stopAsync;
+  int32_t        stopCalled;
+} SUdfdData;
+
 typedef struct SDnode {
   EDndProcType  ptype;
   EDndNodeType  ntype;
@@ -150,6 +164,7 @@ typedef struct SDnode {
   SStartupInfo  startup;
   SDnodeTrans   trans;
   SDnodeData    data;
+  SUdfdData     udfdData;
   TdThreadMutex mutex;
   SMgmtWrapper  wrappers[NODE_END];
 } SDnode;

source/dnode/mgmt/interface/src/dmEnv.c

@@ -53,6 +53,7 @@ void dmCleanup() {
   }
 
   monCleanup();
+  syncCleanUp();
   walCleanUp();
   taosStopCacheRefreshWorker();
   dInfo("dnode env is cleaned up");

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -211,15 +211,13 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_DB_CFG, mmProcessReadMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
 
   // Requests handled by VNODE
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -105,6 +105,7 @@ void    vmStopWorker(SVnodesMgmt *pMgmt);
 int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 void    vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);  // sync integration
 int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) {
   }
 
   SRetrieveTableReq retrieveReq = {0};
-  retrieveReq.type = showType;
   strcpy(retrieveReq.db, db);
   strcpy(retrieveReq.tb, tb);