Merge remote-tracking branch 'origin/3.0' into feature/async

Jenkinsfile2

@@ -33,16 +33,17 @@ def abort_previous(){
     milestone(buildNumber)
 }
 def pre_test(){
-    sh 'hostname'
     sh '''
+        hostname
         date
-        sudo rmtaos || echo "taosd has not installed"
     '''
     sh '''
-        killall -9 taosd ||echo "no taosd running"
-        killall -9 gdb || echo "no gdb running"
-        killall -9 python3.8 || echo "no python program running"
+        cd ${WK}
+        git reset --hard
+        git fetch || git fetch
         cd ${WKC}
+        git reset --hard
+        git fetch || git fetch
     '''
     script {
         if (env.CHANGE_TARGET == 'master') {
@@ -81,10 +82,10 @@ def pre_test(){
             git pull >/dev/null
             git fetch origin +refs/pull/${CHANGE_ID}/merge
             git checkout -qf FETCH_HEAD
-            git log|head -n20
+            git log -5
             cd ${WK}
             git pull >/dev/null
-            git log|head -n20
+            git log -5
         '''
     } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
         sh '''
@@ -92,10 +93,10 @@ def pre_test(){
             git pull >/dev/null
             git fetch origin +refs/pull/${CHANGE_ID}/merge
             git checkout -qf FETCH_HEAD
-            git log|head -n20
+            git log -5
             cd ${WKC}
             git pull >/dev/null
-            git log|head -n20
+            git log -5
         '''
     } else {
         sh '''
@@ -106,83 +107,143 @@ def pre_test(){
         cd ${WKC}
         git submodule update --init --recursive
     '''
-    sh '''
-        cd ${WK}
-        export TZ=Asia/Harbin
-        date
-        rm -rf debug
-        mkdir debug
-        cd debug
-        cmake .. > /dev/null
-        make -j4> /dev/null
-    '''
     sh '''
         cd ${WKPY}
         git reset --hard
         git pull
-        pip3 install .
     '''
     return 1
 }
 def pre_test_win(){
     bat '''
         hostname
+        ipconfig
+        set
         date /t
         time /t
-        taskkill /f /t /im python.exe
-        taskkill /f /t /im bash.exe
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
-        rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine\\debug
-        exit 0
+        rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug || exit 0
+    '''
+    bat '''
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        git reset --hard
+        git fetch || git fetch
     '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
         git reset --hard
         git fetch || git fetch
-        git checkout -f
     '''
     script {
         if (env.CHANGE_TARGET == 'master') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout master
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
                 git checkout master
             '''
         } else if(env.CHANGE_TARGET == '2.0') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout 2.0
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
                 git checkout 2.0
             '''
         } else if(env.CHANGE_TARGET == '3.0') {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout 3.0
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
                 git checkout 3.0
             '''
         } else {
             bat '''
-                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout develop
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
                 git checkout develop
             '''
         }
     }
+    script {
+        if (env.CHANGE_URL =~ /\/TDengine\//) {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git pull
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git pull
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git fetch origin +refs/pull/%CHANGE_ID%/merge
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git checkout -qf FETCH_HEAD
+            '''
+        } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git pull
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git fetch origin +refs/pull/%CHANGE_ID%/merge
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+                git checkout -qf FETCH_HEAD
+            '''
+            bat '''
+                cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+                git pull
+            '''
+        } else {
+            bat '''
+                echo "unmatched reposiotry %CHANGE_URL%"
+            '''
+        }
+    }
+    bat '''
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
+        git branch
+        git log -5
+    '''
     bat '''
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
         git branch
-        git pull || git pull
-        git fetch origin +refs/pull/%CHANGE_ID%/merge
-        git checkout -qf FETCH_HEAD
+        git log -5
+    '''
+    bat '''
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community
+        git submodule update --init --recursive
     '''
 }
 def pre_test_build_win() {
     bat '''
         echo "building ..."
         time /t
-        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine
+        cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal
         mkdir debug
         cd debug
+        time /t
         call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64
         set CL=/MP8
-        cmake .. -G "NMake Makefiles JOM"
-        jom -j 4 || exit 8
+        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
+        time /t
+        cmake .. -G "NMake Makefiles JOM" || exit 7
+        echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
+        time /t
+        jom -j 6 || exit 8
         time /t
     '''
     return 1
@@ -192,6 +253,7 @@ pipeline {
     agent none
     options { skipDefaultCheckout() } 
     environment{
+        WKDIR = '/var/lib/jenkins/workspace'
         WK = '/var/lib/jenkins/workspace/TDinternal'
         WKC = '/var/lib/jenkins/workspace/TDinternal/community'
         WKPY = '/var/lib/jenkins/workspace/taos-connector-python'
@@ -199,6 +261,13 @@ pipeline {
     stages {
         stage('run test') {
             parallel {
+                stage('windows test') {
+                    agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "}
+                    steps {
+                        pre_test_win()
+                        pre_test_build_win()
+                    }
+                }
                 stage('linux test') {
                     agent{label " slave3_0 || slave15 || slave16 || slave17 "}
                     options { skipDefaultCheckout() } 
@@ -206,39 +275,23 @@ pipeline {
                         changeRequest()
                     }
                     steps {
-                        timeout(time: 45, unit: 'MINUTES'){
+                        timeout(time: 40, unit: 'MINUTES'){
                             pre_test()
                             script {
-                                if (env.CHANGE_URL =~ /\/TDengine\//) {
-                                    sh '''
-                                        cd ${WK}/debug
-                                        ctest -VV
-                                    '''
-                                    sh '''
-                                        export LD_LIBRARY_PATH=${WK}/debug/build/lib
-                                        cd ${WKC}/tests/system-test
-                                        ./fulltest.sh
-                                    '''
-                                } else if (env.CHANGE_URL =~ /\/TDinternal\//) {
-                                    sh '''
-                                        cd ${WKC}/debug
-                                        ctest -VV
-                                    '''
-                                    sh '''
-                                        export LD_LIBRARY_PATH=${WKC}/debug/build/lib
-                                        cd ${WKC}/tests/system-test
-                                        ./fulltest.sh
-                                    '''
-                                } else {
-                                    sh '''
-                                        echo "unmatched reposiotry ${CHANGE_URL}"
-                                    '''
-                                }
+                                sh '''
+                                    cd ${WKC}/tests/parallel_test
+                                    date
+                                    time ./container_build.sh -w ${WKDIR} -t 8 -e
+                                    rm -f /tmp/cases.task
+                                    ./collect_cases.sh -e
+                                '''
+                                sh '''
+                                    cd ${WKC}/tests/parallel_test
+                                    export DEFAULT_RETRY_TIME=2
+                                    date
+                                    timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME} -l ${WKDIR}/log -o 480
+                                '''
                             }
-                            sh '''
-                                cd ${WKC}/tests
-                                ./test-all.sh b1fq
-                            '''
                         }
                     }
                 }

cmake/cmake.define

@@ -14,25 +14,6 @@ MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
 MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
 
-find_package(Git QUIET)
-if(GIT_FOUND AND EXISTS "${TD_SOURCE_DIR}/.git")
-# Update submodules as needed
-    option(GIT_SUBMODULE "Check submodules during build" ON)
-    if(GIT_SUBMODULE)
-        message(STATUS "Submodule update")
-        execute_process(COMMAND cd ${TD_SOURCE_DIR} && ${GIT_EXECUTABLE} submodule update --init --recursive
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-                        RESULT_VARIABLE GIT_SUBMOD_RESULT)
-        if(NOT GIT_SUBMOD_RESULT EQUAL "0")
-            message(WARNING "git submodule update --init --recursive failed with ${GIT_SUBMOD_RESULT}, please checkout submodules")
-        endif()
-    endif()
-endif()
-
-if(NOT EXISTS "${TD_SOURCE_DIR}/tools/taos-tools/CMakeLists.txt")
-    message(WARNING "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules manually if you need build them.")
-endif()
-
 if (NOT DEFINED TD_GRANT)
   SET(TD_GRANT FALSE)
 endif()

cmake/cmake.install

 IF (TD_LINUX)
-  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/make_install.sh")
+  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh")
   INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
   INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})")
 ELSEIF (TD_WINDOWS)
   SET(CMAKE_INSTALL_PREFIX C:/TDengine)
 
-  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go     DESTINATION connector)
-  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector)
-  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector)
-  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector)
-  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .)
+  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go     DESTINATION connector)
+  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector)
+  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector)
+  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector)
+  # INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .)
   INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg)
-  INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include)
-  INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include)
+  INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include)
+  INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver)
-  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
   INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
+  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .)
+  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosd.exe DESTINATION .)
+  INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/udfd.exe DESTINATION .)
 
   IF (TD_MVN_INSTALLED)
     INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
+  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat")
+  INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
+  INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})")
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh")
   INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")

cmake/cmake.options

@@ -78,6 +78,12 @@ option(
     OFF
 )
 
+option(
+    BUILD_WITH_BDB
+    "If build with BDB" 
+    OFF
+)
+
 option(
     BUILD_WITH_LUCENE 
     "If build with lucene" 

contrib/CMakeLists.txt

@@ -78,9 +78,9 @@ if(${BUILD_WITH_UV})
 endif(${BUILD_WITH_UV})
 
 # bdb
-#if(${BUILD_WITH_BDB})
-    #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-#endif(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
+    cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})
@@ -226,10 +226,10 @@ endif(${BUILD_WITH_NURAFT})
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE release)
     add_definitions(-DPTW32_STATIC_LIB)
-    add_subdirectory(pthread)
+    add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
-    add_library(pthread STATIC IMPORTED GLOBAL)
-    SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
+    add_library(pthread INTERFACE)
+    target_link_libraries(pthread INTERFACE libpthreadVC3)
 endif()
 
 # iconv
@@ -365,7 +365,7 @@ if(${BUILD_ADDR2LINE})
     if(HAVE_LIBELF_H OR HAVE_LIBELF_LIBELF_H)
         target_link_libraries(libdwarf PUBLIC libelf)
     endif()
-    target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_BINARY_DIR}/contrib)
+    target_include_directories(libdwarf SYSTEM PUBLIC "libdwarf/src/lib/libdwarf" ${CMAKE_CURRENT_BINARY_DIR})
     file(READ "addr2line/addr2line.c" ADDR2LINE_CONTENT)
     string(REPLACE "static int" "int" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")
     string(REPLACE "static void" "void" ADDR2LINE_CONTENT "${ADDR2LINE_CONTENT}")

contrib/test/CMakeLists.txt

@@ -7,9 +7,9 @@ if(${BUILD_WITH_LUCENE})
     add_subdirectory(lucene)
 endif(${BUILD_WITH_LUCENE})
 
-#if(${BUILD_WITH_BDB})
-    #add_subdirectory(bdb)
-#endif(${BUILD_WITH_BDB})
+if(${BUILD_WITH_BDB})
+    add_subdirectory(bdb)
+endif(${BUILD_WITH_BDB})
 
 if(${BUILD_WITH_SQLITE})
     add_subdirectory(sqlite)

example/src/tmq.c

@@ -61,7 +61,7 @@ int32_t init_env() {
   taos_free_result(pRes);
 
   pRes =
-      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
+      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -106,8 +106,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
-  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
+  pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -167,7 +167,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
   tmq_conf_set(conf, "msg.with.table.name", "true");
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   assert(tmq);
   return tmq;
@@ -176,6 +176,7 @@ tmq_t* build_consumer() {
 tmq_list_t* build_topic_list() {
   tmq_list_t* topic_list = tmq_list_new();
   tmq_list_append(topic_list, "topic_ctb_column");
+  /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/
   return topic_list;
 }
 
@@ -190,7 +191,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
   int32_t cnt = 0;
   /*clock_t startTime = clock();*/
   while (running) {
-    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 500);
+    TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0);
     if (tmqmessage) {
       cnt++;
       /*printf("get data\n");*/
@@ -238,7 +239,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
       msg_process(tmqmessage);
       taos_free_result(tmqmessage);
 
-      tmq_commit(tmq, NULL, 1);
+      tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL);
       /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/
     }
   }

include/client/taos.h

@@ -232,9 +232,11 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
 DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
-DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
+DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
+DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
+
 #if 0
-DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
+DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
 DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 #endif
 
@@ -251,7 +253,7 @@ typedef enum tmq_conf_res_t tmq_conf_res_t;
 DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
-DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
+DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);
 
 /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
 

include/common/taosdef.h

@@ -37,7 +37,8 @@ typedef enum {
   TSDB_STREAM_TABLE = 4,  // table created from stream computing
   TSDB_TEMP_TABLE = 5,    // temp table created by nest query
   TSDB_SYSTEM_TABLE = 6,
-  TSDB_TABLE_MAX = 7
+  TSDB_TSMA_TABLE = 7,  // time-range-wise sma
+  TSDB_TABLE_MAX = 8
 } ETableType;
 
 typedef enum {

include/common/tcommon.h

@@ -201,18 +201,17 @@ typedef struct SExprInfo {
 } SExprInfo;
 
 typedef struct {
-  const char* key;
-  int32_t keyLen;
-  uint8_t type;
-  int16_t length;
+  const char*   key;
+  int32_t       keyLen;
+  uint8_t       type;
   union{
     const char* value;
-    int64_t  i;
-    uint64_t u;
-    double   d;
-    float    f;
+    int64_t     i;
+    uint64_t    u;
+    double      d;
+    float       f;
   };
-  int32_t valueLen;
+  int32_t       length;
 } SSmlKv;
 
 #define QUERY_ASC_FORWARD_STEP  1

include/common/tdataformat.h

@@ -29,27 +29,42 @@ extern "C" {
 typedef struct SSchema       SSchema;
 typedef struct STColumn      STColumn;
 typedef struct STSchema      STSchema;
+typedef struct SColVal       SColVal;
 typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
-typedef struct SKVIdx        SKVIdx;
-
-// STSchema
-
-// STSRow2
-int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
-int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+typedef struct STagVal       STagVal;
+typedef struct STag          STag;
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
+// SColVal
+#define ColValNONE               ((SColVal){.type = COL_VAL_NONE, .nData = 0, .pData = NULL})
+#define ColValNULL               ((SColVal){.type = COL_VAL_NULL, .nData = 0, .pData = NULL})
+#define ColValDATA(nData, pData) ((SColVal){.type = COL_VAL_DATA, .nData = (nData), .pData = (pData)})
+
+// STSRow2
+int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow);
+int32_t tTSRowDup(const STSRow2 *pRow, STSRow2 **ppRow);
+void    tTSRowFree(STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
+
 // STSRowBuilder
-int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols);
+int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema);
 void    tTSRowBuilderClear(STSRowBuilder *pBuilder);
 void    tTSRowBuilderReset(STSRowBuilder *pBuilder);
-int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData);
+int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData);
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow);
 
+// STag
+int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag);
+void    tTagFree(STag *pTag);
+void    tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, int32_t *nData);
+int32_t tEncodeTag(SEncoder *pEncoder, STag *pTag);
+int32_t tDecodeTag(SDecoder *pDecoder, const STag **ppTag);
+
 // STRUCT =================
 struct STColumn {
   col_id_t colId;
@@ -68,31 +83,47 @@ struct STSchema {
   STColumn columns[];
 };
 
+#define TSROW_HAS_NONE ((uint8_t)0x1)
+#define TSROW_HAS_NULL ((uint8_t)0x2U)
+#define TSROW_HAS_VAL  ((uint8_t)0x4U)
+#define TSROW_KV_ROW   ((uint8_t)0x10U)
 struct STSRow2 {
   TSKEY    ts;
-  uint32_t flags;
-  union {
-    int32_t sver;
-    int32_t ncols;
-  };
-  uint32_t       nData;
-  const uint8_t *pData;
+  uint8_t  flags;
+  int32_t  sver;
+  uint32_t nData;
+  uint8_t *pData;
 };
 
 struct STSRowBuilder {
-  STColumn *pTColumn;
   STSchema *pTSchema;
+  int32_t   szBitMap1;
+  int32_t   szBitMap2;
   int32_t   szKVBuf;
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
-  int32_t   nCols;
-  int32_t   kvVLen;
-  int32_t   tpVLen;
+  int32_t   iCol;
+  int32_t   vlenKV;
+  int32_t   vlenTP;
   STSRow2   row;
 };
 
-#if 1  //====================================
+typedef enum { COL_VAL_NONE = 0, COL_VAL_NULL = 1, COL_VAL_DATA = 2 } EColValT;
+struct SColVal {
+  EColValT type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+struct STagVal {
+  int16_t  cid;
+  int8_t   type;
+  uint32_t nData;
+  uint8_t *pData;
+};
+
+#if 1  //================================================================================================================================================
 // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
 #define TD_SUPPORT_BITMAP
 #define TD_SUPPORT_READ2

include/common/tglobal.h

@@ -32,9 +32,9 @@ extern char     tsLocalEp[];
 extern uint16_t tsServerPort;
 extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
+extern int32_t  tsNumOfSupportVnodes;
 
 // common
-extern int32_t tsMaxConnections;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
 extern int32_t tsCompressMsgSize;
@@ -46,12 +46,13 @@ extern bool    tsPrintAuth;
 extern int64_t tsTickPerMin[3];
 
 // multi-process
-extern bool    tsMultiProcess;
+extern int32_t tsMultiProcess;
 extern int32_t tsMnodeShmSize;
 extern int32_t tsVnodeShmSize;
 extern int32_t tsQnodeShmSize;
 extern int32_t tsSnodeShmSize;
 extern int32_t tsBnodeShmSize;
+extern int32_t tsNumOfShmThreads;
 
 // queue & threads
 extern int32_t tsNumOfRpcThreads;
@@ -68,6 +69,7 @@ extern int32_t tsNumOfQnodeQueryThreads;
 extern int32_t tsNumOfQnodeFetchThreads;
 extern int32_t tsNumOfSnodeSharedThreads;
 extern int32_t tsNumOfSnodeUniqueThreads;
+extern int64_t tsRpcQueueMemoryAllowed;
 
 // monitor
 extern bool     tsEnableMonitor;
@@ -124,6 +126,10 @@ extern SDiskCfg tsDiskCfg[];
 // udf
 extern bool tsStartUdfd;
 
+// schemaless
+extern char tsSmlChildTableName[];
+extern bool tsSmlDataFormat;
+
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;

include/common/tmsg.h

@@ -70,7 +70,7 @@ typedef uint16_t tmsg_t;
 #define TSDB_IE_TYPE_DNODE_EXT   6
 #define TSDB_IE_TYPE_DNODE_STATE 7
 
-enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX };
+enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX };
 
 enum {
   HEARTBEAT_KEY_USER_AUTHINFO = 1,
@@ -258,6 +258,7 @@ typedef struct {
   char*   tblFName;
   int32_t numOfRows;
   int32_t affectedRows;
+  int64_t sver;
 } SSubmitBlkRsp;
 
 typedef struct {
@@ -274,10 +275,10 @@ int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
-#define COL_SMA_ON  ((int8_t)0x1)
-#define COL_IDX_ON  ((int8_t)0x2)
-#define COL_VAL_SET ((int8_t)0x4)
-
+#define COL_SMA_ON   ((int8_t)0x1)
+#define COL_IDX_ON   ((int8_t)0x2)
+#define COL_SET_NULL ((int8_t)0x10)
+#define COL_SET_VAL  ((int8_t)0x20)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -286,6 +287,9 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+#define COL_IS_SET(FLG)  ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0)
+#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL)))
+
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
 #define SSCHMEA_TYPE(s)  ((s)->type)
@@ -297,6 +301,8 @@ typedef struct SSchema {
 typedef struct {
   int32_t  nCols;
   int32_t  sver;
+  int32_t  tagVer;
+  int32_t  colVer;
   SSchema* pSchema;
 } SSchemaWrapper;
 
@@ -305,6 +311,8 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p
   if (pSW == NULL) return pSW;
   pSW->nCols = pSchemaWrapper->nCols;
   pSW->sver = pSchemaWrapper->sver;
+  pSW->tagVer = pSchemaWrapper->tagVer;
+  pSW->colVer = pSchemaWrapper->colVer;
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) {
     taosMemoryFree(pSW);
@@ -360,6 +368,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
   int32_t tlen = 0;
   tlen += taosEncodeVariantI32(buf, pSW->nCols);
   tlen += taosEncodeVariantI32(buf, pSW->sver);
+  tlen += taosEncodeVariantI32(buf, pSW->tagVer);
+  tlen += taosEncodeVariantI32(buf, pSW->colVer);
   for (int32_t i = 0; i < pSW->nCols; i++) {
     tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]);
   }
@@ -369,6 +379,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
 static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
   buf = taosDecodeVariantI32(buf, &pSW->nCols);
   buf = taosDecodeVariantI32(buf, &pSW->sver);
+  buf = taosDecodeVariantI32(buf, &pSW->tagVer);
+  buf = taosDecodeVariantI32(buf, &pSW->colVer);
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) {
     return NULL;
@@ -383,6 +395,8 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
 static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSchemaWrapper* pSW) {
   if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1;
   if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pSW->tagVer) < 0) return -1;
+  if (tEncodeI32v(pEncoder, pSW->colVer) < 0) return -1;
   for (int32_t i = 0; i < pSW->nCols; i++) {
     if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1;
   }
@@ -393,6 +407,8 @@ static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SEncoder* pEncoder, const SSch
 static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWrapper* pSW) {
   if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
   if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1;
 
   pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
   if (pSW->pSchema == NULL) return -1;
@@ -403,6 +419,21 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SDecoder* pDecoder, SSchemaWra
   return 0;
 }
 
+static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaWrapper* pSW) {
+  if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->tagVer) < 0) return -1;
+  if (tDecodeI32v(pDecoder, &pSW->colVer) < 0) return -1;
+
+  pSW->pSchema = (SSchema*)tDecoderMalloc(pDecoder, pSW->nCols * sizeof(SSchema));
+  if (pSW->pSchema == NULL) return -1;
+  for (int32_t i = 0; i < pSW->nCols; i++) {
+    if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
+  }
+
+  return 0;
+}
+
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
 
 typedef struct {
@@ -438,6 +469,7 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  alterType;
+  int32_t verInBlock;
   int32_t numOfFields;
   SArray* pFields;
   int32_t ttl;
@@ -1463,6 +1495,7 @@ typedef struct {
 typedef struct {
   int64_t consumerId;
   char    cgroup[TSDB_CGROUP_LEN];
+  char    clientId[256];
   SArray* topicNames;  // SArray<char**>
 } SCMSubscribeReq;
 
@@ -1470,6 +1503,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pReq->consumerId);
   tlen += taosEncodeString(buf, pReq->cgroup);
+  tlen += taosEncodeString(buf, pReq->clientId);
 
   int32_t topicNum = taosArrayGetSize(pReq->topicNames);
   tlen += taosEncodeFixedI32(buf, topicNum);
@@ -1483,6 +1517,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
 static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
   buf = taosDecodeFixedI64(buf, &pReq->consumerId);
   buf = taosDecodeStringTo(buf, pReq->cgroup);
+  buf = taosDecodeStringTo(buf, pReq->clientId);
 
   int32_t topicNum;
   buf = taosDecodeFixedI32(buf, &topicNum);
@@ -1613,6 +1648,15 @@ typedef struct {
 int32_t tSerializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
 int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
 
+typedef struct {
+  char   topic[TSDB_TOPIC_FNAME_LEN];
+  char   cgroup[TSDB_CGROUP_LEN];
+  int8_t igNotExists;
+} SMDropCgroupReq;
+
+int32_t tSerializeSMDropCgroupReq(void* buf, int32_t bufLen, SMDropCgroupReq* pReq);
+int32_t tDeserializeSMDropCgroupReq(void* buf, int32_t bufLen, SMDropCgroupReq* pReq);
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  alterType;
@@ -1670,6 +1714,7 @@ typedef struct SVDropStbReq {
 int32_t tEncodeSVDropStbReq(SEncoder* pCoder, const SVDropStbReq* pReq);
 int32_t tDecodeSVDropStbReq(SDecoder* pCoder, SVDropStbReq* pReq);
 
+// TDMT_VND_CREATE_TABLE ==============
 #define TD_CREATE_IF_NOT_EXISTS 0x1
 typedef struct SVCreateTbReq {
   int32_t     flags;
@@ -1759,6 +1804,43 @@ typedef struct {
 int32_t tEncodeSVDropTbBatchRsp(SEncoder* pCoder, const SVDropTbBatchRsp* pRsp);
 int32_t tDecodeSVDropTbBatchRsp(SDecoder* pCoder, SVDropTbBatchRsp* pRsp);
 
+// TDMT_VND_ALTER_TABLE =====================
+typedef struct {
+  const char* tbName;
+  int8_t      action;
+  const char* colName;
+  // TSDB_ALTER_TABLE_ADD_COLUMN
+  int8_t  type;
+  int8_t  flags;
+  int32_t bytes;
+  // TSDB_ALTER_TABLE_DROP_COLUMN
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
+  int32_t colModBytes;
+  // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
+  const char* colNewName;
+  // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
+  const char*    tagName;
+  int8_t         isNull;
+  uint32_t       nTagVal;
+  const uint8_t* pTagVal;
+  // TSDB_ALTER_TABLE_UPDATE_OPTIONS
+  int8_t      updateTTL;
+  int32_t     newTTL;
+  int8_t      updateComment;
+  const char* newComment;
+} SVAlterTbReq;
+
+int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
+int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq);
+
+typedef struct {
+  int32_t code;
+} SVAlterTbRsp;
+
+int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp);
+int32_t tDecodeSVAlterTbRsp(SDecoder* pDecoder, SVAlterTbRsp* pRsp);
+// ======================
+
 typedef struct {
   SMsgHead head;
   int64_t  uid;
@@ -2059,6 +2141,18 @@ enum {
   TOPIC_SUB_TYPE__TABLE,
 };
 
+typedef struct {
+  SMsgHead head;
+  int64_t  leftForVer;
+  int32_t  vgId;
+  int64_t  consumerId;
+  char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+} SMqVDeleteReq;
+
+typedef struct {
+  int8_t reserved;
+} SMqVDeleteRsp;
+
 typedef struct {
   int64_t leftForVer;
   int32_t vgId;
@@ -2160,26 +2254,23 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
 int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq);
 
 typedef struct {
-  int8_t   version;       // for compatibility(default 0)
-  int8_t   intervalUnit;  // MACRO: TIME_UNIT_XXX
-  int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
-  int8_t   timezoneInt;   // sma data expired if timezone changes.
-  char     indexName[TSDB_INDEX_NAME_LEN];
-  int32_t  exprLen;
-  int32_t  tagsFilterLen;
-  int64_t  indexUid;
-  tb_uid_t tableUid;  // super/child/common table uid
-  int64_t  interval;
-  int64_t  offset;  // use unit by precision of DB
-  int64_t  sliding;
-  char*    expr;  // sma expression
-  char*    tagsFilter;
+  int8_t      version;       // for compatibility(default 0)
+  int8_t      intervalUnit;  // MACRO: TIME_UNIT_XXX
+  int8_t      slidingUnit;   // MACRO: TIME_UNIT_XXX
+  int8_t      timezoneInt;   // sma data expired if timezone changes.
+  char        indexName[TSDB_INDEX_NAME_LEN];
+  int32_t     exprLen;
+  int32_t     tagsFilterLen;
+  int64_t     indexUid;
+  tb_uid_t    tableUid;  // super/child/common table uid
+  int64_t     interval;
+  int64_t     offset;  // use unit by precision of DB
+  int64_t     sliding;
+  const char* expr;  // sma expression
+  const char* tagsFilter;
 } STSma;  // Time-range-wise SMA
 
-typedef struct {
-  int64_t ver;  // use a general definition
-  STSma   tSma;
-} SVCreateTSmaReq;
+typedef STSma SVCreateTSmaReq;
 
 typedef struct {
   int8_t  type;  // 0 status report, 1 update data
@@ -2188,7 +2279,6 @@ typedef struct {
 } STSmaMsg;
 
 typedef struct {
-  int64_t ver;  // use a general definition
   int64_t indexUid;
   char    indexName[TSDB_INDEX_NAME_LEN];
 } SVDropTSmaReq;
@@ -2197,28 +2287,21 @@ typedef struct {
   int tmp;  // TODO: to avoid compile error
 } SVCreateTSmaRsp, SVDropTSmaRsp;
 
+#if 0
 int32_t tSerializeSVCreateTSmaReq(void** buf, SVCreateTSmaReq* pReq);
 void*   tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
 int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
 void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
+#endif
 
-// RSma: Rollup SMA
-typedef struct {
-  int64_t  interval;
-  int32_t  retention;  // unit: day
-  uint16_t days;       // unit: day
-  int8_t   intervalUnit;
-} SSmaParams;
-
-typedef struct {
-  STSma   tsma;
-  float   xFilesFactor;
-  SArray* smaParams;  // SSmaParams
-} SRSma;
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
+int32_t tEncodeSVDropTSmaReq(SEncoder* pCoder, const SVDropTSmaReq* pReq);
+int32_t tDecodeSVDropTSmaReq(SDecoder* pCoder, SVDropTSmaReq* pReq);
 
 typedef struct {
-  uint32_t number;
-  STSma*   tSma;
+  int32_t number;
+  STSma*  tSma;
 } STSmaWrapper;
 
 static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
@@ -2228,113 +2311,45 @@ static FORCE_INLINE void tdDestroyTSma(STSma* pSma) {
   }
 }
 
-static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
+static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
   if (pSW) {
     if (pSW->tSma) {
-      for (uint32_t i = 0; i < pSW->number; ++i) {
-        tdDestroyTSma(pSW->tSma + i);
+      if (deepCopy) {
+        for (uint32_t i = 0; i < pSW->number; ++i) {
+          tdDestroyTSma(pSW->tSma + i);
+        }
       }
       taosMemoryFreeClear(pSW->tSma);
     }
   }
 }
 
-static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
-  tdDestroyTSmaWrapper(pSW);
-  taosMemoryFree(pSW);
+static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) {
+  tdDestroyTSmaWrapper(pSW, deepCopy);
+  taosMemoryFreeClear(pSW);
   return NULL;
 }
 
-static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
-  int32_t tlen = 0;
+int32_t tEncodeSVCreateTSmaReq(SEncoder* pCoder, const SVCreateTSmaReq* pReq);
+int32_t tDecodeSVCreateTSmaReq(SDecoder* pCoder, SVCreateTSmaReq* pReq);
 
-  tlen += taosEncodeFixedI8(buf, pSma->version);
-  tlen += taosEncodeFixedI8(buf, pSma->intervalUnit);
-  tlen += taosEncodeFixedI8(buf, pSma->slidingUnit);
-  tlen += taosEncodeFixedI8(buf, pSma->timezoneInt);
-  tlen += taosEncodeString(buf, pSma->indexName);
-  tlen += taosEncodeFixedI32(buf, pSma->exprLen);
-  tlen += taosEncodeFixedI32(buf, pSma->tagsFilterLen);
-  tlen += taosEncodeFixedI64(buf, pSma->indexUid);
-  tlen += taosEncodeFixedI64(buf, pSma->tableUid);
-  tlen += taosEncodeFixedI64(buf, pSma->interval);
-  tlen += taosEncodeFixedI64(buf, pSma->offset);
-  tlen += taosEncodeFixedI64(buf, pSma->sliding);
-
-  if (pSma->exprLen > 0) {
-    tlen += taosEncodeString(buf, pSma->expr);
-  }
+int32_t tEncodeTSma(SEncoder* pCoder, const STSma* pSma);
+int32_t tDecodeTSma(SDecoder* pCoder, STSma* pSma);
 
-  if (pSma->tagsFilterLen > 0) {
-    tlen += taosEncodeString(buf, pSma->tagsFilter);
+static int32_t tEncodeTSmaWrapper(SEncoder* pEncoder, const STSmaWrapper* pReq) {
+  if (tEncodeI32(pEncoder, pReq->number) < 0) return -1;
+  for (int32_t i = 0; i < pReq->number; ++i) {
+    tEncodeTSma(pEncoder, pReq->tSma + i);
   }
-
-  return tlen;
-}
-
-static FORCE_INLINE int32_t tEncodeTSmaWrapper(void** buf, const STSmaWrapper* pSW) {
-  int32_t tlen = 0;
-
-  tlen += taosEncodeFixedU32(buf, pSW->number);
-  for (uint32_t i = 0; i < pSW->number; ++i) {
-    tlen += tEncodeTSma(buf, pSW->tSma + i);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeTSma(void* buf, STSma* pSma) {
-  buf = taosDecodeFixedI8(buf, &pSma->version);
-  buf = taosDecodeFixedI8(buf, &pSma->intervalUnit);
-  buf = taosDecodeFixedI8(buf, &pSma->slidingUnit);
-  buf = taosDecodeFixedI8(buf, &pSma->timezoneInt);
-  buf = taosDecodeStringTo(buf, pSma->indexName);
-  buf = taosDecodeFixedI32(buf, &pSma->exprLen);
-  buf = taosDecodeFixedI32(buf, &pSma->tagsFilterLen);
-  buf = taosDecodeFixedI64(buf, &pSma->indexUid);
-  buf = taosDecodeFixedI64(buf, &pSma->tableUid);
-  buf = taosDecodeFixedI64(buf, &pSma->interval);
-  buf = taosDecodeFixedI64(buf, &pSma->offset);
-  buf = taosDecodeFixedI64(buf, &pSma->sliding);
-
-  if (pSma->exprLen > 0) {
-    if ((buf = taosDecodeString(buf, &pSma->expr)) == NULL) {
-      tdDestroyTSma(pSma);
-      return NULL;
-    }
-  } else {
-    pSma->expr = NULL;
-  }
-
-  if (pSma->tagsFilterLen > 0) {
-    if ((buf = taosDecodeString(buf, &pSma->tagsFilter)) == NULL) {
-      tdDestroyTSma(pSma);
-      return NULL;
-    }
-  } else {
-    pSma->tagsFilter = NULL;
-  }
-
-  return buf;
+  return 0;
 }
 
-static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
-  buf = taosDecodeFixedU32(buf, &pSW->number);
-
-  pSW->tSma = (STSma*)taosMemoryCalloc(pSW->number, sizeof(STSma));
-  if (pSW->tSma == NULL) {
-    return NULL;
-  }
-
-  for (uint32_t i = 0; i < pSW->number; ++i) {
-    if ((buf = tDecodeTSma(buf, pSW->tSma + i)) == NULL) {
-      for (uint32_t j = i; j >= 0; --i) {
-        tdDestroyTSma(pSW->tSma + j);
-      }
-      taosMemoryFree(pSW->tSma);
-      return NULL;
-    }
+static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
+  if (tDecodeI32(pDecoder, &pReq->number) < 0) return -1;
+  for (int32_t i = 0; i < pReq->number; ++i) {
+    tDecodeTSma(pDecoder, pReq->tSma + i);
   }
-  return buf;
+  return 0;
 }
 
 typedef struct {
@@ -2574,6 +2589,18 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
   taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
 }
 
+typedef struct {
+  int64_t streamId;
+  int32_t taskId;
+  int32_t sourceVg;
+  int64_t sourceVer;
+  SArray* data;  // SArray<SSDataBlock>
+} SStreamDispatchReq;
+
+typedef struct {
+  int8_t inputStatus;
+} SStreamDispatchRsp;
+
 #define TD_AUTO_CREATE_TABLE 0x1
 typedef struct {
   int64_t        suid;

include/common/tmsgcb.h

@@ -22,9 +22,10 @@
 extern "C" {
 #endif
 
-typedef struct SRpcMsg      SRpcMsg;
-typedef struct SEpSet       SEpSet;
-typedef struct SMgmtWrapper SMgmtWrapper;
+typedef struct SRpcMsg        SRpcMsg;
+typedef struct SEpSet         SEpSet;
+typedef struct SMgmtWrapper   SMgmtWrapper;
+typedef struct SRpcHandleInfo SRpcHandleInfo;
 
 typedef enum {
   QUERY_QUEUE,
@@ -37,18 +38,18 @@ typedef enum {
   QUEUE_MAX,
 } EQueueType;
 
-typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
-typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
-typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
-typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
-typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
-typedef void (*SendRedirectRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp, const SEpSet* pNewEpSet);
-typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
-typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
-typedef void (*ReportStartup)(SMgmtWrapper* pWrapper, const char* name, const char* desc);
+typedef int32_t (*PutToQueueFp)(void* pMgmt, SRpcMsg* pMsg);
+typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
+typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
+typedef void (*SendRspFp)(SRpcMsg* pMsg);
+typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
+typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
+typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
+typedef void (*ReportStartup)(const char* name, const char* desc);
 
 typedef struct {
-  SMgmtWrapper*           pWrapper;
+  void*                   mgmt;
+  void*                   clientRpc;
   PutToQueueFp            queueFps[QUEUE_MAX];
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
@@ -57,17 +58,16 @@ typedef struct {
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
   ReportStartup           reportStartupFp;
-  void*                   clientRpc;
 } SMsgCb;
 
-void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
-int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
-int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
-int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
-void    tmsgSendRsp(const SRpcMsg* pRsp);
-void    tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet);
-void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
-void    tmsgReleaseHandle(void* handle, int8_t type);
+void    tmsgSetDefault(const SMsgCb* msgcb);
+int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg);
+int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype);
+int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
+void    tmsgSendRsp(SRpcMsg* pMsg);
+void    tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
+void    tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
+void    tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
 void    tmsgReportStartup(const char* name, const char* desc);
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -178,6 +178,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)

include/common/ttime.h

@@ -59,10 +59,11 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
  *    precision == TSDB_TIME_PRECISION_NANO,  it returns timestamp in nanosecond.
  */
 static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
-  int64_t factor = (precision == TSDB_TIME_PRECISION_MILLI) ? 1000 :
-                   (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000 : 1000000000;
-  time_t t = taosTime(NULL);
-  struct tm * tm= taosLocalTime(&t, NULL);
+  int64_t    factor = (precision == TSDB_TIME_PRECISION_MILLI)   ? 1000
+                      : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000000
+                                                                 : 1000000000;
+  time_t     t = taosTime(NULL);
+  struct tm* tm = taosLocalTime(&t, NULL);
   tm->tm_hour = 0;
   tm->tm_min = 0;
   tm->tm_sec = 0;
@@ -79,13 +80,13 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
 
 int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
-char getPrecisionUnit(int32_t precision);
+char    getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
 int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
-int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal);
+int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
 
-void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
+void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t time, int32_t precision);
 
 #ifdef __cplusplus
 }

include/common/ttypes.h

@@ -179,6 +179,8 @@ typedef struct {
     }                                          \
   } while (0)
 
+//TODO: use varchar(0) to represent NULL type
+#define IS_NULL_TYPE(_t)             ((_t) == TSDB_DATA_TYPE_NULL)
 #define IS_SIGNED_NUMERIC_TYPE(_t)   ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT)
 #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
 #define IS_FLOAT_TYPE(_t)            ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)

include/dnode/mgmt/dnode.h

@@ -22,66 +22,28 @@
 extern "C" {
 #endif
 
-/* ------------------------ TYPES EXPOSED ---------------- */
-typedef struct SDnode SDnode;
-
 /**
- * @brief Initialize the environment
+ * @brief Initialize the dnode
  *
+ * @param rtype for internal debug usage, default is 0
  * @return int32_t 0 for success and -1 for failure
  */
-int32_t dmInit();
+int32_t dmInit(int8_t rtype);
 
 /**
- * @brief Clear the environment
+ * @brief Cleanup the dnode
  */
 void dmCleanup();
 
-/* ------------------------ SDnode ----------------------- */
-typedef struct {
-  int32_t   numOfSupportVnodes;
-  uint16_t  serverPort;
-  char      dataDir[PATH_MAX];
-  char      localEp[TSDB_EP_LEN];
-  char      localFqdn[TSDB_FQDN_LEN];
-  char      firstEp[TSDB_EP_LEN];
-  char      secondEp[TSDB_EP_LEN];
-  SDiskCfg *disks;
-  int32_t   numOfDisks;
-  int8_t    ntype;
-} SDnodeOpt;
-
-typedef enum { DND_EVENT_START = 0, DND_EVENT_STOP = 1, DND_EVENT_CHILD = 2 } EDndEvent;
-
 /**
- * @brief Initialize and start the dnode.
- *
- * @param pOption Option of the dnode.
- * @return SDnode* The dnode object.
+ * @brief Run dnode.
  */
-SDnode *dmCreate(const SDnodeOpt *pOption);
+int32_t dmRun();
 
 /**
- * @brief Stop and cleanup the dnode.
- *
- * @param pDnode The dnode object to close.
- */
-void dmClose(SDnode *pDnode);
-
-/**
- * @brief Run dnode until specific event is receive.
- *
- * @param pDnode The dnode object to run.
- */
-int32_t dmRun(SDnode *pDnode);
-
-/**
- * @brief Handle event in the dnode.
- *
- * @param pDnode The dnode object to close.
- * @param event The event to handle.
+ * @brief Stop dnode.
  */
-void dmSetEvent(SDnode *pDnode, EDndEvent event);
+void dmStop();
 
 #ifdef __cplusplus
 }

include/dnode/mnode/mnode.h

@@ -88,7 +88,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
  * @param pMsg The request msg.
  * @return int32_t 0 for success, -1 for failure.
  */
-int32_t mndProcessMsg(SNodeMsg *pMsg);
+int32_t mndProcessMsg(SRpcMsg *pMsg);
 
 /**
  * @brief Generate machine code

include/dnode/mnode/sdb/sdb.h

@@ -333,23 +333,23 @@ SSdbRow *sdbAllocRow(int32_t objSize);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
 typedef struct SSdb {
-  SMnode     *pMnode;
-  char       *currDir;
-  char       *syncDir;
-  char       *tmpDir;
-  int64_t     lastCommitVer;
-  int64_t     curVer;
-  int64_t     tableVer[SDB_MAX];
-  int64_t     maxId[SDB_MAX];
-  EKeyType    keyTypes[SDB_MAX];
-  SHashObj   *hashObjs[SDB_MAX];
-  SRWLatch    locks[SDB_MAX];
-  SdbInsertFp insertFps[SDB_MAX];
-  SdbUpdateFp updateFps[SDB_MAX];
-  SdbDeleteFp deleteFps[SDB_MAX];
-  SdbDeployFp deployFps[SDB_MAX];
-  SdbEncodeFp encodeFps[SDB_MAX];
-  SdbDecodeFp decodeFps[SDB_MAX];
+  SMnode        *pMnode;
+  char          *currDir;
+  char          *syncDir;
+  char          *tmpDir;
+  int64_t        lastCommitVer;
+  int64_t        curVer;
+  int64_t        tableVer[SDB_MAX];
+  int64_t        maxId[SDB_MAX];
+  EKeyType       keyTypes[SDB_MAX];
+  SHashObj      *hashObjs[SDB_MAX];
+  TdThreadRwlock locks[SDB_MAX];
+  SdbInsertFp    insertFps[SDB_MAX];
+  SdbUpdateFp    updateFps[SDB_MAX];
+  SdbDeleteFp    deleteFps[SDB_MAX];
+  SdbDeployFp    deployFps[SDB_MAX];
+  SdbEncodeFp    encodeFps[SDB_MAX];
+  SdbDecodeFp    decodeFps[SDB_MAX];
 } SSdb;
 
 #ifdef __cplusplus

include/libs/catalog/catalog.h

@@ -59,6 +59,11 @@ typedef struct SMetaData {
   SArray    *pQnodeList;  // qnode list, SArray<SQueryNodeAddr>
 } SMetaData;
 
+typedef struct STbSVersion {
+  char* tbFName;
+  int32_t sver;
+} STbSVersion;
+
 typedef struct SCatalogCfg {
   uint32_t maxTblCacheNum;
   uint32_t maxDBCacheNum;
@@ -186,6 +191,8 @@ int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg);
  */
 int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* dbFName);
 
+int32_t catalogChkTbMetaVersion(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, SArray* pTables);
+
 /**
  * Force refresh a table's local cached meta data. 
  * @param pCatalog (input, got with catalogGetHandle)

include/libs/executor/executor.h

@@ -42,7 +42,7 @@ typedef struct SReadHandle {
 #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
 
 typedef enum {
-  OPTR_EXEC_MODEL_BATCH  = 0x1,
+  OPTR_EXEC_MODEL_BATCH = 0x1,
   OPTR_EXEC_MODEL_STREAM = 0x2,
 } EOPTR_EXEC_MODEL;
 
@@ -81,7 +81,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
  * @param isAdd
  * @return
  */
-int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd);
+int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd);
 
 /**
  * Create the exec task object according to task json
@@ -95,6 +95,15 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
 int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
                         qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model);
 
+/**
+ *
+ * @param tinfo
+ * @param sversion
+ * @param tversion
+ * @return
+ */
+int32_t qGetQueriedTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* tableName, int32_t* sversion, int32_t* tversion);
+
 /**
  * The main task execution function, including query on both table and multiple tables,
  * which are decided according to the tag or table name query conditions
@@ -169,7 +178,7 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
 
 void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
 
-int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t *resNum, SExplainExecInfo **pRes);
+int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes);
 
 #ifdef __cplusplus
 }

include/libs/function/function.h

@@ -170,32 +170,18 @@ typedef struct SInputColumnInfoData {
 
 // sql function runtime context
 typedef struct SqlFunctionCtx {
-  SInputColumnInfoData input;
-  SResultDataInfo      resDataInfo;
-  uint32_t             order;  // data block scanner order: asc|desc
-  ////////////////////////////////////////////////////////////////
-  int32_t          startRow;   // start row index
-  int32_t          size;       // handled processed row number
-  SColumnInfoData* pInput;
-  SColumnDataAgg   agg;
-  int16_t          inputType;    // TODO remove it
-  int16_t          inputBytes;   // TODO remove it
-  bool             hasNull;      // null value exist in current block, TODO remove it
-  bool             requireNull;  // require null in some function, TODO remove it
-  int32_t          columnIndex;  // TODO remove it
-  uint8_t          currentStage;  // record current running step, default: 0
-  bool             isAggSet;
-  int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block, TODO remove it
-  bool             stableQuery;
-  /////////////////////////////////////////////////////////////////
-  int16_t          functionId;    // function id
-  char *           pOutput;       // final result output buffer, point to sdata->data
-  int32_t          numOfParams;
-  SFunctParam     *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
-  int64_t         *ptsList;       // corresponding timestamp array list
-  SColumnInfoData *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
-  int32_t          offset;
-  SVariant         tag;
+  SInputColumnInfoData   input;
+  SResultDataInfo        resDataInfo;
+  uint32_t               order;  // data block scanner order: asc|desc
+  uint8_t                scanFlag;  // record current running step, default: 0
+  int16_t                functionId;    // function id
+  char                  *pOutput;       // final result output buffer, point to sdata->data
+  int32_t                numOfParams;
+  SFunctParam           *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  int64_t               *ptsList;       // corresponding timestamp array list
+  SColumnInfoData       *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
+  int32_t                offset;
+  SVariant               tag;
   struct  SResultRowEntryInfo *resultInfo;
   SSubsidiaryResInfo     subsidiaries;
   SPoint1                start;
@@ -309,7 +295,7 @@ void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
 
 /**
- * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
+ * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
  * @return error code
  */
 int32_t udfcOpen();

include/libs/function/functionMgt.h

@@ -41,6 +41,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
   FUNCTION_TYPE_HISTOGRAM,
+  FUNCTION_TYPE_HYPERLOGLOG,
 
   // nonstandard SQL function
   FUNCTION_TYPE_BOTTOM = 500,
@@ -141,6 +142,8 @@ void fmFuncMgtDestroy();
 
 int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
 
+bool fmIsBuiltinFunc(const char* pFunc);
+
 bool fmIsAggFunc(int32_t funcId);
 bool fmIsScalarFunc(int32_t funcId);
 bool fmIsNonstandardSQLFunc(int32_t funcId);

include/libs/function/tudf.h

@@ -39,16 +39,6 @@ extern "C" {
 //======================================================================================
 //begin API to taosd and qworker
 
-typedef void *UdfcFuncHandle;
-
-/**
- * setup udf
- * @param udf, in
- * @param handle, out
- * @return error code
- */
-int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
-
 typedef struct SUdfColumnMeta {
   int16_t type;
   int32_t bytes;
@@ -95,32 +85,48 @@ typedef struct SUdfInterBuf {
   char* buf;
   int8_t numOfResult; //zero or one
 } SUdfInterBuf;
+typedef void *UdfcFuncHandle;
 
+//low level APIs
+/**
+ * setup udf
+ * @param udf, in
+ * @param funcHandle, out
+ * @return error code
+ */
+int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle);
 // output: interBuf
-int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
+int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
 // input: block, state
 // output: newState
-int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
+int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
 // input: interBuf
 // output: resultData
-int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
+int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
 // input: interbuf1, interbuf2
 // output: resultBuf
-int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
+int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
 // input: block
 // output: resultData
-int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
 /**
  * tearn down udf
  * @param handle
  * @return
  */
-int32_t teardownUdf(UdfcFuncHandle handle);
+int32_t doTeardownUdf(UdfcFuncHandle handle);
+
+void freeUdfInterBuf(SUdfInterBuf *buf);
 
+//high level APIs
 bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
+
+int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output);
+
+int32_t cleanUpUdfs();
 // end API to taosd and qworker
 //=============================================================================================================================
 // begin API to UDF writer.

include/libs/nodes/cmdnodes.h

@@ -78,7 +78,7 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType  type;
-  char       comment[TSDB_STB_COMMENT_LEN];
+  char       comment[TSDB_TB_COMMENT_LEN];
   int32_t    delay;
   float      filesFactor;
   SNodeList* pRollupFuncs;
@@ -90,7 +90,7 @@ typedef struct SColumnDefNode {
   ENodeType type;
   char      colName[TSDB_COL_NAME_LEN];
   SDataType dataType;
-  char      comments[TSDB_STB_COMMENT_LEN];
+  char      comments[TSDB_TB_COMMENT_LEN];
   bool      sma;
 } SColumnDefNode;
 

include/libs/nodes/nodes.h

@@ -208,6 +208,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_SORT,
   QUERY_NODE_PHYSICAL_PLAN_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_FILL,
   QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,
@@ -240,6 +241,7 @@ typedef struct SNodeList {
 
 #define SNodeptr void*
 
+int32_t  nodesNodeSize(ENodeType type);
 SNodeptr nodesMakeNode(ENodeType type);
 void     nodesDestroyNode(SNodeptr pNode);
 

include/libs/nodes/querynodes.h

@@ -48,6 +48,7 @@ typedef struct SExprNode {
   ENodeType type;
   SDataType resType;
   char      aliasName[TSDB_COL_NAME_LEN];
+  char      userAlias[TSDB_COL_NAME_LEN];
   SArray*   pAssociation;
 } SExprNode;
 
@@ -235,6 +236,7 @@ typedef struct SSelectStmt {
   bool        isTimeOrderQuery;
   bool        hasAggFuncs;
   bool        hasRepeatScanFuncs;
+  bool        hasNonstdSQLFunc;
 } SSelectStmt;
 
 typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
@@ -247,6 +249,7 @@ typedef struct SSetOperator {
   SNode*           pRight;
   SNodeList*       pOrderByList;  // SOrderByExprNode
   SNode*           pLimit;
+  char             stmtName[TSDB_TABLE_NAME_LEN];
 } SSetOperator;
 
 typedef enum ESqlClause {
@@ -325,6 +328,7 @@ typedef struct SQuery {
   bool           showRewrite;
   int32_t        placeholderNum;
   SArray*        pPlaceholderValues;
+  SNode*         pPrepareRoot;
 } SQuery;
 
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
@@ -345,6 +349,7 @@ bool nodesIsUnaryOp(const SOperatorNode* pOp);
 bool nodesIsArithmeticOp(const SOperatorNode* pOp);
 bool nodesIsComparisonOp(const SOperatorNode* pOp);
 bool nodesIsJsonOp(const SOperatorNode* pOp);
+bool nodesIsRegularOp(const SOperatorNode* pOp);
 
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);

include/libs/parser/parser.h

@@ -62,7 +62,7 @@ int32_t     qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t
 void        qDestroyStmtDataBlock(void* pBlock);
 STableMeta* qGetTableMetaInDataBlock(void* pDataBlock);
 
-int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
+int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
 int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
 int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
 int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
@@ -77,8 +77,8 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
 
 void*   smlInitHandle(SQuery* pQuery);
 void    smlDestroyHandle(void* pHandle);
-int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format,
-                    STableMeta* pTableMeta, char* tableName, char* msgBuf, int16_t msgBufLen);
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+                    char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
 #ifdef __cplusplus

include/libs/planner/planner.h

@@ -34,7 +34,6 @@ typedef struct SPlanContext {
   bool     showRewrite;
   int8_t   triggerType;
   int64_t  watermark;
-  int32_t  placeholderNum;
   char*    pMsg;
   int32_t  msgLen;
 } SPlanContext;
@@ -48,9 +47,6 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId,
-                       bool* pEmptyResult);
-
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);
 int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan);

include/libs/qcom/query.h

@@ -51,14 +51,12 @@ typedef struct STableComInfo {
 } STableComInfo;
 
 typedef struct SIndexMeta {
-
 #ifdef WINDOWS
   size_t avoidCompilationErrors;
 #endif
 
 } SIndexMeta;
 
-
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -95,7 +93,7 @@ typedef struct SDBVgInfo {
   int32_t   vgVersion;
   int8_t    hashMethod;
   int32_t   numOfTable;  // DB's table num, unit is TSDB_TABLE_NUM_UNIT
-  SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
+  SHashObj* vgHash;      // key:vgId, value:SVgroupInfo
 } SDBVgInfo;
 
 typedef struct SUseDbOutput {
@@ -135,7 +133,7 @@ typedef struct SMsgSendInfo {
 } SMsgSendInfo;
 
 typedef struct SQueryNodeStat {
-  int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
+  int32_t tableNum;  // vg table number, unit is TSDB_TABLE_NUM_UNIT
 } SQueryNodeStat;
 
 int32_t initTaskQueue();
@@ -172,7 +170,7 @@ const SSchema* tGetTbnameColumnSchema();
 bool           tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
 int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
-char *jobTaskStatusStr(int32_t status);
+char*   jobTaskStatusStr(int32_t status);
 
 SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name);
 
@@ -184,38 +182,43 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define SET_META_TYPE_TABLE(t)      (t) = META_TYPE_TABLE
 #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
 
-#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
-#define NEED_CLIENT_REFRESH_VG_ERROR(_code) ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
+#define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \
+  ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST)
+#define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
+  ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
-#define NEED_CLIENT_HANDLE_ERROR(_code) (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
+#define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
+  (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
+   NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
 
-#define NEED_SCHEDULER_RETRY_ERROR(_code) ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
+#define NEED_SCHEDULER_RETRY_ERROR(_code) \
+  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
 
 #define REQUEST_MAX_TRY_TIMES 5
 
-#define qFatal(...)                                                     \
-  do {                                                                  \
-    if (qDebugFlag & DEBUG_FATAL) {                                     \
-      taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
-    }                                                                   \
+#define qFatal(...)                                                                           \
+  do {                                                                                        \
+    if (qDebugFlag & DEBUG_FATAL) {                                                           \
+      taosPrintLog("QRY FATAL ", DEBUG_FATAL, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
+    }                                                                                         \
   } while (0)
-#define qError(...)                                                     \
-  do {                                                                  \
-    if (qDebugFlag & DEBUG_ERROR) {                                     \
-      taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
-    }                                                                   \
+#define qError(...)                                                                           \
+  do {                                                                                        \
+    if (qDebugFlag & DEBUG_ERROR) {                                                           \
+      taosPrintLog("QRY ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
+    }                                                                                         \
   } while (0)
-#define qWarn(...)                                                    \
-  do {                                                                \
-    if (qDebugFlag & DEBUG_WARN) {                                    \
-      taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
-    }                                                                 \
+#define qWarn(...)                                                                          \
+  do {                                                                                      \
+    if (qDebugFlag & DEBUG_WARN) {                                                          \
+      taosPrintLog("QRY WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
+    }                                                                                       \
   } while (0)
-#define qInfo(...)                                               \
-  do {                                                           \
-    if (qDebugFlag & DEBUG_INFO) {                               \
-      taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
-    }                                                            \
+#define qInfo(...)                                                                     \
+  do {                                                                                 \
+    if (qDebugFlag & DEBUG_INFO) {                                                     \
+      taosPrintLog("QRY ", DEBUG_INFO, tsLogEmbedded ? 255 : qDebugFlag, __VA_ARGS__); \
+    }                                                                                  \
   } while (0)
 #define qDebug(...)                                               \
   do {                                                            \
@@ -236,10 +239,30 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
     }                                                                    \
   } while (0)
 
-#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
-#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
-#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
-
+#define QRY_ERR_RET(c)                \
+  do {                                \
+    int32_t _code = c;                \
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+      return _code;                   \
+    }                                 \
+  } while (0)
+#define QRY_RET(c)                    \
+  do {                                \
+    int32_t _code = c;                \
+    if (_code != TSDB_CODE_SUCCESS) { \
+      terrno = _code;                 \
+    }                                 \
+    return _code;                     \
+  } while (0)
+#define QRY_ERR_JRET(c)              \
+  do {                               \
+    code = c;                        \
+    if (code != TSDB_CODE_SUCCESS) { \
+      terrno = code;                 \
+      goto _return;                  \
+    }                                \
+  } while (0)
 
 #ifdef __cplusplus
 }

include/libs/scheduler/scheduler.h

@@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
  * @param nodeList  Qnode/Vnode address list, element is SQueryNodeAddr
  * @return
  */
-int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, bool needRes, SQueryResult *pRes);
+int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SQueryResult *pRes);
 
 /**
  * Process the query job, generated according to the query physical plan.

include/libs/stream/tstream.h

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "os.h"
 #include "tdatablock.h"
 #include "tmsg.h"
 #include "tmsgcb.h"
@@ -29,8 +30,23 @@ extern "C" {
 typedef struct SStreamTask SStreamTask;
 
 enum {
-  STREAM_TASK_STATUS__RUNNING = 1,
-  STREAM_TASK_STATUS__STOP,
+  TASK_STATUS__IDLE = 1,
+  TASK_STATUS__EXECUTING,
+  TASK_STATUS__CLOSING,
+};
+
+enum {
+  TASK_INPUT_STATUS__NORMAL = 1,
+  TASK_INPUT_STATUS__BLOCKED,
+  TASK_INPUT_STATUS__RECOVER,
+  TASK_INPUT_STATUS__STOP,
+  TASK_INPUT_STATUS__FAILED,
+};
+
+enum {
+  TASK_OUTPUT_STATUS__NORMAL = 1,
+  TASK_OUTPUT_STATUS__WAIT,
+  TASK_OUTPUT_STATUS__BLOCKED,
 };
 
 enum {
@@ -38,10 +54,65 @@ enum {
   STREAM_CREATED_BY__SMA,
 };
 
+enum {
+  STREAM_INPUT__DATA_SUBMIT = 1,
+  STREAM_INPUT__DATA_BLOCK,
+  STREAM_INPUT__CHECKPOINT,
+};
+
 typedef struct {
-  int32_t nodeId;  // 0 for snode
-  SEpSet  epSet;
-} SStreamTaskEp;
+  int8_t type;
+
+  int32_t sourceVg;
+  int64_t sourceVer;
+
+  int32_t*    dataRef;
+  SSubmitReq* data;
+} SStreamDataSubmit;
+
+typedef struct {
+  int8_t type;
+
+  int32_t sourceVg;
+  int64_t sourceVer;
+
+  SArray* blocks;  // SArray<SSDataBlock*>
+} SStreamDataBlock;
+
+typedef struct {
+  int8_t type;
+} SStreamCheckpoint;
+
+static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
+  SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosMemoryCalloc(1, sizeof(SStreamDataSubmit));
+  if (pDataSubmit == NULL) return NULL;
+  pDataSubmit->data = pReq;
+  pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
+  if (pDataSubmit->data == NULL) goto FAIL;
+  *pDataSubmit->dataRef = 1;
+  return pDataSubmit;
+FAIL:
+  taosMemoryFree(pDataSubmit);
+  return NULL;
+}
+
+static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
+  //
+  atomic_add_fetch_32(pDataSubmit->dataRef, 1);
+}
+
+static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) {
+  int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1);
+  ASSERT(ref >= 0);
+  if (ref == 0) {
+    taosMemoryFree(pDataSubmit->data);
+    taosMemoryFree(pDataSubmit->dataRef);
+    taosFreeQitem(pDataSubmit);
+  }
+}
+
+int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
+void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
 
 typedef struct {
   void* inputHandle;
@@ -122,9 +193,15 @@ enum {
   TASK_SINK__FETCH,
 };
 
+enum {
+  TASK_INPUT_TYPE__SUMBIT_BLOCK = 1,
+  TASK_INPUT_TYPE__DATA_BLOCK,
+};
+
 struct SStreamTask {
   int64_t streamId;
   int32_t taskId;
+  int8_t  inputType;
   int8_t  status;
 
   int8_t  sourceType;
@@ -155,9 +232,13 @@ struct SStreamTask {
     STaskDispatcherShuffle shuffleDispatcher;
   };
 
-  // msg buffer
-  int32_t     memUsed;
+  int8_t inputStatus;
+  int8_t outputStatus;
+
   STaosQueue* inputQ;
+  STaosQall*  inputQAll;
+  STaosQueue* outputQ;
+  STaosQall*  outputQAll;
 
   // application storage
   void* ahandle;
@@ -199,10 +280,22 @@ typedef struct {
   SArray* res;  // SArray<SSDataBlock>
 } SStreamSinkReq;
 
-int32_t streamEnqueueData(SStreamTask* pTask, const void* input, int32_t inputType);
+typedef struct {
+  SMsgHead head;
+  int64_t  streamId;
+  int32_t  taskId;
+} SStreamTaskRunReq;
+
+int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
+int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
+int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
 
 int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, int32_t inputType, int32_t workId);
 
+int32_t streamTaskRun(SStreamTask* pTask);
+
+int32_t streamTaskHandleInput(SStreamTask* pTask, void* data);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/sync/sync.h

@@ -20,14 +20,11 @@
 extern "C" {
 #endif
 
-#include <stdbool.h>
-#include <stdint.h>
-//#include <tdatablock.h>
+#include "os.h"
+
 #include "cJSON.h"
 #include "tdef.h"
-//#include "taosdef.h"
-//#include "trpc.h"
-//#include "wal.h"
+#include "tmsgcb.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -132,11 +129,10 @@ typedef struct SSyncInfo {
   char        path[TSDB_FILENAME_LEN];
   SWal*       pWal;
   SSyncFSM*   pFsm;
+  SMsgCb*     msgcb;
 
-  void* rpcClient;
-  int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
-  void* queue;
-  int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
+  int32_t (*FpSendMsg)(const SEpSet* pEpSet, SRpcMsg* pMsg);
+  int32_t (*FpEqMsg)(const SMsgCb* msgcb, SRpcMsg* pMsg);
 
 } SSyncInfo;
 
@@ -144,6 +140,7 @@ int32_t     syncInit();
 void        syncCleanUp();
 int64_t     syncOpen(const SSyncInfo* pSyncInfo);
 void        syncStart(int64_t rid);
+void        syncStartStandBy(int64_t rid);
 void        syncStop(int64_t rid);
 int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
 ESyncState  syncGetMyRole(int64_t rid);

include/libs/sync/syncTools.h

@@ -20,13 +20,10 @@
 extern "C" {
 #endif
 
-#include <stdbool.h>
-#include <stdint.h>
-//#include <tdatablock.h>
+#include "os.h"
+
 #include "cJSON.h"
-//#include "taosdef.h"
 #include "trpc.h"
-//#include "wal.h"
 
 // ------------------ ds -------------------
 typedef struct SRaftId {
@@ -43,8 +40,7 @@ void       syncNodeRelease(SSyncNode* pNode);
 
 int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
 int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
-void    syncSetQ(int64_t rid, void* queueHandle);
-void    syncSetRpc(int64_t rid, void* rpcHandle);
+void    syncSetMsgCb(int64_t rid, const SMsgCb* msgcb);
 char*   sync2SimpleStr(int64_t rid);
 
 // set timer ms

include/libs/transport/trpc.h

@@ -26,45 +26,43 @@ extern "C" {
 
 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
+#define IsReq(pMsg)      (pMsg->msgType & 1U)
 
 extern int tsRpcHeadSize;
 
-typedef struct SRpcConnInfo {
+typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
-  uint32_t serverIp;
   char     user[TSDB_USER_LEN];
 } SRpcConnInfo;
 
-typedef struct SRpcMsg {
-  tmsg_t  msgType;
-  void *  pCont;
-  int     contLen;
-  int32_t code;
+typedef struct SRpcHandleInfo {
+  // rpc info
   void *  handle;         // rpc handle returned to app
-  void *  ahandle;        // app handle set by client
   int64_t refId;          // refid, used by server
-  int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
-  int     persistHandle;  // persist handle or not
+  int32_t noResp;         // has response or not(default 0, 0: resp, 1: no resp);
+  int32_t persistHandle;  // persist handle or not
 
-} SRpcMsg;
+  // app info
+  void *ahandle;  // app handle set by client
+  void *wrapper;  // wrapper handle
+  void *node;     // node mgmt handle
 
-typedef struct {
-  char     user[TSDB_USER_LEN];
-  uint32_t clientIp;
-  uint16_t clientPort;
-  SRpcMsg  rpcMsg;
-  int32_t  rspLen;
-  void *   pRsp;
-  void *   pNode;
-} SNodeMsg;
+  // resp info
+  void *  rsp;
+  int32_t rspLen;
+} SRpcHandleInfo;
+
+typedef struct SRpcMsg {
+  tmsg_t         msgType;
+  void *         pCont;
+  int32_t        contLen;
+  int32_t        code;
+  SRpcHandleInfo info;
+  SRpcConnInfo   conn;
+} SRpcMsg;
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
-typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
-///
-// // SRpcMsg code
-// REDIERE,
-// NOT READY, EpSet
 typedef bool (*RpcRfp)(int32_t code);
 
 typedef struct SRpcInit {
@@ -77,18 +75,11 @@ typedef struct SRpcInit {
   int      idleTime;      // milliseconds, 0 means idle timer is disabled
 
   // the following is for client app ecurity only
-  char *user;     // user name
-  char  spi;      // security parameter index
-  char  encrypt;  // encrypt algorithm
-  char *secret;   // key for authentication
-  char *ckey;     // ciphering key
+  char *user;  // user name
 
   // call back to process incoming msg, code shall be ignored by server app
   RpcCfp cfp;
 
-  // call back to retrieve the client auth info, for server app only
-  RpcAfp afp;
-
   // user defined retry func
   RpcRfp rfp;
 

include/os/os.h

@@ -58,9 +58,6 @@ extern "C" {
 #else
 #include <winsock.h>
 #endif
-
-#define __typeof(a) auto
-
 #endif
 
 #include <errno.h>

include/os/osAtomic.h

@@ -23,92 +23,92 @@ extern "C" {
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-  #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
-  #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID
-  #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
-  #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
-  #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
-  #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
-  #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
-  #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
-  #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
-  #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
-  #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
-  #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
-  #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
-  #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
+#define __atomic_load_n             __ATOMIC_LOAD_N_FUNC_TAOS_FORBID
+#define __atomic_store_n            __ATOMIC_STORE_N_FUNC_TAOS_FORBID
+#define __atomic_exchange_n         __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID
+#define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID
+#define __atomic_add_fetch          __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_add          __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID
+#define __atomic_sub_fetch          __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_sub          __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID
+#define __atomic_and_fetch          __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_and          __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID
+#define __atomic_or_fetch           __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_or           __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID
+#define __atomic_xor_fetch          __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID
+#define __atomic_fetch_xor          __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID
 #endif
 
-int8_t atomic_load_8(int8_t volatile *ptr);
+int8_t  atomic_load_8(int8_t volatile *ptr);
 int16_t atomic_load_16(int16_t volatile *ptr);
 int32_t atomic_load_32(int32_t volatile *ptr);
 int64_t atomic_load_64(int64_t volatile *ptr);
-void* atomic_load_ptr(void *ptr);
-void atomic_store_8(int8_t volatile *ptr, int8_t val);
-void atomic_store_16(int16_t volatile *ptr, int16_t val);
-void atomic_store_32(int32_t volatile *ptr, int32_t val);
-void atomic_store_64(int64_t volatile *ptr, int64_t val);
-void atomic_store_ptr(void *ptr, void *val);
-int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_load_ptr(void *ptr);
+void    atomic_store_8(int8_t volatile *ptr, int8_t val);
+void    atomic_store_16(int16_t volatile *ptr, int16_t val);
+void    atomic_store_32(int32_t volatile *ptr, int32_t val);
+void    atomic_store_64(int64_t volatile *ptr, int64_t val);
+void    atomic_store_ptr(void *ptr, void *val);
+int8_t  atomic_exchange_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val);
-void* atomic_exchange_ptr(void *ptr, void *val);
-int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
+void   *atomic_exchange_ptr(void *ptr, void *val);
+int8_t  atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval);
 int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval);
 int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval);
 int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval);
-void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
-int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval);
+int8_t  atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_add_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_add_fetch_ptr(void *ptr, void *val);
+int8_t  atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_add_ptr(void *ptr, void *val);
-int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_fetch_add_ptr(void *ptr, void *val);
+int8_t  atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_sub_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_sub_fetch_ptr(void *ptr, void *val);
+int8_t  atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_sub_ptr(void *ptr, void *val);
-int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_fetch_sub_ptr(void *ptr, void *val);
+int8_t  atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_and_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_and_fetch_ptr(void *ptr, void *val);
+int8_t  atomic_fetch_and_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_and_ptr(void *ptr, void *val);
-int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_fetch_and_ptr(void *ptr, void *val);
+int8_t  atomic_or_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_or_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_or_fetch_ptr(void *ptr, void *val);
+int8_t  atomic_fetch_or_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_or_ptr(void *ptr, void *val);
-int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_fetch_or_ptr(void *ptr, void *val);
+int8_t  atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val);
-void* atomic_xor_fetch_ptr(void *ptr, void *val);
-int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
+void   *atomic_xor_fetch_ptr(void *ptr, void *val);
+int8_t  atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val);
 int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val);
 int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val);
 int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val);
-void* atomic_fetch_xor_ptr(void *ptr, void *val);
+void   *atomic_fetch_xor_ptr(void *ptr, void *val);
 
 #ifdef __cplusplus
 }

include/os/osFile.h

@@ -66,7 +66,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
 int32_t taosUmaskFile(int32_t maskVal);
  
 int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
-int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno);
+int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
 int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
 bool    taosCheckExistFile(const char *pathname);
  

include/os/osMath.h

@@ -23,11 +23,13 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#define TSWAP(a, b)             \
-  do {                          \
-    __typeof(a) __tmp = (a); \
-    (a) = (b);               \
-    (b) = __tmp;                \
+#define TSWAP(a, b)                              \
+  do {                                           \
+    char *__tmp = taosMemoryMalloc(sizeof(a));   \
+    memcpy(__tmp, &(a), sizeof(a));              \
+    memcpy(&(a), &(b), sizeof(a));               \
+    memcpy(&(b), __tmp, sizeof(a));              \
+    taosMemoryFree(__tmp);                       \
   } while (0)
 
 #ifdef WINDOWS

include/os/osString.h

@@ -37,6 +37,14 @@ typedef int32_t TdUcs4;
     #define wcstombs WCSTOMBS_FUNC_TAOS_FORBID
     #define wcsncpy WCSNCPY_FUNC_TAOS_FORBID
     #define wchar_t WCHAR_T_TYPE_TAOS_FORBID
+    #define strcasestr STR_CASE_STR_FORBID
+    #define strtoll STR_TO_LL_FUNC_TAOS_FORBID
+    #define strtoull STR_TO_ULL_FUNC_TAOS_FORBID
+    #define strtol STR_TO_L_FUNC_TAOS_FORBID
+    #define strtoul STR_TO_UL_FUNC_TAOS_FORBID
+    #define strtod STR_TO_LD_FUNC_TAOS_FORBID
+    #define strtold STR_TO_D_FUNC_TAOS_FORBID
+    #define strtof STR_TO_F_FUNC_TAOS_FORBID
 #endif
 
 #ifdef WINDOWS
@@ -69,6 +77,19 @@ int32_t taosMbsToWchars(TdWchar *pWchars, const char *pStrs, int32_t size);
 int32_t taosWcharToMb(char *pStr, TdWchar wchar);
 int32_t taosWcharsToMbs(char *pStrs, TdWchar *pWchars, int32_t size);
 
+char *taosStrCaseStr(const char *str, const char *pattern);
+
+int64_t taosStr2Int64(const char *str, char** pEnd, int32_t radix);
+uint64_t taosStr2UInt64(const char *str, char** pEnd, int32_t radix);
+int32_t taosStr2Int32(const char *str, char** pEnd, int32_t radix);
+uint32_t taosStr2UInt32(const char *str, char** pEnd, int32_t radix);
+int16_t taosStr2Int16(const char *str, char** pEnd, int32_t radix);
+uint16_t taosStr2UInt16(const char *str, char** pEnd, int32_t radix);
+int8_t taosStr2Int8(const char *str, char** pEnd, int32_t radix);
+uint8_t taosStr2UInt8(const char *str, char** pEnd, int32_t radix);
+double taosStr2Double(const char *str, char** pEnd);
+float taosStr2Float(const char *str, char** pEnd);
+
 #ifdef __cplusplus
 }
 #endif

include/util/taoserror.h

@@ -89,6 +89,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x0115)
 #define TSDB_CODE_DUP_KEY                       TAOS_DEF_ERROR_CODE(0, 0x0116)
 #define TSDB_CODE_NEED_RETRY                    TAOS_DEF_ERROR_CODE(0, 0x0117)
+#define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE       TAOS_DEF_ERROR_CODE(0, 0x0118)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0140)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0141)
@@ -280,6 +281,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
 #define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
+#define TSDB_CODE_MND_TOPIC_SUBSCRIBED          TAOS_DEF_ERROR_CODE(0, 0x03EB)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -322,6 +324,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0516)
 #define TSDB_CODE_VND_HASH_MISMATCH             TAOS_DEF_ERROR_CODE(0, 0x0517)
 #define TSDB_CODE_VND_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0518)
+#define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
+#define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
+#define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
@@ -353,7 +358,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061A)
 #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061B)
 #define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x061C)
-#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x062D)
+#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x061D)
+#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x061E)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)
@@ -635,6 +641,15 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_PERMISSION_DENIED         TAOS_DEF_ERROR_CODE(0, 0x2644)
 #define TSDB_CODE_PAR_INVALID_STREAM_QUERY      TAOS_DEF_ERROR_CODE(0, 0x2645)
 #define TSDB_CODE_PAR_INVALID_INTERNAL_PK       TAOS_DEF_ERROR_CODE(0, 0x2646)
+#define TSDB_CODE_PAR_INVALID_TIMELINE_FUNC     TAOS_DEF_ERROR_CODE(0, 0x2647)
+#define TSDB_CODE_PAR_INVALID_PASSWD            TAOS_DEF_ERROR_CODE(0, 0x2648)
+#define TSDB_CODE_PAR_INVALID_ALTER_TABLE       TAOS_DEF_ERROR_CODE(0, 0x2649)
+#define TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY   TAOS_DEF_ERROR_CODE(0, 0x264A)
+#define TSDB_CODE_PAR_INVALID_MODIFY_COL        TAOS_DEF_ERROR_CODE(0, 0x264B)
+#define TSDB_CODE_PAR_INVALID_TBNAME            TAOS_DEF_ERROR_CODE(0, 0x264C)
+#define TSDB_CODE_PAR_INVALID_FUNCTION_NAME     TAOS_DEF_ERROR_CODE(0, 0x264D)
+#define TSDB_CODE_PAR_COMMENT_TOO_LONG          TAOS_DEF_ERROR_CODE(0, 0x264E)
+#define TSDB_CODE_PAR_NOT_ALLOWED_FUNC          TAOS_DEF_ERROR_CODE(0, 0x264F)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
@@ -656,6 +671,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_UDF_LOAD_UDF_FAILURE          TAOS_DEF_ERROR_CODE(0, 0x2905)
 #define TSDB_CODE_UDF_INVALID_STATE             TAOS_DEF_ERROR_CODE(0, 0x2906)
 #define TSDB_CODE_UDF_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0x2907)
+#define TSDB_CODE_UDF_NO_FUNC_HANDLE            TAOS_DEF_ERROR_CODE(0, 0x2908)
+#define TSDB_CODE_UDF_INVALID_BUFSIZE           TAOS_DEF_ERROR_CODE(0, 0x2909)
+#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE       TAOS_DEF_ERROR_CODE(0, 0x290A)
 
 #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
 #define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)

include/util/tcoding.h

@@ -63,14 +63,14 @@ static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return
 
 static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
   if (buf != NULL) {
-    ((int8_t *)(*buf))[0] = value ? 1 : 0;
+    ((int8_t *)(*buf))[0] = (value ? 1 : 0);
     *buf = POINTER_SHIFT(*buf, sizeof(int8_t));
   }
   return (int32_t)sizeof(int8_t);
 }
 
 static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
-  *value = ((int8_t *)buf)[0] == 0 ? false : true;
+  *value = ( (((int8_t *)buf)[0] == 0) ? false : true );
   return POINTER_SHIFT(buf, sizeof(int8_t));
 }
 

include/util/tdef.h

@@ -218,8 +218,8 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_SQL_SHOW_LEN    1024
 #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u)  // sql length should be less than 1mb
 
-#define TSDB_APP_NAME_LEN    TSDB_UNI_LEN
-#define TSDB_STB_COMMENT_LEN 1024
+#define TSDB_APP_NAME_LEN   TSDB_UNI_LEN
+#define TSDB_TB_COMMENT_LEN 1025
 
 /**
  *  In some scenarios uint16_t (0~65535) is used to store the row len.
@@ -425,9 +425,12 @@ enum {
   SND_WORKER_TYPE__UNIQUE,
 };
 
-#define MNODE_HANDLE   -1
-#define QNODE_HANDLE   1
 #define DEFAULT_HANDLE 0
+#define MNODE_HANDLE   -1
+#define QNODE_HANDLE   -2
+#define SNODE_HANDLE   -3
+#define VNODE_HANDLE   -4
+#define BNODE_HANDLE   -5
 
 #define TSDB_CONFIG_OPTION_LEN 16
 #define TSDB_CONIIG_VALUE_LEN  48

include/util/tencode.h

@@ -18,7 +18,6 @@
 
 #include "tcoding.h"
 #include "tlist.h"
-// #include "tfreelist.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -457,52 +456,189 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
   return p;
 }
 
-static FORCE_INLINE int32_t tPutBinary(uint8_t* p, const uint8_t* pData, uint32_t nData) {
-  int      n = 0;
-  uint32_t v = nData;
+// ===========================================
+#define tPutV(p, v)                    \
+  do {                                 \
+    int32_t n = 0;                     \
+    for (;;) {                         \
+      if (v <= 0x7f) {                 \
+        if (p) p[n] = v;               \
+        n++;                           \
+        break;                         \
+      }                                \
+      if (p) p[n] = (v & 0x7f) | 0x80; \
+      n++;                             \
+      v >>= 7;                         \
+    }                                  \
+    return n;                          \
+  } while (0)
 
-  for (;;) {
-    if (v <= 0x7f) {
-      if (p) p[n] = v;
-      n++;
-      break;
-    }
+#define tGetV(p, v)                              \
+  do {                                           \
+    int32_t n = 0;                               \
+    if (v) *v = 0;                               \
+    for (;;) {                                   \
+      if (p[n] <= 0x7f) {                        \
+        if (v) (*v) |= (p[n] << (7 * n));        \
+        n++;                                     \
+        break;                                   \
+      }                                          \
+      if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \
+      n++;                                       \
+    }                                            \
+    return n;                                    \
+  } while (0)
 
-    if (p) p[n] = (v & 0x7f) | 0x80;
-    n++;
-    v >>= 7;
-  }
+// PUT
+static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) {
+  if (p) ((uint8_t*)p)[0] = v;
+  return sizeof(uint8_t);
+}
 
-  if (p) {
-    memcpy(p + n, pData, nData);
-  }
+static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) {
+  if (p) ((int8_t*)p)[0] = v;
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) {
+  if (p) ((uint16_t*)p)[0] = v;
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) {
+  if (p) ((int16_t*)p)[0] = v;
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) {
+  if (p) ((uint32_t*)p)[0] = v;
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) {
+  if (p) ((int32_t*)p)[0] = v;
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) {
+  if (p) ((uint64_t*)p)[0] = v;
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
+  if (p) ((int64_t*)p)[0] = v;
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); }
+
+static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); }
+
+static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); }
+
+static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); }
+
+// GET
+static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) {
+  if (v) *v = ((uint8_t*)p)[0];
+  return sizeof(uint8_t);
+}
+
+static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) {
+  if (v) *v = ((int8_t*)p)[0];
+  return sizeof(int8_t);
+}
+
+static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) {
+  if (v) *v = ((uint16_t*)p)[0];
+  return sizeof(uint16_t);
+}
+
+static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) {
+  if (v) *v = ((int16_t*)p)[0];
+  return sizeof(int16_t);
+}
+
+static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) {
+  if (v) *v = ((uint32_t*)p)[0];
+  return sizeof(uint32_t);
+}
+
+static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) {
+  if (v) *v = ((int32_t*)p)[0];
+  return sizeof(int32_t);
+}
+
+static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) {
+  if (v) *v = ((uint64_t*)p)[0];
+  return sizeof(uint64_t);
+}
+
+static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) {
+  if (v) *v = ((int64_t*)p)[0];
+  return sizeof(int64_t);
+}
+
+static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) {
+  int32_t  n;
+  uint16_t tv;
+
+  n = tGetU16v(p, &tv);
+  if (v) *v = ZIGZAGD(int16_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) {
+  int32_t  n;
+  uint32_t tv;
+
+  n = tGetU32v(p, &tv);
+  if (v) *v = ZIGZAGD(int32_t, tv);
+
+  return n;
+}
+
+static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); }
+
+static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) {
+  int32_t  n;
+  uint64_t tv;
+
+  n = tGetU64v(p, &tv);
+  if (v) *v = ZIGZAGD(int64_t, tv);
+
+  return n;
+}
+
+// =====================
+static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) {
+  int n = 0;
+
+  n += tPutU32v(p ? p + n : p, nData);
+  if (p) memcpy(p + n, pData, nData);
   n += nData;
 
   return n;
 }
 
-static FORCE_INLINE int32_t tGetBinary(const uint8_t* p, const uint8_t** ppData, uint32_t* nData) {
+static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) {
   int32_t  n = 0;
-  uint32_t tv = 0;
-  uint32_t t;
-
-  for (;;) {
-    if (p[n] <= 0x7f) {
-      t = p[n];
-      tv |= (t << (7 * n));
-      n++;
-      break;
-    }
-
-    t = p[n] & 0x7f;
-    tv |= (t << (7 * n));
-    n++;
-  }
+  uint32_t nt;
 
-  if (nData) *nData = n;
+  n += tGetU32v(p, &nt);
+  if (nData) *nData = nt;
   if (ppData) *ppData = p + n;
+  n += nt;
 
-  n += tv;
   return n;
 }
 

include/util/thash.h

@@ -40,6 +40,7 @@ typedef void (*_hash_free_fn_t)(void *);
  */
 uint32_t MurmurHash3_32(const char *key, uint32_t len);
 
+uint64_t MurmurHash3_64(const char *key, uint32_t len);
 /**
  *
  * @param key

include/util/tjson.h

@@ -22,17 +22,12 @@
 extern "C" {
 #endif
 
-#ifdef WINDOWS
-#define tjsonGetNumberValue(pJson, pName, val) -1
-#else
-#define tjsonGetNumberValue(pJson, pName, val) \
-  ({ \
-    uint64_t _tmp = 0; \
-    int32_t _code = tjsonGetUBigIntValue(pJson, pName, &_tmp); \
-    val = _tmp; \
-    _code; \
-  })
-#endif
+#define tjsonGetNumberValue(pJson, pName, val, code)  \
+  do {                                                \
+    uint64_t _tmp = 0;                                \
+    code = tjsonGetBigIntValue(pJson, pName, &_tmp); \
+    val = _tmp;                                       \
+  } while (0)
 
 typedef void SJson;
 

include/util/tlog.h

@@ -61,6 +61,7 @@ extern int32_t tqDebugFlag;
 extern int32_t fsDebugFlag;
 extern int32_t metaDebugFlag;
 extern int32_t fnDebugFlag;
+extern int32_t smaDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

include/util/tprocess.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_PROCESS_H_
-#define _TD_UTIL_PROCESS_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef enum { PROC_FUNC_REQ = 1, PROC_FUNC_RSP, PROC_FUNC_REGIST, PROC_FUNC_RELEASE } EProcFuncType;
-
-typedef struct SProcObj SProcObj;
-typedef void *(*ProcMallocFp)(int32_t contLen);
-typedef void *(*ProcFreeFp)(void *pCont);
-typedef void (*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
-                              EProcFuncType ftype);
-
-typedef struct {
-  ProcConsumeFp childConsumeFp;
-  ProcMallocFp  childMallocHeadFp;
-  ProcFreeFp    childFreeHeadFp;
-  ProcMallocFp  childMallocBodyFp;
-  ProcFreeFp    childFreeBodyFp;
-  ProcConsumeFp parentConsumeFp;
-  ProcMallocFp  parentMallocHeadFp;
-  ProcFreeFp    parentFreeHeadFp;
-  ProcMallocFp  parentMallocBodyFp;
-  ProcFreeFp    parentFreeBodyFp;
-  SShm          shm;
-  void         *parent;
-  const char   *name;
-  bool          isChild;
-} SProcCfg;
-
-SProcObj *taosProcInit(const SProcCfg *pCfg);
-void      taosProcCleanup(SProcObj *pProc);
-int32_t   taosProcRun(SProcObj *pProc);
-void      taosProcStop(SProcObj *pProc);
-
-int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                            void *handle, int64_t handleRef, EProcFuncType ftype);
-int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle);
-void    taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
-void    taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                             EProcFuncType ftype);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_UTIL_PROCESS_H_*/

include/util/tqueue.h

@@ -48,18 +48,24 @@ typedef struct {
   int32_t threadNum;
 } SQueueInfo;
 
+typedef enum {
+  DEF_QITEM = 0,
+  RPC_QITEM = 1,
+} EQItype;
+
 typedef void (*FItem)(SQueueInfo *pInfo, void *pItem);
 typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems);
 
 STaosQueue *taosOpenQueue();
 void        taosCloseQueue(STaosQueue *queue);
 void        taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
-void       *taosAllocateQitem(int32_t size);
+void       *taosAllocateQitem(int32_t size, EQItype itype);
 void        taosFreeQitem(void *pItem);
 void        taosWriteQitem(STaosQueue *queue, void *pItem);
 int32_t     taosReadQitem(STaosQueue *queue, void **ppItem);
 bool        taosQueueEmpty(STaosQueue *queue);
-int32_t     taosQueueSize(STaosQueue *queue);
+int32_t     taosQueueItemSize(STaosQueue *queue);
+int64_t     taosQueueMemorySize(STaosQueue *queue);
 
 STaosQall *taosAllocateQall();
 void       taosFreeQall(STaosQall *qall);
@@ -77,8 +83,8 @@ int32_t    taosGetQueueNumber(STaosQset *qset);
 int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp);
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
-int32_t taosGetQueueItemsNumber(STaosQueue *queue);
-int32_t taosGetQsetItemsNumber(STaosQset *qset);
+
+extern int64_t tsRpcQueueMemoryAllowed;
 
 #ifdef __cplusplus
 }

include/util/tskiplist.h

@@ -56,10 +56,10 @@ typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutS
 
 typedef struct SSkipList {
   uint32_t           seed;
+  uint16_t           len;
   __compar_fn_t      comparFn;
   __sl_key_fn_t      keyFn;
   TdThreadRwlock    *lock;
-  uint16_t           len;
   uint8_t            maxLevel;
   uint8_t            flags;
   uint8_t            type;  // static info above

packaging/cfg/nginxd.service

+[Unit]
+Description=Nginx For TDengine Service
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=forking
+PIDFile=/usr/local/nginxd/logs/nginx.pid
+ExecStart=/usr/local/nginxd/sbin/nginx
+ExecStop=/usr/local/nginxd/sbin/nginx -s stop
+TimeoutStopSec=1000000s
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+TimeoutStartSec=0
+StandardOutput=null
+Restart=always
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target

packaging/cfg/taos.cfg

+########################################################
+#                                                      #
+#                  TDengine Configuration              #
+#   Any questions, please email support@taosdata.com   #
+#                                                      #
+########################################################
+
+# first fully qualified domain name (FQDN) for TDengine system
+# firstEp                   hostname:6030
+
+# local fully qualified domain name (FQDN)
+# fqdn                      hostname
+
+# first port number for the connection (12 continuous UDP/TCP port number are used) 
+# serverPort                6030
+
+# log file's directory
+# logDir                    /var/log/taos
+
+# data file's directory
+# dataDir                   /var/lib/taos
+
+# temporary file's directory
+# tempDir                   /tmp/
+
+# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only   
+# arbitrator                arbitrator_hostname:6042     
+
+# number of threads per CPU core
+# numOfThreadsPerCore       1.0
+
+# number of threads to commit cache data
+# numOfCommitThreads        4
+
+# the proportion of total CPU cores available for query processing
+# 2.0: the query threads will be set to double of the CPU cores.
+# 1.0: all CPU cores are available for query processing [default].
+# 0.5: only half of the CPU cores are available for query.
+# 0.0: only one core available.
+# ratioOfQueryCores        1.0
+
+# the last_row/first/last aggregator will not change the original column name in the result fields
+keepColumnName            1
+
+# number of management nodes in the system
+# numOfMnodes               1
+
+# enable/disable backuping vnode directory when removing vnode
+# vnodeBak                  1
+
+# enable/disable installation / usage report
+# telemetryReporting        1
+
+# enable/disable load balancing
+# balance                   1
+
+# role for dnode. 0 - any, 1 - mnode, 2 - dnode
+# role                      0
+
+# max timer control blocks
+# maxTmrCtrl                512
+
+# time interval of system monitor, seconds 
+# monitorInterval           30
+
+# number of seconds allowed for a dnode to be offline, for cluster only 
+# offlineThreshold          864000
+
+# RPC re-try timer, millisecond
+# rpcTimer                  300
+
+# RPC maximum time for ack, seconds. 
+# rpcMaxTime                600
+
+# time interval of dnode status reporting to mnode, seconds, for cluster only 
+# statusInterval            1
+
+# time interval of heart beat from shell to dnode, seconds
+# shellActivityTimer        3
+
+# minimum sliding window time, milli-second
+# minSlidingTime            10
+
+# minimum time window, milli-second
+# minIntervalTime           10
+
+# maximum delay before launching a stream computation, milli-second
+# maxStreamCompDelay        20000
+
+# maximum delay before launching a stream computation for the first time, milli-second
+# maxFirstStreamCompDelay   10000
+
+# retry delay when a stream computation fails, milli-second
+# retryStreamCompDelay      10
+
+# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9
+# streamCompDelayRatio      0.1
+
+# max number of vgroups per db, 0 means configured automatically
+# maxVgroupsPerDb           0
+
+# max number of tables per vnode
+# maxTablesPerVnode         1000000
+
+# cache block size (Mbyte)
+# cache                     16
+
+# number of cache blocks per vnode
+# blocks                    6
+
+# number of days per DB file
+# days                  10
+
+# number of days to keep DB file
+# keep                  3650
+
+# minimum rows of records in file block
+# minRows               100
+
+# maximum rows of records in file block
+# maxRows               4096
+
+# the number of acknowledgments required for successful data writing
+# quorum                1     
+
+# enable/disable compression
+# comp                  2
+
+# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync
+# walLevel              1
+
+# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away
+# fsync                 3000
+
+# number of replications, for cluster only 
+# replica               1
+
+# the compressed rpc message, option:
+#  -1 (no compression)
+#   0 (all message compressed),
+# > 0 (rpc message body which larger than this value will be compressed)
+# compressMsgSize       -1
+
+# query retrieved column data compression option:
+#  -1 (no compression)
+#   0 (all retrieved column data compressed),
+# > 0 (any retrieved column size greater than this value all data will be compressed.)
+# compressColData       -1
+
+# max length of an SQL
+# maxSQLLength          65480
+
+# max length of WildCards
+# maxWildCardsLength    100
+
+# the maximum number of records allowed for super table time sorting
+# maxNumOfOrderedRes    100000
+
+# system time zone
+# timezone              Asia/Shanghai (CST, +0800)
+# system time zone (for windows 10)
+# timezone              UTC-8
+
+# system locale
+# locale                en_US.UTF-8
+
+# default system charset
+# charset               UTF-8
+
+# max number of connections allowed in dnode
+# maxShellConns         5000
+
+# max number of connections allowed in client
+# maxConnections        5000
+
+# stop writing logs when the disk size of the log folder is less than this value
+# minimalLogDirGB       1.0
+
+# stop writing temporary files when the disk size of the tmp folder is less than this value
+# minimalTmpDirGB       1.0
+
+# if disk free space is less than this value, taosd service exit directly within startup process
+# minimalDataDirGB      2.0
+
+# One mnode is equal to the number of vnode consumed
+# mnodeEqualVnodeNum    4
+
+# enbale/disable http service
+# http                  1
+
+# enable/disable system monitor 
+# monitor               1
+
+# enable/disable recording the SQL statements via restful interface
+# httpEnableRecordSql   0
+
+# number of threads used to process http requests
+# httpMaxThreads        2
+
+# maximum number of rows returned by the restful interface
+# restfulRowLimit       10240
+
+# database name must be specified in restful interface if the following parameter is set, off by default
+# httpDbNameMandatory 1
+
+# http keep alive, default is 30 seconds
+# httpKeepAlive   30000
+
+# The following parameter is used to limit the maximum number of lines in log files.
+# max number of lines per log filters
+# numOfLogLines         10000000
+
+# enable/disable async log
+# asyncLog              1
+
+# time of keeping log files, days
+# logKeepDays           0
+
+
+# The following parameters are used for debug purpose only.
+# debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
+# 131: output warning and error 
+# 135: output debug, warning and error
+# 143: output trace, debug, warning and error to log
+# 199: output debug, warning and error to both screen and file
+# 207: output trace, debug, warning and error to both screen and file
+
+# debug flag for all log type, take effect when non-zero value
+# debugFlag             0
+
+# debug flag for meta management messages
+# mDebugFlag            135
+
+# debug flag for dnode messages
+# dDebugFlag            135
+
+# debug flag for sync module
+# sDebugFlag            135
+
+# debug flag for WAL
+# wDebugFlag            135
+
+# debug flag for SDB
+# sdbDebugFlag          135
+
+# debug flag for RPC 
+# rpcDebugFlag          131
+
+# debug flag for TAOS TIMER
+# tmrDebugFlag          131
+
+# debug flag for TDengine client 
+# cDebugFlag            131
+
+# debug flag for JNI
+# jniDebugFlag          131
+
+# debug flag for storage
+# uDebugFlag            131
+
+# debug flag for http server
+# httpDebugFlag         131
+
+# debug flag for monitor
+# monDebugFlag          131
+
+# debug flag for query
+# qDebugFlag            131
+
+# debug flag for vnode
+# vDebugFlag            131
+
+# debug flag for TSDB
+# tsdbDebugFlag         131
+
+# debug flag for continue query
+# cqDebugFlag           131
+
+# enable/disable recording the SQL in taos client
+# enableRecordSql    0
+
+# generate core file when service crash
+# enableCoreFile        1
+
+# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
+# maxBinaryDisplayWidth 30
+
+# enable/disable stream (continuous query)
+# stream                1
+
+# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode
+# retrieveBlockingModel    0
+
+# the maximum allowed query buffer size in MB during query processing for each data node
+# -1 no limit (default)
+# 0  no query allowed, queries are disabled
+# queryBufferSize         -1
+
+# percent of redundant data in tsdb meta will compact meta data,0 means donot compact
+# tsdbMetaCompactRatio    0
+
+# default string type used for storing JSON String, options can be binary/nchar, default is nchar
+# defaultJSONStrType      nchar
+
+# force TCP transmission 
+# rpcForceTcp        0
+
+# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks
+# walFlushSize         1024
+
+# unit Hour. Latency of data migration
+# keepTimeOffset     0

packaging/cfg/taosd.service

+[Unit]
+Description=TDengine server service
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/taosd
+ExecStartPre=/usr/local/taos/bin/startPre.sh
+TimeoutStopSec=1000000s
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+TimeoutStartSec=0
+StandardOutput=null
+Restart=always
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target

packaging/cfg/tarbitratord.service

+[Unit]
+Description=TDengine arbitrator service
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/tarbitrator
+TimeoutStopSec=1000000s
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+TimeoutStartSec=0
+StandardOutput=null
+Restart=always
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target

packaging/check_package.sh

+#!/bin/bash
+#
+# This file is used to install database on linux systems. The operating system
+# is required to use systemd to manage services at boot
+
+set -e
+#set -x
+
+verMode=edge
+pagMode=full
+
+iplist=""
+serverFqdn=""
+
+# -----------------------Variables definition---------------------
+script_dir="../release"
+# Dynamic directory
+data_dir="/var/lib/taos"
+log_dir="/var/log/taos"
+
+data_link_dir="/usr/local/taos/data"
+log_link_dir="/usr/local/taos/log"
+
+cfg_install_dir="/etc/taos"
+
+bin_link_dir="/usr/bin"
+lib_link_dir="/usr/lib"
+lib64_link_dir="/usr/lib64"
+inc_link_dir="/usr/include"
+
+#install main path
+install_main_dir="/usr/local/taos"
+
+# old bin dir
+sbin_dir="/usr/local/taos/bin"
+
+temp_version=""
+fin_result=""
+
+service_config_dir="/etc/systemd/system"
+nginx_port=6060
+nginx_dir="/usr/local/nginxd"
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo "
+fi
+
+# =============================  get input parameters =================================================
+
+# install.sh -v [server | client]  -e [yes | no] -i [systemd | service | ...]
+
+# set parameters by default value
+interactiveFqdn=yes   # [yes | no]
+verType=server        # [server | client]
+initType=systemd      # [systemd | service | ...]
+
+while getopts "hv:d:" arg
+do
+  case $arg in
+    d)
+      #echo "interactiveFqdn=$OPTARG"
+      script_dir=$( echo $OPTARG )
+      ;;
+    h)
+      echo "Usage: `basename $0` -d scripy_path"
+      exit 0
+      ;;
+    ?) #unknow option
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}"
+
+function kill_process() {
+  pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo}kill -9 $pid   || :
+  fi
+}
+
+function check_file() {
+    #check file whether exists
+    if [ ! -e $1/$2 ];then
+        echo -e "$1/$2 \033[31mnot exists\033[0m!quit"
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8
+    fi
+}
+
+function get_package_name() {
+  var=$1
+	if [[ $1 =~ 'aarch' ]];then
+		echo ${var::-21}
+	else
+		echo ${var::-17}
+	fi
+}
+
+function check_link() {
+    #check Link whether exists or broken
+    if [ -L $1 ] ; then
+        if [ ! -e $1 ] ; then
+            echo -e "$1 \033[31Broken link\033[0m"
+            fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+            echo -e $fin_result
+            exit 8
+        fi
+    else
+        echo -e "$1 \033[31mnot exists\033[0m!quit"
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8
+    fi
+}
+
+function check_main_path() {
+    #check install main dir and all sub dir
+    main_dir=("" "cfg" "bin" "connector" "driver" "examples" "include" "init.d")
+    for i in "${main_dir[@]}";do
+        check_file ${install_main_dir} $i
+    done
+    if [ "$verMode" == "cluster" ]; then
+        nginx_main_dir=("admin" "conf" "html" "sbin" "logs")
+        for i in "${nginx_main_dir[@]}";do
+            check_file ${nginx_dir}  $i
+        done
+    fi
+    echo -e "Check main path:\033[32mOK\033[0m!"
+}
+
+function check_bin_path() {
+    # check install bin dir and all sub dir
+    bin_dir=("taos" "taosd" "taosadapter" "taosdemo" "remove.sh" "tarbitrator" "set_core.sh")
+    for i in "${bin_dir[@]}";do
+        check_file ${sbin_dir} $i
+    done
+    lbin_dir=("taos" "taosd" "taosadapter" "taosdemo" "rmtaos" "tarbitrator" "set_core")
+    for i in "${lbin_dir[@]}";do
+        check_link ${bin_link_dir}/$i
+    done
+    if [ "$verMode" == "cluster" ]; then
+        check_file ${nginx_dir}/sbin nginx
+    fi
+    echo -e "Check bin  path:\033[32mOK\033[0m!"
+}
+
+function check_lib_path() {
+    # check all links
+    check_link ${lib_link_dir}/libtaos.so
+    check_link ${lib_link_dir}/libtaos.so.1
+
+    if [[ -d ${lib64_link_dir}  ]]; then
+        check_link ${lib64_link_dir}/libtaos.so
+        check_link ${lib64_link_dir}/libtaos.so.1
+    fi
+    echo -e "Check lib  path:\033[32mOK\033[0m!"
+}
+
+function check_header_path() {
+	# check all header
+	header_dir=("taos.h" "taosdef.h" "taoserror.h")
+    for i in "${header_dir[@]}";do
+        check_link ${inc_link_dir}/$i
+    done
+    echo -e "Check bin  path:\033[32mOK\033[0m!"
+}
+
+function check_taosadapter_config_dir() {
+	# check all config
+	check_file ${cfg_install_dir} taosadapter.toml
+	check_file ${cfg_install_dir} taosadapter.service
+	check_file ${install_main_dir}/cfg taosadapter.toml.org
+	echo -e "Check conf path:\033[32mOK\033[0m!"
+}
+
+function check_config_dir() {
+	# check all config
+	check_file ${cfg_install_dir} taos.cfg
+	check_file ${install_main_dir}/cfg taos.cfg.org
+	echo -e "Check conf path:\033[32mOK\033[0m!"
+}
+
+function check_log_path() {
+	# check log path
+	check_file ${log_dir}
+	echo -e "Check log  path:\033[32mOK\033[0m!"
+}
+
+function check_data_path() {
+	# check data path
+	check_file ${data_dir}
+	echo -e "Check data path:\033[32mOK\033[0m!"
+}
+
+function install_TDengine() {
+	cd ${script_dir}
+	tar zxf $1
+  temp_version=$(get_package_name $1)
+	cd $(get_package_name $1)
+  echo -e "\033[32muninstall TDengine && install TDengine...\033[0m"
+	rmtaos >/dev/null 2>&1 || echo 'taosd not installed' && echo -e '\n\n' |./install.sh >/dev/null 2>&1
+  echo -e "\033[32mTDengine has been installed!\033[0m"
+  echo -e "\033[32mTDengine is starting...\033[0m"
+  kill_process taos && systemctl start taosd && sleep 10
+}
+
+function test_TDengine() {
+    check_main_path
+    check_bin_path
+    check_lib_path
+    check_header_path
+    check_config_dir
+    check_taosadapter_config_dir
+    check_log_path
+    check_data_path
+    result=`taos -s 'create database test ;create table test.tt(ts timestamp ,i int);insert into test.tt values(now,11);select * from test.tt' 2>&1 ||:`
+    if [[ $result =~ "Unable to establish" ]];then
+        echo -e "\033[31mTDengine connect failed\033[0m"
+        fin_result=$fin_result"\033[31m$temp_version\033[0m test failed!\n"
+        echo -e $fin_result
+        exit 8
+    fi
+    echo -e "Check TDengine connect:\033[32mOK\033[0m!"
+    fin_result=$fin_result"\033[32m$temp_version\033[0m test OK!\n"
+}
+# ## ==============================Main program starts from here============================
+TD_package_name=`ls ${script_dir}/*server*gz |awk -F '/' '{print $NF}' `
+temp=`pwd`
+for i in $TD_package_name;do
+	if [[ $i =~ 'enterprise' ]];then
+		verMode="cluster"
+	else
+		verMode=""
+	fi
+  cd $temp
+	install_TDengine $i
+	test_TDengine
+done
+echo "============================================================"
+echo -e $fin_result

packaging/deb/DEBIAN/control

+Package: tdengine
+Version: 1.0.0
+Section: utils
+Priority: optional
+#Essential: no
+#Depends: no
+#Suggests: no
+Architecture: amd64
+Installed-Size: 66666
+Maintainer: support@taosdata.com
+Provides: taosdata
+Homepage: http://taosdata.com
+Description: Big Data Platform Designed and Optimized for IoT.

packaging/deb/DEBIAN/postinst

+#!/bin/bash
+#set -x
+#path=`pwd`
+insmetaPath="/usr/local/taos/script"
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo "
+fi
+
+${csudo}chmod -R 744 ${insmetaPath}
+cd ${insmetaPath}
+${csudo}./post.sh

packaging/deb/DEBIAN/postrm

+#!/bin/bash
+

packaging/deb/DEBIAN/preinst

+#!/bin/bash
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo "
+fi
+
+# Stop the service if running
+if pidof taosd &> /dev/null; then
+    if pidof systemd &> /dev/null; then
+        ${csudo}systemctl stop taosd || :
+    elif $(which service  &> /dev/null); then
+        ${csudo}service taosd stop || :
+    else
+        pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
+        if [ -n "$pid" ]; then
+           ${csudo}kill -9 $pid   || :
+        fi
+    fi
+    echo "Stop taosd service success!"
+    sleep 1
+fi
+
+# if taos.cfg already softlink, remove it
+cfg_install_dir="/etc/taos"
+install_main_dir="/usr/local/taos"
+if [ -f "${install_main_dir}/taos.cfg" ]; then
+    ${csudo}rm -f ${install_main_dir}/cfg/taos.cfg   || :
+fi
+
+if [ -f "${install_main_dir}/taosadapter.toml" ]; then
+    ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.toml     || :
+fi
+
+if [ -f "${install_main_dir}/taosadapter.service" ]; then
+    ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || :
+fi
+
+# there can not libtaos.so*, otherwise ln -s  error
+${csudo}rm -f ${install_main_dir}/driver/libtaos*   || :

packaging/deb/DEBIAN/prerm

+#!/bin/bash
+
+insmetaPath="/usr/local/taos/script"
+
+csudo=""
+if command -v sudo > /dev/null; then
+    csudo="sudo "
+fi
+
+${csudo}chmod -R 744 ${insmetaPath}  || :
+#cd ${insmetaPath}
+#${csudo}./preun.sh
+if [ -f ${insmetaPath}/preun.sh ]; then
+    cd ${insmetaPath}
+    ${csudo}./preun.sh
+else
+    bin_link_dir="/usr/bin"
+    lib_link_dir="/usr/lib"
+    inc_link_dir="/usr/include"
+
+    data_link_dir="/usr/local/taos/data"
+    log_link_dir="/usr/local/taos/log"
+    cfg_link_dir="/usr/local/taos/cfg"
+
+    # Remove all links
+    ${csudo}rm -f ${bin_link_dir}/taos       || :
+    ${csudo}rm -f ${bin_link_dir}/taosd      || :
+    ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
+    ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
+    ${csudo}rm -f ${cfg_link_dir}/*          || :
+    ${csudo}rm -f ${inc_link_dir}/taos.h     || :
+    ${csudo}rm -f ${lib_link_dir}/libtaos.*  || :
+
+    ${csudo}rm -f ${log_link_dir}            || :
+    ${csudo}rm -f ${data_link_dir}           || :
+
+    pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}')
+    if [ -n "$pid" ]; then
+        ${csudo}kill -9 $pid   || :
+    fi
+fi
+

packaging/deb/makedeb.sh

+#!/bin/bash
+#
+# Generate deb package for ubuntu
+set -e
+# set -x
+
+#curr_dir=$(pwd)
+compile_dir=$1
+output_dir=$2
+tdengine_ver=$3
+cpuType=$4
+osType=$5
+verMode=$6
+verType=$7
+
+script_dir="$(dirname $(readlink -f $0))"
+top_dir="$(readlink -f ${script_dir}/../..)"
+pkg_dir="${top_dir}/debworkroom"
+
+#echo "curr_dir: ${curr_dir}"
+#echo "top_dir: ${top_dir}"
+#echo "script_dir: ${script_dir}"
+echo "compile_dir: ${compile_dir}"
+echo "pkg_dir: ${pkg_dir}"
+
+if [ -d ${pkg_dir} ]; then
+    rm -rf ${pkg_dir}
+fi
+mkdir -p ${pkg_dir}
+cd ${pkg_dir}
+
+libfile="libtaos.so.${tdengine_ver}"
+
+# create install dir
+install_home_path="/usr/local/taos"
+mkdir -p ${pkg_dir}${install_home_path}
+mkdir -p ${pkg_dir}${install_home_path}/bin
+mkdir -p ${pkg_dir}${install_home_path}/cfg
+#mkdir -p ${pkg_dir}${install_home_path}/connector
+mkdir -p ${pkg_dir}${install_home_path}/driver
+mkdir -p ${pkg_dir}${install_home_path}/examples
+mkdir -p ${pkg_dir}${install_home_path}/include
+#mkdir -p ${pkg_dir}${install_home_path}/init.d
+mkdir -p ${pkg_dir}${install_home_path}/script
+
+cp ${compile_dir}/../packaging/cfg/taos.cfg         ${pkg_dir}${install_home_path}/cfg
+if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
+    cp ${compile_dir}/test/cfg/taosadapter.toml		${pkg_dir}${install_home_path}/cfg || :
+fi
+if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
+    cp ${compile_dir}/test/cfg/taosadapter.service	${pkg_dir}${install_home_path}/cfg || :
+fi
+
+#cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_path}/init.d
+cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
+cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
+cp ${compile_dir}/../packaging/tools/startPre.sh    ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/../packaging/tools/set_core.sh    ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb    ${pkg_dir}${install_home_path}/bin
+
+cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
+#cp ${compile_dir}/build/bin/taosBenchmark           ${pkg_dir}${install_home_path}/bin
+
+if [ -f "${compile_dir}/build/bin/taosadapter" ]; then
+    cp ${compile_dir}/build/bin/taosadapter                    ${pkg_dir}${install_home_path}/bin ||:
+fi
+
+cp ${compile_dir}/build/bin/taos                    ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_path}/driver
+cp ${compile_dir}/../include/client/taos.h          ${pkg_dir}${install_home_path}/include
+cp ${compile_dir}/../include/common/taosdef.h       ${pkg_dir}${install_home_path}/include
+cp ${compile_dir}/../include/util/taoserror.h       ${pkg_dir}${install_home_path}/include
+cp -r ${top_dir}/examples/*                         ${pkg_dir}${install_home_path}/examples
+#cp -r ${top_dir}/src/connector/python               ${pkg_dir}${install_home_path}/connector
+#cp -r ${top_dir}/src/connector/go                   ${pkg_dir}${install_home_path}/connector
+#cp -r ${top_dir}/src/connector/nodejs               ${pkg_dir}${install_home_path}/connector
+#cp ${compile_dir}/build/lib/taos-jdbcdriver*.*  ${pkg_dir}${install_home_path}/connector ||:
+
+install_user_local_path="/usr/local"
+
+if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then
+    mkdir -p ${pkg_dir}${install_user_local_path}/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3}
+    cp ${compile_dir}/build/bin/jemalloc-config ${pkg_dir}${install_user_local_path}/bin/
+    if [ -f ${compile_dir}/build/bin/jemalloc.sh ]; then
+        cp ${compile_dir}/build/bin/jemalloc.sh ${pkg_dir}${install_user_local_path}/bin/
+    fi
+    if [ -f ${compile_dir}/build/bin/jeprof ]; then
+        cp ${compile_dir}/build/bin/jeprof ${pkg_dir}${install_user_local_path}/bin/
+    fi
+    if [ -f ${compile_dir}/build/include/jemalloc/jemalloc.h ]; then
+        cp ${compile_dir}/build/include/jemalloc/jemalloc.h ${pkg_dir}${install_user_local_path}/include/jemalloc/
+    fi
+    if [ -f ${compile_dir}/build/lib/libjemalloc.so.2 ]; then
+        cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/
+        ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so
+    fi
+    if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then
+        cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/
+    fi
+    if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then
+        cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/
+    fi
+    if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then
+        cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/
+    fi
+    if [ -f ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ]; then
+        cp ${compile_dir}/build/share/doc/jemalloc/jemalloc.html ${pkg_dir}${install_user_local_path}/share/doc/jemalloc/
+    fi
+    if [ -f ${compile_dir}/build/share/man/man3/jemalloc.3 ]; then
+        cp ${compile_dir}/build/share/man/man3/jemalloc.3 ${pkg_dir}${install_user_local_path}/share/man/man3/
+    fi
+fi
+
+cp -r ${compile_dir}/../packaging/deb/DEBIAN        ${pkg_dir}/
+chmod 755 ${pkg_dir}/DEBIAN/*
+
+# modify version of control
+debver="Version: "$tdengine_ver
+sed -i "2c$debver" ${pkg_dir}/DEBIAN/control
+
+#get taos version, then set deb name
+if [ "$verMode" == "cluster" ]; then
+  debname="TDengine-server-"${tdengine_ver}-${osType}-${cpuType}
+elif [ "$verMode" == "edge" ]; then
+  debname="TDengine-server"-${tdengine_ver}-${osType}-${cpuType}
+else
+  echo "unknow verMode, nor cluster or edge"
+  exit 1
+fi
+
+if [ "$verType" == "beta" ]; then
+  debname="TDengine-server-"${tdengine_ver}-${verType}-${osType}-${cpuType}".deb"
+elif [ "$verType" == "stable" ]; then
+  debname=${debname}".deb"
+else
+  echo "unknow verType, nor stabel or beta"
+  exit 1
+fi
+
+# make deb package
+dpkg -b ${pkg_dir} $debname
+echo "make deb package success!"
+
+cp ${pkg_dir}/*.deb ${output_dir}
+
+# clean temp dir
+rm -rf ${pkg_dir}

packaging/deb/taosd

+#!/bin/bash
+#
+# Modified from original source: Elastic Search
+# https://github.com/elasticsearch/elasticsearch
+# Thank you to the Elastic Search authors
+#
+# chkconfig: 2345 99 01
+#
+### BEGIN INIT INFO
+# Provides:          TDengine
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:     $local_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts TDengine taosd
+# Description:       Starts TDengine taosd, a time-series database engine
+### END INIT INFO
+
+set -e
+
+PATH="/bin:/usr/bin:/sbin:/usr/sbin"
+NAME="TDengine"
+USER="root"
+GROUP="root"
+DAEMON="/usr/local/taos/bin/taosd"
+DAEMON_OPTS=""
+
+HTTPD_NAME="taosadapter"
+DAEMON_HTTPD_NAME=$HTTPD_NAME
+DAEMON_HTTPD="/usr/local/taos/bin/$HTTPD_NAME"
+
+PID_FILE="/var/run/$NAME.pid"
+APPARGS=""
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+. /lib/lsb/init-functions
+
+case "$1" in
+    start)
+
+        log_action_begin_msg "Starting TDengine..."
+        $DAEMON_HTTPD &
+        if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then
+
+            touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE"
+
+            if [ -n "$MAX_OPEN_FILES" ]; then
+                ulimit -n $MAX_OPEN_FILES
+            fi
+
+            start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS
+
+            log_end_msg $?
+        fi
+        ;;
+
+    stop)
+        log_action_begin_msg "Stopping TDengine..."
+        pkill -9 $DAEMON_HTTPD_NAME
+        set +e
+        if [ -f "$PID_FILE" ]; then
+            start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null
+            if [ $? -eq 1 ]; then
+                log_action_cont_msg "TSD is not running but pid file exists, cleaning up"
+            elif [ $? -eq 3 ]; then
+                PID="`cat $PID_FILE`"
+                log_failure_msg "Failed to stop TDengine (pid $PID)"
+                exit 1
+            fi
+            rm -f "$PID_FILE"
+        else
+            log_action_cont_msg "TDengine was not running"
+        fi
+        log_action_end_msg 0
+        set -e
+        ;;
+
+    restart|force-reload)
+        if [ -f "$PID_FILE" ]; then
+            $0 stop
+            sleep 1
+        fi
+        $0 start
+        ;;
+    status)
+        status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME"
+        ;;
+    *)
+        exit 1
+        ;;
+esac
+
+exit 0

packaging/deb/tarbitratord

+#!/bin/bash 
+#
+# Modified from original source: Elastic Search
+# https://github.com/elasticsearch/elasticsearch
+# Thank you to the Elastic Search authors
+#
+# chkconfig: 2345 99 01
+#
+### BEGIN INIT INFO
+# Provides:          taoscluster
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:     $local_fs $network $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Starts taoscluster tarbitrator
+# Description:       Starts taoscluster tarbitrator, a arbitrator
+### END INIT INFO
+
+set -e
+
+PATH="/bin:/usr/bin:/sbin:/usr/sbin"
+NAME="taoscluster"
+USER="root"
+GROUP="root"
+DAEMON="/usr/local/taos/bin/tarbitrator"
+DAEMON_OPTS=""
+PID_FILE="/var/run/$NAME.pid"
+APPARGS=""
+
+# Maximum number of open files
+MAX_OPEN_FILES=65535
+
+. /lib/lsb/init-functions
+
+case "$1" in
+    start)
+
+        log_action_begin_msg "Starting tarbitrator..."
+        if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then
+
+            touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE"
+
+            if [ -n "$MAX_OPEN_FILES" ]; then
+                ulimit -n $MAX_OPEN_FILES
+            fi
+
+            start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS
+
+            log_end_msg $?
+        fi
+        ;;
+
+    stop)
+        log_action_begin_msg "Stopping tarbitrator..."
+        set +e
+        if [ -f "$PID_FILE" ]; then
+            start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null
+            if [ $? -eq 1 ]; then
+                log_action_cont_msg "TSD is not running but pid file exists, cleaning up"
+            elif [ $? -eq 3 ]; then
+                PID="`cat $PID_FILE`"
+                log_failure_msg "Failed to stop tarbitrator (pid $PID)"
+                exit 1
+            fi
+            rm -f "$PID_FILE"
+        else
+            log_action_cont_msg "tarbitrator was not running"
+        fi
+        log_action_end_msg 0
+        set -e
+        ;;
+
+    restart|force-reload)
+        if [ -f "$PID_FILE" ]; then
+            $0 stop
+            sleep 1
+        fi
+        $0 start
+        ;;
+    status)
+        status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME"
+        ;;
+    *)
+        exit 1
+        ;;
+esac
+
+exit 0

packaging/docker/Dockerfile

+FROM ubuntu:18.04
+
+WORKDIR /root
+
+ARG pkgFile
+ARG dirName
+ARG cpuType
+RUN echo ${pkgFile} && echo ${dirName}
+
+COPY ${pkgFile} /root/
+RUN tar -zxf ${pkgFile}
+WORKDIR /root/
+RUN cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root
+RUN rm /root/${pkgFile}
+RUN rm -rf /root/${dirName}
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get clean && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \
+    LC_CTYPE=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8
+
+COPY ./bin/* /usr/bin/
+
+ENV TINI_VERSION v0.19.0
+RUN bash -c 'echo -e "Downloading tini-${cpuType} ..."'
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--", "/usr/bin/entrypoint.sh"]
+CMD ["taosd"]
+VOLUME [ "/var/lib/taos", "/var/log/taos", "/corefile" ]

packaging/docker/bin/entrypoint.sh

+#!/bin/sh
+set -e
+# for TZ awareness
+if [ "$TZ" != "" ]; then
+    ln -sf /usr/share/zoneinfo/$TZ /etc/localtime
+    echo $TZ >/etc/timezone
+fi
+
+# option to disable taosadapter, default is no
+DISABLE_ADAPTER=${TAOS_DISABLE_ADAPTER:-0}
+unset TAOS_DISABLE_ADAPTER
+
+# to get mnodeEpSet from data dir
+DATA_DIR=${TAOS_DATA_DIR:-/var/lib/taos}
+
+# append env to custom taos.cfg
+CFG_DIR=/tmp/taos
+CFG_FILE=$CFG_DIR/taos.cfg
+
+mkdir -p $CFG_DIR >/dev/null 2>&1
+
+[ -f /etc/taos/taos.cfg ] && cat /etc/taos/taos.cfg | grep -E -v "^#|^\s*$" >$CFG_FILE
+env-to-cfg >>$CFG_FILE
+
+FQDN=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep fqdn | tail -n1 | sed -E 's/.*fqdn\s+//')
+
+# ensure the fqdn is resolved as localhost
+grep "$FQDN" /etc/hosts >/dev/null || echo "127.0.0.1 $FQDN" >>/etc/hosts
+
+# parse first ep host and port
+FIRST_EP_HOST=${TAOS_FIRST_EP%:*}
+FIRST_EP_PORT=${TAOS_FIRST_EP#*:}
+
+# in case of custom server port
+SERVER_PORT=$(cat $CFG_FILE | grep -E -v "^#|^$" | grep serverPort | tail -n1 | sed -E 's/.*serverPort\s+//')
+SERVER_PORT=${SERVER_PORT:-6030}
+
+# for other binaries like interpreters
+if echo $1 | grep -E "taosd$" - >/dev/null; then
+    true # will run taosd
+else
+    cp -f $CFG_FILE /etc/taos/taos.cfg || true
+    $@
+    exit $?
+fi
+
+set +e
+ulimit -c unlimited
+# set core files pattern, maybe failed
+sysctl -w kernel.core_pattern=/corefile/core-$FQDN-%e-%p >/dev/null >&1
+set -e
+
+if [ "$DISABLE_ADAPTER" = "0" ]; then
+    which taosadapter >/dev/null && taosadapter &
+    # wait for 6041 port ready
+    for _ in $(seq 1 20); do
+        nc -z localhost 6041 && break
+        sleep 0.5
+    done
+fi
+
+# if has mnode ep set or the host is first ep or not for cluster, just start.
+if [ -f "$DATA_DIR/dnode/mnodeEpSet.json" ] ||
+    [ "$TAOS_FQDN" = "$FIRST_EP_HOST" ]; then
+    $@ -c $CFG_DIR
+# others will first wait the first ep ready.
+else
+    if [ "$TAOS_FIRST_EP" = "" ]; then
+        echo "run TDengine with single node."
+        $@ -c $CFG_DIR
+        exit $?
+    fi
+    while true; do
+        es=0
+        taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -n startup >/dev/null || es=$?
+        if [ "$es" -eq 0 ]; then
+            taos -h $FIRST_EP_HOST -P $FIRST_EP_PORT -s "create dnode \"$FQDN:$SERVER_PORT\";"
+            break
+        fi
+        sleep 1s
+    done
+    $@ -c $CFG_DIR
+fi

packaging/docker/bin/env-to-cfg

+#!/bin/sh
+set -e
+self=$0
+
+snake_to_camel_case() {
+  echo $1 | awk -F _ '{printf "%s", $1; for(i=2; i<=NF; i++) printf "%s", toupper(substr($i,1,1)) substr($i,2); print"";}'
+}
+
+if echo $1 | grep -E "^$" - >/dev/null; then
+  export |grep -E 'TAOS_.*' -o| sed 's/TAOS_//' |tr A-Z a-z | awk -F"=" '{print "name=$(""'$self' " $1"); echo $name "$2}' |sh
+else
+  snake_to_camel_case $1
+fi

packaging/docker/docker-compose.yml

+version: "3"
+
+networks:
+  inter:
+  api:
+
+services:
+  arbitrator:
+    image: tdengine/tdengine:$VERSION
+    command: tarbitrator
+    networks:
+      - inter
+  td-1:
+    image: tdengine/tdengine:$VERSION
+    networks:
+      - inter
+    environment:
+      TAOS_FQDN: "td-1"
+      TAOS_FIRST_EP: "td-1"
+      TAOS_NUM_OF_MNODES: "2"
+      TAOS_REPLICA: "2"
+      TAOS_ARBITRATOR: arbitrator:6042
+    volumes:
+      - taosdata-td1:/var/lib/taos/
+      - taoslog-td1:/var/log/taos/
+  td-2:
+    image: tdengine/tdengine:$VERSION
+    networks:
+      - inter
+    environment:
+      TAOS_FQDN: "td-2"
+      TAOS_FIRST_EP: "td-1"
+      TAOS_NUM_OF_MNODES: "2"
+      TAOS_REPLICA: "2"
+      TAOS_ARBITRATOR: arbitrator:6042
+    volumes:
+      - taosdata-td2:/var/lib/taos/
+      - taoslog-td2:/var/log/taos/
+  adapter:
+    image: tdengine/tdengine:$VERSION
+    command: taosadapter
+    networks:
+      - inter
+    environment:
+      TAOS_FIRST_EP: "td-1"
+      TOAS_SECOND_EP: "td-2"
+    deploy:
+      replicas: 4
+      update_config:
+        parallelism: 4
+  nginx:
+    image: nginx
+    depends_on:
+      - adapter
+    networks:
+      - inter
+      - api
+    ports:
+      - 6041:6041
+      - 6044:6044/udp
+    command: [
+        "sh",
+        "-c",
+        "while true;
+        do curl -s http://adapter:6041/-/ping >/dev/null && break;
+        done;
+        printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}'
+        > /etc/nginx/conf.d/rest.conf;
+        printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}'
+        >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf;
+        nginx -g 'daemon off;'",
+      ]
+volumes:
+  taosdata-td1:
+  taoslog-td1:
+  taosdata-td2:
+  taoslog-td2:

packaging/docker/dockerManifest.sh

+#!/bin/bash
+set -e
+#set -x
+
+# dockerbuild.sh 
+#             -n [version number]
+#             -p [xxxx]
+#             -V [stable | beta]
+
+# set parameters by default value
+version=""
+passWord=""
+verType=""
+
+while getopts "hn:p:V:" arg
+do
+  case $arg in
+    n)
+      #echo "version=$OPTARG"
+      version=$(echo $OPTARG)
+      ;;
+    p)
+      #echo "passWord=$OPTARG"
+      passWord=$(echo $OPTARG)
+      ;;
+    V)
+      #echo "verType=$OPTARG"
+      verType=$(echo $OPTARG)
+      ;;    
+    h)
+      echo "Usage: `basename $0` -n [version number] "
+      echo "                      -p [password for docker hub] "
+      exit 0
+      ;;
+    ?) #unknow option 
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+echo "version=${version}"
+
+#docker manifest rm tdengine/tdengine
+#docker manifest rm tdengine/tdengine:${version}
+if [ "$verType" == "beta" ]; then
+  docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version}
+  docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest
+  docker manifest rm tdengine/tdengine-beta:${version}
+  docker manifest rm tdengine/tdengine-beta:latest
+  docker manifest create -a tdengine/tdengine-beta:${version} tdengine/tdengine-amd64-beta:${version} tdengine/tdengine-aarch64-beta:${version} tdengine/tdengine-aarch32-beta:${version}
+  docker manifest create -a tdengine/tdengine-beta:latest tdengine/tdengine-amd64-beta:latest tdengine/tdengine-aarch64-beta:latest tdengine/tdengine-aarch32-beta:latest
+  docker manifest inspect  tdengine/tdengine:latest 
+  docker manifest inspect  tdengine/tdengine:${version}
+  docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
+  docker manifest push tdengine/tdengine-beta:${version}
+  docker manifest push tdengine/tdengine-beta:latest
+elif [ "$verType" == "stable" ]; then
+  docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version}
+  docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
+  docker manifest rm tdengine/tdengine:latest
+  docker manifest rm tdengine/tdengine:${version}
+  docker manifest create -a tdengine/tdengine:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version}
+  docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
+  docker manifest inspect  tdengine/tdengine:latest 
+  docker manifest inspect  tdengine/tdengine:${version}
+  docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
+  docker manifest push tdengine/tdengine:${version}
+  docker manifest push tdengine/tdengine:latest
+else
+  echo "unknow verType, nor stabel or beta"
+  exit 1
+fi
+
+# docker manifest create -a tdengine/${dockername}:${version} tdengine/tdengine-amd64:${version} tdengine/tdengine-aarch64:${version} tdengine/tdengine-aarch32:${version}
+# docker manifest create -a tdengine/${dockername}:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest
+
+# docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
+
+# docker manifest push tdengine/tdengine:latest
+
+# # how set latest version ???

packaging/docker/dockerbuild.sh

+#!/bin/bash
+#
+
+set -e
+#set -x
+
+# dockerbuild.sh 
+#             -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...]  
+#             -n [version number]
+#             -p [password for docker hub]
+#             -V [stable | beta]
+#             -f [pkg file]
+
+# set parameters by default value
+cpuType=""
+cpuTypeAlias=""
+version=""
+passWord=""
+pkgFile=""
+verType="stable"
+dockerLatest="n"
+
+while getopts "hc:n:p:f:V:a:b:" arg
+do
+  case $arg in
+    c)
+      #echo "cpuType=$OPTARG"
+      cpuType=$(echo $OPTARG)
+      ;;
+    n)
+      #echo "version=$OPTARG"
+      version=$(echo $OPTARG)
+      ;;
+    p)
+      #echo "passWord=$OPTARG"
+      passWord=$(echo $OPTARG)
+      ;;
+    f)
+      #echo "pkgFile=$OPTARG"
+      pkgFile=$(echo $OPTARG)
+      ;;
+    b)
+      #echo "branchName=$OPTARG"
+      branchName=$(echo $OPTARG)
+      ;;
+    V)
+      #echo "verType=$OPTARG"
+      verType=$(echo $OPTARG)
+      ;;
+    a)
+      #echo "dockerLatest=$OPTARG"
+      dockerLatest=$(echo $OPTARG)
+      ;;
+    h)
+      echo "Usage: `basename $0`  -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] "
+      echo "                      -n [version number] "
+      echo "                      -p [password for docker hub] "
+      echo "                      -V [stable | beta] "
+      echo "                      -f [pkg file] "
+      echo "                      -a [y | n ]   "
+      exit 0
+      ;;
+    ?) #unknow option 
+      echo "unkonw argument"
+      exit 1
+      ;;
+  esac
+done
+
+
+# Check_verison()
+# {
+# }
+
+
+if [ "$verType" == "beta" ]; then
+  dockername=${cpuType}-${verType}
+  dirName=${pkgFile%-beta*}
+elif [ "$verType" == "stable" ]; then
+  dockername=${cpuType}
+  dirName=${pkgFile%-Linux*}
+else
+  echo "unknow verType, nor stabel or beta"
+  exit 1
+fi
+
+
+echo "cpuType=${cpuType} version=${version} pkgFile=${pkgFile} verType=${verType} "
+echo "$(pwd)"
+echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh===="
+
+scriptDir=$(dirname $(readlink -f $0))
+comunityArchiveDir=/nas/TDengine/v$version/community   # community version’package directory
+communityDir=${scriptDir}/../../../community
+DockerfilePath=${communityDir}/packaging/docker/
+Dockerfile=${communityDir}/packaging/docker/Dockerfile
+cd ${scriptDir}
+cp -f ${comunityArchiveDir}/${pkgFile}  .
+
+echo "dirName=${dirName}"
+
+if [[ "${cpuType}" == "x64" ]] || [[ "${cpuType}" == "amd64" ]]; then
+    cpuTypeAlias="amd64"
+elif [[ "${cpuType}" == "aarch64" ]]; then
+    cpuTypeAlias="arm64"
+elif [[ "${cpuType}" == "aarch32" ]]; then
+    cpuTypeAlias="armhf"
+else
+    echo "Unknown cpuType: ${cpuType}"
+    exit 1
+fi
+
+docker build --rm -f "${Dockerfile}"  --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias}
+docker login -u tdengine -p ${passWord}  #replace the docker registry username and password
+docker push tdengine/tdengine-${dockername}:${version}
+
+if [ -n "$(docker ps -aq)" ] ;then 
+  echo "delete docker process"
+  docker stop $(docker ps -aq)
+  docker rm $(docker ps -aq)
+fi
+
+if [  -n "$(pidof taosd)" ] ;then
+   echo "kill taosd "
+   kill -9 $(pidof taosd)
+fi
+
+if [ -n "$(pidof power)" ]  ;then
+  echo "kill power "
+  kill -9 $(pidof power)
+fi
+
+
+echo ">>>>>>>>>>>>> check whether  tdengine/tdengine-${dockername}:${version}  has been published"
+docker run -d --name doctest -p 6030-6049:6030-6049 -p 6030-6049:6030-6049/udp  tdengine/tdengine-${dockername}:${version}
+sleep 2
+curl -u root:taosdata -d 'show variables;'  127.0.0.1:6041/rest/sql > temp1.data
+data_version=$( cat temp1.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1  | jq ".[1]")
+echo "${data_version}" 
+if [ "${data_version}" == "\"${version}\"" ] ; then
+    echo  "docker version is right "
+else
+    echo  "docker version is wrong "
+    exit 1
+fi
+rm -rf  temp1.data
+
+# set this version to latest version 
+if  [ ${dockerLatest} == 'y' ]  ;then
+  docker tag tdengine/tdengine-${dockername}:${version} tdengine/tdengine-${dockername}:latest
+  docker push tdengine/tdengine-${dockername}:latest
+  echo ">>>>>>>>>>>>> check whether  tdengine/tdengine-${dockername}:latest has been published correctly"
+  docker run -d --name doctestla -p 7030-7049:6030-6049 -p 7030-7049:6030-6049/udp   tdengine/tdengine-${dockername}:latest 
+  sleep 2
+  curl -u root:taosdata -d 'show variables;'  127.0.0.1:7041/rest/sql >  temp2.data
+  version_latest=` cat temp2.data |jq .data| jq '.[]' |grep "version" -A 2 -B 1  | jq ".[1]" `
+  echo "${version_latest}" 
+  if [ "${version_latest}" == "\"${version}\"" ] ; then
+      echo  "docker version is right "
+  else 
+      echo  "docker version is wrong "
+      exit 1 
+  fi
+fi
+rm -rf  temp2.data
+
+if [ -n "$(docker ps -aq)" ] ;then 
+  echo "delte docker process"
+  docker stop $(docker ps -aq)
+  docker rm $(docker ps -aq)
+fi
+
+cd ${scriptDir}
+rm -f ${pkgFile}

packaging/tools/get_client.sh

+#!/bin/bash
+#
+
+log_dir=$1
+result_file=$2
+
+if [ ! -n "$1" ];then
+  echo "Pleas input the director of taosdlog."
+  echo "usage: ./get_client.sh <taosdlog directory> <result file>"
+  exit 1
+else
+  log_dir=$1
+fi
+
+if [ ! -n "$2" ];then
+  result_file=clientInfo.txt
+else
+  result_file=$2
+fi
+
+grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file}

packaging/tools/run_taosd_and_taosadapter.sh

+#!/bin/bash
+[[ -x /usr/bin/taosadapter ]] && /usr/bin/taosadapter &
+taosd