Merge branch 'main' of https://github.com/taosdata/TDengine into mark/sml

.gitignore

@@ -16,7 +16,6 @@ debug/
 release/
 target/
 debs/
-deps/
 rpms/
 mac/
 *.pyc

CMakeLists.txt

@@ -15,11 +15,15 @@ SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR})
 set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")
 set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib")
 
+
+
+
 include(${TD_SUPPORT_DIR}/cmake.platform)
 include(${TD_SUPPORT_DIR}/cmake.define)
 include(${TD_SUPPORT_DIR}/cmake.options)
 include(${TD_SUPPORT_DIR}/cmake.version)
 
+
 # contrib
 add_subdirectory(contrib)
 

README-CN.md

@@ -68,14 +68,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
 ```bash
 sudo yum install epel-release
 sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 gflags git openssl-devel
 sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
 ```
 
 ### CentOS 8/Fedora/Rocky Linux
 
 ```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ gflags make cmake epel-release git openssl-devel
 ```
 
 #### 在 CentOS 上构建 taosTools 安装依赖软件
@@ -117,7 +117,7 @@ scl enable devtoolset-9 -- bash
 ### macOS
 
 ```
-brew install argp-standalone pkgconfig
+brew install argp-standalone gflags pkgconfig
 ```
 
 ### 设置 golang 开发环境

README.md

@@ -76,14 +76,14 @@ sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-d
 ```bash
 sudo yum install epel-release
 sudo yum update
-sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel
+sudo yum install -y gcc gcc-c++ make cmake3 gflags git openssl-devel
 sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake
 ```
 
 ### CentOS 8/Fedora/Rocky Linux
 
 ```bash
-sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel
+sudo dnf install -y gcc gcc-c++ make cmake epel-release gflags git openssl-devel
 ```
 
 #### Install build dependencies for taosTools on CentOS
@@ -124,7 +124,7 @@ scl enable devtoolset-9 -- bash
 ### macOS
 
 ```
-brew install argp-standalone pkgconfig
+brew install argp-standalone gflags pkgconfig
 ```
 
 ### Setup golang environment

cmake/cmake.define

@@ -115,18 +115,6 @@ ELSE ()
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
     ENDIF ()
 
-    IF (${BUILD_SANITIZER})
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
-	      MESSAGE(STATUS "Compile with Address Sanitizer!")
-    ELSEIF (${BUILD_RELEASE})
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
-    ELSE ()
-        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
-    ENDIF ()
-
     # disable all assert
     IF ((${DISABLE_ASSERT} MATCHES "true") OR (${DISABLE_ASSERTS} MATCHES "true"))
         ADD_DEFINITIONS(-DDISABLE_ASSERT)
@@ -168,4 +156,20 @@ ELSE ()
         MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
     ENDIF()
 
+    # build mode
+    SET(CMAKE_C_FLAGS_REL "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
+    SET(CMAKE_CXX_FLAGS_REL "${CMAKE_CXX_FLAGS} -Werror -Wno-reserved-user-defined-literal -Wno-literal-suffix -Werror=return-type -fPIC -O3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
+
+    IF (${BUILD_SANITIZER})
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}     -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
+	      MESSAGE(STATUS "Compile with Address Sanitizer!")
+    ELSEIF (${BUILD_RELEASE})
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}")
+    ELSE ()
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -g3 -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reserved-user-defined-literal -g3 -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
+    ENDIF ()
+
 ENDIF ()

cmake/cmake.options

@@ -189,3 +189,9 @@ option(
    "If build release version"
    OFF
 )
+
+option(
+   BUILD_CONTRIB
+   "If build thirdpart from source"
+   OFF
+)

cmake/cmake.platform

@@ -172,5 +172,15 @@ ENDIF()
 
 MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR})
 
+set(TD_DEPS_DIR "x86")
+if (TD_LINUX)
+  IF (TD_ARM_64 OR TD_ARM_32)
+    set(TD_DEPS_DIR "arm")
+  ELSE() 
+    set(TD_DEPS_DIR "x86")
+  ENDIF()
+endif()
+MESSAGE(STATUS "DEPS_DIR" ${TD_DEPS_DIR})
+
 MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
 MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.5.0")
+  SET(TD_VER_NUMBER "3.0.5.2.alpha")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

cmake/rocksdb_CMakeLists.txt.in

 
 # rocksdb
-ExternalProject_Add(rocksdb
-        GIT_REPOSITORY https://github.com/facebook/rocksdb.git
-        GIT_TAG v8.1.1
-        SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb"
-        CONFIGURE_COMMAND ""
-        BUILD_COMMAND ""
-        INSTALL_COMMAND ""
-        TEST_COMMAND ""
+if (${BUILD_CONTRIB})
+        ExternalProject_Add(rocksdb
+	        URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
+  	        URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b
+  	        DOWNLOAD_NO_PROGRESS 1
+	        DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" 
+	        SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" 
+            CONFIGURE_COMMAND ""
+            BUILD_COMMAND ""
+            INSTALL_COMMAND ""
+            TEST_COMMAND ""
         )
+else()
+    if (NOT ${TD_LINUX})
+        ExternalProject_Add(rocksdb
+	        URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
+  	        URL_HASH MD5=3b4c97ee45df9c8a5517308d31ab008b
+  	        DOWNLOAD_NO_PROGRESS 1
+	        DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" 
+	        SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb" 
+            CONFIGURE_COMMAND ""
+            BUILD_COMMAND ""
+            INSTALL_COMMAND ""
+            TEST_COMMAND ""
+        )
+    endif()
+endif()

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG 3.0
+        GIT_TAG main
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

contrib/CMakeLists.txt

@@ -77,11 +77,25 @@ if(${BUILD_WITH_LEVELDB})
     cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_LEVELDB})
 
-# rocksdb
-if(${BUILD_WITH_ROCKSDB})
-    cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    add_definitions(-DUSE_ROCKSDB)
-endif(${BUILD_WITH_ROCKSDB})
+
+if (${BUILD_CONTRIB})
+    if(${BUILD_WITH_ROCKSDB})
+        cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+        add_definitions(-DUSE_ROCKSDB)
+    endif()
+else() 
+    if (NOT ${TD_LINUX})
+        if(${BUILD_WITH_ROCKSDB})
+            cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+            add_definitions(-DUSE_ROCKSDB)
+        endif(${BUILD_WITH_ROCKSDB})
+    else()
+        if(${BUILD_WITH_ROCKSDB})
+            add_definitions(-DUSE_ROCKSDB)
+        endif(${BUILD_WITH_ROCKSDB})
+    endif()
+endif()
+
 
 # canonical-raft
 if(${BUILD_WITH_CRAFT})
@@ -175,8 +189,8 @@ if(${BUILD_TEST})
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
         )
     endif(${TD_DARWIN})
-    
-    
+
+
 endif(${BUILD_TEST})
 
 # cJson
@@ -227,63 +241,113 @@ endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
 # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev
-if(${BUILD_WITH_ROCKSDB})
+if (${BUILD_WITH_UV})
     if(${TD_LINUX})
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}")
+        IF ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+            SET(CMAKE_BUILD_TYPE Release)
+        endif()
     endif(${TD_LINUX})
-    MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
-
-    if(${TD_DARWIN})
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
-    endif(${TD_DARWIN})
-
-    if (${TD_DARWIN_ARM64})
-        set(HAS_ARMV8_CRC true)
-    endif(${TD_DARWIN_ARM64})
-
-    if (${TD_WINDOWS})
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")  
-    endif(${TD_WINDOWS})
-
-
-    if(${TD_DARWIN})
-        option(HAVE_THREAD_LOCAL "" OFF)
-        option(WITH_IOSTATS_CONTEXT "" OFF)
-        option(WITH_PERF_CONTEXT "" OFF)
-    endif(${TD_DARWIN})
-
-    if(${TD_WINDOWS})
-	    option(WITH_JNI "" OFF)
-    endif(${TD_WINDOWS})
-
-    if(${TD_WINDOWS})
-        option(WITH_MD_LIBRARY "build with MD" OFF)
-        set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
-    endif(${TD_WINDOWS})
-    
-    
-    option(WITH_FALLOCATE "" OFF)
-    option(WITH_JEMALLOC "" OFF)
-    option(WITH_GFLAGS "" OFF)
-    option(PORTABLE "" ON)
-    option(WITH_LIBURING "" OFF)
-    option(FAIL_ON_WARNINGS OFF)
-
-    option(WITH_TESTS "" OFF)
-    option(WITH_BENCHMARK_TOOLS "" OFF)
-    option(WITH_TOOLS "" OFF)
-    option(WITH_LIBURING "" OFF)
-    IF (TD_LINUX) 
-    option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    ELSE()
-    option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    ENDIF()
-    add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
-    target_include_directories(
-        rocksdb
-        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
-    )
-endif(${BUILD_WITH_ROCKSDB})
+endif (${BUILD_WITH_UV})
+
+if (${BUILD_WITH_ROCKSDB})
+    if (${BUILD_CONTRIB})
+        if(${TD_LINUX})
+            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result")
+            if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+                SET(CMAKE_BUILD_TYPE Release)
+            endif()
+        endif(${TD_LINUX})
+        MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+
+        if(${TD_DARWIN})
+            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
+        endif(${TD_DARWIN})
+
+        if (${TD_DARWIN_ARM64})
+            set(HAS_ARMV8_CRC true)
+        endif(${TD_DARWIN_ARM64})
+
+        if (${TD_WINDOWS})
+            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+            option(WITH_JNI "" OFF)
+            option(WITH_MD_LIBRARY "build with MD" OFF)
+            set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
+        endif(${TD_WINDOWS})
+
+
+        if(${TD_DARWIN})
+            option(HAVE_THREAD_LOCAL "" OFF)
+            option(WITH_IOSTATS_CONTEXT "" OFF)
+            option(WITH_PERF_CONTEXT "" OFF)
+        endif(${TD_DARWIN})
+
+        option(WITH_FALLOCATE "" OFF)
+        option(WITH_JEMALLOC "" OFF)
+        option(WITH_GFLAGS "" OFF)
+        option(PORTABLE "" ON)
+        option(WITH_LIBURING "" OFF)
+        option(FAIL_ON_WARNINGS OFF)
+
+        option(WITH_TESTS "" OFF)
+        option(WITH_BENCHMARK_TOOLS "" OFF)
+        option(WITH_TOOLS "" OFF)
+        option(WITH_LIBURING "" OFF)
+
+        option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
+        add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
+        target_include_directories(
+            rocksdb
+            PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
+        )
+    else() 
+        if (NOT ${TD_LINUX})
+            MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+            if(${TD_DARWIN})
+                SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
+            endif(${TD_DARWIN})
+
+            if (${TD_DARWIN_ARM64})
+                set(HAS_ARMV8_CRC true)
+            endif(${TD_DARWIN_ARM64})
+
+            if (${TD_WINDOWS})
+                SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+                option(WITH_JNI "" OFF)
+                option(WITH_MD_LIBRARY "build with MD" OFF)
+                set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
+            endif(${TD_WINDOWS})
+
+
+            if(${TD_DARWIN})
+                option(HAVE_THREAD_LOCAL "" OFF)
+                option(WITH_IOSTATS_CONTEXT "" OFF)
+                option(WITH_PERF_CONTEXT "" OFF)
+            endif(${TD_DARWIN})
+
+            option(WITH_FALLOCATE "" OFF)
+            option(WITH_JEMALLOC "" OFF)
+            option(WITH_GFLAGS "" OFF)
+            option(PORTABLE "" ON)
+            option(WITH_LIBURING "" OFF)
+            option(FAIL_ON_WARNINGS OFF)
+
+            option(WITH_TESTS "" OFF)
+            option(WITH_BENCHMARK_TOOLS "" OFF)
+            option(WITH_TOOLS "" OFF)
+            option(WITH_LIBURING "" OFF)
+
+            option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
+            add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
+            target_include_directories(
+                rocksdb
+                PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
+            )
+        endif() 
+      
+    endif()
+endif()
 
 # lucene
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
@@ -291,10 +355,10 @@ if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
     add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
-      lucene++ 
+      lucene++
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
-    ) 
-     
+    )
+
 endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
@@ -354,7 +418,7 @@ if(${BUILD_MSVCREGEX})
     target_include_directories(msvcregex
         PRIVATE "msvcregex"
     )
-    target_link_libraries(msvcregex 
+    target_link_libraries(msvcregex
         INTERFACE Shell32
     )
     SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
@@ -414,8 +478,8 @@ if(${BUILD_WITH_BDB})
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb"
     )
-    target_link_libraries(bdb 
-        INTERFACE pthread 
+    target_link_libraries(bdb
+        INTERFACE pthread
     )
 endif(${BUILD_WITH_BDB})
 
@@ -427,12 +491,12 @@ if(${BUILD_WITH_SQLITE})
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/.libs/libsqlite3.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/sqlite"
     )
-    target_link_libraries(sqlite 
-        INTERFACE m 
-        INTERFACE pthread 
+    target_link_libraries(sqlite
+        INTERFACE m
+        INTERFACE pthread
     )
     if(NOT TD_WINDOWS)
-        target_link_libraries(sqlite 
+        target_link_libraries(sqlite
             INTERFACE dl
         )
     endif(NOT TD_WINDOWS)
@@ -440,22 +504,22 @@ endif(${BUILD_WITH_SQLITE})
 
 # addr2line
 if(${BUILD_ADDR2LINE})
-    if(NOT ${TD_WINDOWS})    
-        check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H) 
-        check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H ) 
-        check_include_file( "inttypes.h"      HAVE_INTTYPES_H   ) 
-        check_include_file( "stddef.h"        HAVE_STDDEF_H   ) 
-        check_include_file( "stdlib.h"        HAVE_STDLIB_H   ) 
-        check_include_file( "string.h"        HAVE_STRING_H   ) 
-        check_include_file( "memory.h"        HAVE_MEMORY_H   ) 
-        check_include_file( "strings.h"       HAVE_STRINGS_H  ) 
+    if(NOT ${TD_WINDOWS})
+        check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H)
+        check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H )
+        check_include_file( "inttypes.h"      HAVE_INTTYPES_H   )
+        check_include_file( "stddef.h"        HAVE_STDDEF_H   )
+        check_include_file( "stdlib.h"        HAVE_STDLIB_H   )
+        check_include_file( "string.h"        HAVE_STRING_H   )
+        check_include_file( "memory.h"        HAVE_MEMORY_H   )
+        check_include_file( "strings.h"       HAVE_STRINGS_H  )
         check_include_file( "stdint.h"        HAVE_STDINT_H   )
         check_include_file( "unistd.h"        HAVE_UNISTD_H   )
         check_include_file( "sgidefs.h"       HAVE_SGIDEFS_H  )
         check_include_file( "stdafx.h"        HAVE_STDAFX_H   )
-        check_include_file( "elf.h"           HAVE_ELF_H      ) 
-        check_include_file( "libelf.h"        HAVE_LIBELF_H   ) 
-        check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) 
+        check_include_file( "elf.h"           HAVE_ELF_H      )
+        check_include_file( "libelf.h"        HAVE_LIBELF_H   )
+        check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H)
         check_include_file( "alloca.h"        HAVE_ALLOCA_H   )
         check_include_file( "elfaccess.h"     HAVE_ELFACCESS_H)
         check_include_file( "sys/elf_386.h"   HAVE_SYS_ELF_386_H  )
@@ -463,7 +527,7 @@ if(${BUILD_ADDR2LINE})
         check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
         check_include_file( "sys/ia64/elf.h"  HAVE_SYS_IA64_ELF_H )
         set(VERSION 0.3.1)
-        set(PACKAGE_VERSION "\"${VERSION}\"") 
+        set(PACKAGE_VERSION "\"${VERSION}\"")
         configure_file(libdwarf/cmake/config.h.cmake config.h)
         file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
         add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
@@ -485,8 +549,16 @@ endif(${BUILD_ADDR2LINE})
 
 # geos
 if(${BUILD_GEOS})
+    if(${TD_LINUX})
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}")
+        IF ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+            SET(CMAKE_BUILD_TYPE Release)
+        endif()
+    endif(${TD_LINUX})
     option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF)
     add_subdirectory(geos EXCLUDE_FROM_ALL)
+    unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
     target_include_directories(
         geos_c
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/geos/include>

docs/en/05-get-started/03-package.md

@@ -20,6 +20,19 @@ The standard server installation package includes `taos`, `taosd`, `taosAdapter`
 
 The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS.
 
+## Operating environment requirements
+In the Linux system, the minimum requirements for the operating environment are as follows:
+
+linux core version - 3.10.0-1160.83.1.el7.x86_64;
+
+glibc version - 2.17;
+
+If compiling and installing through clone source code, it is also necessary to meet the following requirements:
+
+cmake version - 3.26.4 or above;
+
+gcc version - 9.3.1 or above;
+
 ## Installation
 
 <Tabs>

docs/en/07-develop/08-cache.md

@@ -10,10 +10,10 @@ TDengine uses various kinds of caching techniques to efficiently write and query
 
 TDengine uses an insert-driven cache management policy, known as first in, first out (FIFO). This policy differs from read-driven "least recently used (LRU)" cache management. A FIFO policy stores the latest data in cache and flushes the oldest data from cache to disk when the cache usage reaches a threshold. In IoT use cases, the most recent data or the current state is most important. The cache policy in TDengine, like much of the design and architecture of TDengine, is based on the nature of IoT data.
 
-When you create a database, you can configure the size of the write cache on each vnode. The **vgroups** parameter determines the number of vgroups that process data in the database, and the **buffer** parameter determines the size of the write cache for each vnode.
+When you create a database, you can configure the size of the write cache on each vnode. The **vgroups** parameter determines the number of vgroups that process data in the database, and the **buffer** parameter determines the size of the write cache for each vnode. The unit of buffer is MB.
 
 ```sql
-create database db0 vgroups 100 buffer 16MB
+create database db0 vgroups 100 buffer 16
 ```
 
 In theory, larger cache sizes are always better. However, at a certain point, it becomes impossible to improve performance by increasing cache size. In most scenarios, you can retain the default cache settings.
@@ -28,10 +28,10 @@ When you create a database, you can configure whether the latest data from every
 
 ## Metadata Cache
 
-To improve query and write performance, each vnode caches the metadata that it receives. When you create a database, you can configure the size of the metadata cache through the *pages* and *pagesize* parameters.
+To improve query and write performance, each vnode caches the metadata that it receives. When you create a database, you can configure the size of the metadata cache through the *pages* and *pagesize* parameters. The unit of pagesize is kb.
 
 ```sql
-create database db0 pages 128 pagesize 16kb
+create database db0 pages 128 pagesize 16
 ```
 
 The preceding SQL statement creates 128 pages on each vnode in the `db0` database. Each page has a 16 KB metadata cache.

docs/en/12-taos-sql/22-meta.md

@@ -81,7 +81,7 @@ Provides information about user-created databases. Similar to SHOW DATABASES.
 | 3   |       ntables        | INT              | Number of standard tables and subtables (not including supertables) |
 | 4   |       vgroups        | INT              | Number of vgroups. It should be noted that `vnodes` is a TDengine keyword and needs to be escaped with ` when used as a column name.                            |
 | 6   |       replica        | INT              | Number of replicas. It should be noted that `replica` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                            |
-| 7   |        strict        | BINARY(3)        | Strong consistency. It should be noted that `strict` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                          |
+| 7   |        strict        | BINARY(4)        | Obsoleted |
 | 8   |       duration       | INT              | Duration for storage of single files. It should be noted that `duration` is a TDengine keyword and needs to be escaped with ` when used as a column name.                          |
 | 9   |         keep         | INT              | Data retention period. It should be noted that `keep` is a TDengine keyword and needs to be escaped with ` when used as a column name.                                      |
 | 10  |        buffer        | INT              | Write cache size per vnode, in MB. It should be noted that `buffer` is a TDengine keyword and needs to be escaped with ` when used as a column name.            |

docs/en/14-reference/03-connector/04-java.mdx

@@ -288,6 +288,7 @@ The configuration parameters in the URL are as follows:
 - httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false.
 - messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true.
 - useSSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection.
+- httpPoolSize: size of REST concurrent requests. The default value is 20.
 
 **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection.
 
@@ -355,6 +356,7 @@ The configuration parameters in properties are as follows.
 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false.
 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
 - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection.
+- TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20.
   For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only).
 
 ### Priority of configuration parameters
@@ -419,6 +421,19 @@ while(resultSet.next()){
 
 > The query is consistent with operating a relational database. When using subscripts to get the contents of the returned fields, you have to start from 1. However, we recommend using the field names to get the values of the fields in the result set.
 
+### execute SQL with reqId
+
+This reqId can be used to request link tracing.
+
+```java
+AbstractStatement aStmt = (AbstractStatement) connection.createStatement();
+aStmt.execute("create database if not exists db", 1L);
+aStmt.executeUpdate("use db", 2L);
+try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) {
+    Timestamp ts = rs.getTimestamp(1);
+}
+```
+
 ### Writing data via parameter binding
 
 TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases.
@@ -936,6 +951,14 @@ public class SchemalessWsTest {
 </TabItem>
 </Tabs>
 
+### Schemaless with reqId
+
+This reqId can be used to request link tracing.
+
+```java
+writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L);
+```
+
 ### Data Subscription
 
 The TDengine Java Connector supports subscription functionality with the following application API.
@@ -993,7 +1016,7 @@ while(true) {
 
 #### Assignment subscription Offset
 
-```
+```java
 long position(TopicPartition partition) throws SQLException;
 Map<TopicPartition, Long> position(String topic) throws SQLException;
 Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
@@ -1002,6 +1025,29 @@ Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
 void seek(TopicPartition partition, long offset) throws SQLException;
 ```
 
+Example usage is as follows.
+
+```java
+String topic = "offset_seek_test";
+Map<TopicPartition, Long> offset = null;
+try (TaosConsumer<ResultBean> consumer = new TaosConsumer<>(properties)) {
+    consumer.subscribe(Collections.singletonList(topic));
+    for (int i = 0; i < 10; i++) {
+        if (i == 3) {
+            // Saving consumption position
+            offset = consumer.position(topic);
+        }
+        if (i == 5) {
+            // reset consumption to the previously saved position
+            for (Map.Entry<TopicPartition, Long> entry : offset.entrySet()) {
+                consumer.seek(entry.getKey(), entry.getValue());
+            }
+        }
+        ConsumerRecords<ResultBean> records = consumer.poll(Duration.ofMillis(500));
+    }
+}
+```
+
 #### Close subscriptions
 
 ```java
@@ -1308,3 +1354,7 @@ For additional troubleshooting, see [FAQ](../../../train-faq/faq).
 ## API Reference
 
 [taos-jdbcdriver doc](https://docs.taosdata.com/api/taos-jdbcdriver)
+
+```
+
+```

docs/en/14-reference/03-connector/index.mdx

@@ -48,7 +48,6 @@ Comparing the connector support for TDengine functional features as follows.
 | **Parameter Binding**         | Support       | Support    | Support       | Support       | Support       | Support       |
 | **Subscription (TMQ)**        | Support       | Support    | Support       | Support       | Support       | Support       |
 | **Schemaless**                | Support       | Support    | Support       | Support       | Support       | Support       |
-| **DataFrame**                 | Not Supported | Support    | Not Supported | Not Supported | Not Supported | Not Supported |
 
 :::info
 The different database framework specifications for various programming languages do not mean that all C/C++ interfaces need a wrapper.
@@ -60,11 +59,10 @@ The different database framework specifications for various programming language
 | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- |
 | **Connection Management**              | Support       | Support         | Support       | Support       | Support       | Support       |
 | **Regular Query**                      | Support       | Support         | Support       | Support       | Support       | Support       |
-| **Parameter Binding**                  | Not Supported | Not Supported   | Support       | Support       | Not Supported | Support       |
+| **Parameter Binding**                  | Supported | Not Supported   | Support       | Support       | Not Supported | Support       |
 | **Subscription (TMQ) **                | Supported     | Support         | Support       | Not Supported | Not Supported | Support       |
-| **Schemaless**                         | Not Supported | Not Supported   | Not Supported | Not Supported | Not Supported | Not Supported |
+| **Schemaless**                         | Supported | Not Supported   | Supported | Not Supported | Not Supported | Not Supported |
 | **Bulk Pulling (based on WebSocket) ** | Support       | Support         | Support       | Support       | Support       | Support       |
-| **DataFrame**                          | Not Supported | Support         | Not Supported | Not Supported | Not Supported | Not Supported |
 
 :::warning
 

docs/en/14-reference/12-config/index.md

@@ -19,14 +19,18 @@ taosd -C
 
 ## Configuration File on Client Side
 
-TDengine CLI `taos` is the tool for users to interact with TDengine. It can share same configuration file as `taosd` or use a separate configuration file. When launching `taos`, parameter `-c` can be used to specify the location where its configuration file is. For example `taos -c /home/cfg` means `/home/cfg/taos.cfg` will be used. If `-c` is not used, the default location of the configuration file is `/etc/taos`. For more details please use `taos --help` to get.
+TDengine CLI `taos` is the tool for users to interact with TDengine. It can share same configuration file as `taosd` or use a separate configuration file. When launching `taos`, parameter `-c` can be used to specify the location where its configuration file is. For example:
 
-```bash
-taos -C
 ```
+taos -c /home/cfg
+```
+
+means `/home/cfg/taos.cfg` will be used. If `-c` is not used, the default location of the configuration file is `/etc/taos`. For more details please use `taos --help` to get.
+
+Parameter `-C` can be used on the CLI of `taos` to show its configuration, like below:
 
 ```bash
-taos --dump-config
+taos -C
 ```
 
 ## Configuration Parameters
@@ -77,8 +81,9 @@ The parameters described in this document by the effect that they have on the sy
 | Default Value | 6030                                                  |
 
 :::note
-- Ensure that your firewall rules do not block TCP port 6042  on any host in the cluster. Below table describes the ports used by TDengine in details.
+Ensure that your firewall rules do not block TCP port 6042  on any host in the cluster. Below table describes the ports used by TDengine in details.
 :::
+
 | Protocol | Default Port | Description                                                                                               | How to configure                                                                               |
 | :------- | :----------- | :-------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------- |
 | TCP      | 6030         | Communication between client and server. In a multi-node cluster, communication between nodes. serverPort |
@@ -120,6 +125,8 @@ The parameters described in this document by the effect that they have on the sy
 :::note
 Please note the `taoskeeper` needs to be installed and running to create the `log` database and receiving metrics sent by `taosd` as the full monitoring solution.
 
+:::
+
 ### monitor
 
 | Attribute   | Description                                                                                                                                                                                                                                                                                                           |

docs/en/27-train-faq/01-faq.md

@@ -56,7 +56,7 @@ This error indicates that the client could not connect to the server. Perform th
 
 7. If you are using the Python, Java, Go, Rust, C#, or Node.js connector on Linux to connect to the server, verify that `libtaos.so` is in the `/usr/local/taos/driver` directory and `/usr/local/taos/driver` is in the `LD_LIBRARY_PATH` environment variable.
 
-8. If you are using macOS, verify that `libtaos.dylib` is in the `/usr/local/lib` directory and `/usr/local/lib` is in the `LD_LIBRARY_PATH` environment variable..
+8. If you are using macOS, verify that `libtaos.dylib` is in the `/usr/local/lib` directory and `/usr/local/lib` is in the `DYLD_LIBRARY_PATH` environment variable..
 
 9. If you are using Windows, verify that `C:\TDengine\driver\taos.dll` is in the `PATH` environment variable. If possible, move `taos.dll` to the `C:\Windows\System32` directory.
 

docs/en/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.5.1
+
+<Release type="tdengine" version="3.0.5.1" />
+
 ## 3.0.5.0
 
 <Release type="tdengine" version="3.0.5.0" />

docs/en/28-releases/02-tools.md

@@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat
 
 import Release from "/components/ReleaseV3";
 
+## 2.5.2
+
+<Release type="tools" version="2.5.2" />
+
 ## 2.5.1
 
 <Release type="tools" version="2.5.1" />

docs/zh/05-get-started/03-package.md

@@ -16,6 +16,20 @@ TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）
 
 在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。
 
+## 运行环境要求
+在linux系统中，运行环境最低要求如下:
+
+linux 内核版本 - 3.10.0-1160.83.1.el7.x86_64;
+
+glibc 版本    -  2.17;
+
+如果通过clone源码进行编译安装，还需要满足:
+
+cmake版本 - 3.26.4或以上;
+
+gcc 版本  - 9.3.1或以上;
+
+
 ## 安装
 
 <Tabs>

docs/zh/07-develop/08-cache.md

@@ -10,10 +10,10 @@ description: "TDengine 内部的缓存设计"
 
 TDengine 采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine 充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。
 
-每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。
+每个 vnode 的写入缓存大小在创建数据库时决定，创建数据库时的两个关键参数 vgroups 和 buffer 分别决定了该数据库中的数据由多少个 vgroup 处理，以及向其中的每个 vnode 分配多少写入缓存。buffer 的单位是MB。
 
 ```sql
-create database db0 vgroups 100 buffer 16MB
+create database db0 vgroups 100 buffer 16
 ```
 
 理论上缓存越大越好，但超过一定阈值后再增加缓存对写入性能提升并无帮助，一般情况下使用默认值即可。
@@ -28,10 +28,10 @@ create database db0 vgroups 100 buffer 16MB
 
 ## 元数据缓存
 
-为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。
+为了更高效地处理查询和写入，每个 vnode 都会缓存自己曾经获取到的元数据。元数据缓存由创建数据库时的两个参数 pages 和 pagesize 决定。pagesize 的单位是 kb。
 
 ```sql
-create database db0 pages 128 pagesize 16kb
+create database db0 pages 128 pagesize 16
 ```
 
 上述语句会为数据库 db0 的每个 vnode 创建 128 个 page，每个 page 16kb 的元数据缓存。

docs/zh/08-connector/14-java.mdx

@@ -291,6 +291,7 @@ url 中的配置参数如下：
 - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。
 - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。
 - useSSL: 连接中是否使用 SSL。
+- httpPoolSize: REST 并发请求大小，默认 20。
 
 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。
 
@@ -358,6 +359,7 @@ properties 中的配置参数如下：
 - TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。
 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。
 - TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。
+- TSDBDriver.HTTP_POOL_SIZE: REST 并发请求大小，默认 20。
   此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。
 
 ### 配置参数的优先级
@@ -422,6 +424,19 @@ while(resultSet.next()){
 
 > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
 
+### 执行带有 reqId 的 SQL
+
+此 reqId 可用于请求链路追踪。
+
+```java
+AbstractStatement aStmt = (AbstractStatement) connection.createStatement();
+aStmt.execute("create database if not exists db", 1L);
+aStmt.executeUpdate("use db", 2L);
+try (ResultSet rs = aStmt.executeQuery("select * from tb", 3L)) {
+    Timestamp ts = rs.getTimestamp(1);
+}
+```
+
 ### 通过参数绑定写入数据
 
 TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。
@@ -939,6 +954,14 @@ public class SchemalessWsTest {
 </TabItem>
 </Tabs>
 
+### 执行带有 reqId 的无模式写入
+
+此 reqId 可用于请求链路追踪。
+
+```java
+writer.write(lineDemo, SchemalessProtocolType.LINE, SchemalessTimestampType.NANO_SECONDS, 1L);
+```
+
 ### 数据订阅
 
 TDengine Java 连接器支持订阅功能，应用 API 如下：
@@ -996,7 +1019,7 @@ while(true) {
 
 #### 指定订阅 Offset
 
-```
+```java
 long position(TopicPartition partition) throws SQLException;
 Map<TopicPartition, Long> position(String topic) throws SQLException;
 Map<TopicPartition, Long> beginningOffsets(String topic) throws SQLException;
@@ -1005,6 +1028,29 @@ Map<TopicPartition, Long> endOffsets(String topic) throws SQLException;
 void seek(TopicPartition partition, long offset) throws SQLException;
 ```
 
+示例代码：
+
+```java
+String topic = "offset_seek_test";
+Map<TopicPartition, Long> offset = null;
+try (TaosConsumer<ResultBean> consumer = new TaosConsumer<>(properties)) {
+    consumer.subscribe(Collections.singletonList(topic));
+    for (int i = 0; i < 10; i++) {
+        if (i == 3) {
+            // Saving consumption position
+            offset = consumer.position(topic);
+        }
+        if (i == 5) {
+            // reset consumption to the previously saved position
+            for (Map.Entry<TopicPartition, Long> entry : offset.entrySet()) {
+                consumer.seek(entry.getKey(), entry.getValue());
+            }
+        }
+        ConsumerRecords<ResultBean> records = consumer.poll(Duration.ofMillis(500));
+    }
+}
+```
+
 #### 关闭订阅
 
 ```java

docs/zh/08-connector/index.md

@@ -45,9 +45,8 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
 | **连接管理**        | 支持     | 支持       | 支持   | 支持   | 支持        | 支持     |
 | **普通查询**        | 支持     | 支持       | 支持   | 支持   | 支持        | 支持     |
 | **参数绑定**        | 支持     | 支持       | 支持   | 支持   | 支持        | 支持     |
-| **数据订阅（TMQ）** | 暂不支持 | 支持       | 支持   | 支持   | 支持        | 支持     |
+| **数据订阅（TMQ）** | 支持 | 支持       | 支持   | 支持   | 支持        | 支持     |
 | **Schemaless**      | 支持     | 支持       | 支持   | 支持   | 支持        | 支持     |
-| **DataFrame**       | 不支持   | 支持       | 不支持 | 不支持 | 不支持      | 不支持   |
 
 :::info
 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。
@@ -59,11 +58,10 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- |
 | **连接管理**                   | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
 | **普通查询**                   | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
-| **参数绑定**                   | 暂不支持 | 暂不支持   | 支持     | 支持     | 暂不支持    | 支持     |
+| **参数绑定**                   | 支持 | 暂不支持   | 支持     | 支持     | 暂不支持    | 支持     |
 | **数据订阅（TMQ）**            | 支持     | 支持       | 支持     | 暂不支持 | 暂不支持    | 支持     |
-| **Schemaless**                 | 支持     | 暂不支持   | 暂不支持 | 暂不支持 | 暂不支持    | 暂不支持 |
+| **Schemaless**                 | 支持     | 暂不支持   | 支持 | 暂不支持 | 暂不支持    | 支持 |
 | **批量拉取（基于 WebSocket）** | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
-| **DataFrame**                  | 不支持   | 支持       | 不支持   | 不支持   | 不支持      | 不支持   |
 
 :::warning
 

docs/zh/12-taos-sql/22-meta.md

@@ -81,7 +81,7 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库，提供对数
 | 3   |       ntables        | INT              | 数据库中表的数量，包含子表和普通表但不包含超级表 |
 | 4   |       vgroups        | INT              | 数据库中有多少个 vgroup。需要注意，`vgroups` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                          |
 | 6   |       replica        | INT              | 副本数。需要注意，`replica` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                           |
-| 7   |        strict        | BINARY(3)        | 强一致性。需要注意，`strict` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                         |
+| 7   |        strict        | BINARY(4)        | 废弃参数 |
 | 8   |       duration       | INT              | 单文件存储数据的时间跨度。需要注意，`duration` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                         |
 | 9   |         keep         | INT              | 数据保留时长。需要注意，`keep` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。                                     |
 | 10  |        buffer        | INT              | 每个 vnode 写缓存的内存块大小，单位 MB。需要注意，`buffer` 为 TDengine 关键字，作为列名使用时需要使用 ` 进行转义。           |

docs/zh/14-reference/12-config/index.md

@@ -19,16 +19,20 @@ taosd -C
 
 ## 为客户端指定配置文件
 
-TDengine 系统的前台交互客户端应用程序为 taos，以及应用驱动，它可以与 taosd 共享同一个配置文件 taos.cfg，也可以使用单独指定配置文件。运行 taos 时，使用参数-c 指定配置文件目录，如 taos -c /home/cfg，表示使用/home/cfg/目录下的 taos.cfg 配置文件中的参数，缺省目录是/etc/taos。更多 taos 的使用方法请见帮助信息 `taos --help`。
+TDengine 系统的前台交互客户端应用程序为 taos，以及应用驱动，它可以与 taosd 共享同一个配置文件 taos.cfg，也可以使用单独指定配置文件。运行 taos 时，使用参数-c 指定配置文件目录，如：
 
-```bash
-taos -C
 ```
+taos -c /home/cfg
+```
+
+表示使用/home/cfg/目录下的 taos.cfg 配置文件中的参数，缺省目录是/etc/taos。 另外可以使用 `-C` 显示当前服务器配置参数：
 
 ```bash
-taos --dump-config
+taos -C
 ```
 
+更多 taos 的使用方法请见帮助信息 `taos --help`。
+
 ## 配置参数详细列表
 
 :::note
@@ -139,6 +143,8 @@ taos --dump-config
 :::note
 请注意，完整的监控功能需要安装并运行 `taoskeeper` 服务。taoskeeper 负责接收监控指标数据并创建 `log` 库。
 
+:::
+
 ### monitor
 
 | 属性     | 说明                                                                                                                                                                                               |

docs/zh/21-tdinternal/01-arch.md

@@ -165,9 +165,7 @@ Vnode 会保持一个数据版本号（version），对内存数据进行持久
 
 ### 同步复制
 
-对于数据一致性要求更高的场景，异步数据复制提供的最终一致性无法满足要求。因此 TDengine 提供同步复制的机制供用户选择。在创建数据库时，除指定副本数 `replica` 之外，用户还需要指定新的参数 `strict`。如果 `strict` 等于 1，它表示每次 leader 转发给副本时，需要等待半数以上副本达成一致后，才能通知应用，数据在 follower 已经写入成功。如果在一定的时间内，得不到半数以上副本的确认，leader vnode 将返回错误给应用。
-
-采用同步复制，系统的性能会有所下降，而且 latency 会增加。因为元数据要强一致，mnode 之间的数据同步缺省就是采用的同步复制。
+对于数据一致性要求更高的场景，异步数据复制提供的最终一致性无法满足要求。因此 TDengine 3.0 使用了同步复制的机制（参照 RAFT 协议的标准实现）。每次 leader vnode 转发给其他副本时，需要等待半数以上（包含自己）副本达成一致后，才能通知应用写入成功。如果在一定的时间内，得不到半数以上副本的确认，leader vnode 将返回错误给应用。
 
 ## 缓存与持久化
 

docs/zh/28-releases/01-tdengine.md

@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.5.1
+
+<Release type="tdengine" version="3.0.5.1" />
+
 ## 3.0.5.0
 
 <Release type="tdengine" version="3.0.5.0" />

docs/zh/28-releases/02-tools.md

@@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下：
 
 import Release from "/components/ReleaseV3";
 
+## 2.5.2
+
+<Release type="tools" version="2.5.2" />
+
 ## 2.5.1
 
 <Release type="tools" version="2.5.1" />

examples/JDBC/consumer-demo/pom.xml

@@ -22,7 +22,7 @@
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>30.1.1-jre</version>
+            <version>32.0.0-jre</version>
         </dependency>
     </dependencies>
 

include/common/tmsg.h

@@ -3354,6 +3354,17 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
   taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteMqSubTopicEp);
 }
 
+typedef struct {
+  int32_t      vgId;
+  STqOffsetVal offset;
+  int64_t      rows;
+}OffsetRows;
+
+typedef struct{
+  char       topicName[TSDB_TOPIC_FNAME_LEN];
+  SArray*    offsetRows;
+}TopicOffsetRows;
+
 #define TD_AUTO_CREATE_TABLE 0x1
 typedef struct {
   int64_t       suid;

include/common/tmsgdef.h

@@ -310,6 +310,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME_PUSH, "vnode-tmq-consume-push", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
 

include/libs/function/functionMgt.h

@@ -216,7 +216,7 @@ bool fmIsUserDefinedFunc(int32_t funcId);
 bool fmIsDistExecFunc(int32_t funcId);
 bool fmIsForbidFillFunc(int32_t funcId);
 bool fmIsForbidStreamFunc(int32_t funcId);
-bool fmIsForbidSuperTableFunc(int32_t funcId);
+bool fmIsForbidSysTableFunc(int32_t funcId);
 bool fmIsIntervalInterpoFunc(int32_t funcId);
 bool fmIsInterpFunc(int32_t funcId);
 bool fmIsLastRowFunc(int32_t funcId);

include/libs/wal/wal.h

@@ -214,7 +214,6 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
 
 void walRefFirstVer(SWal *, SWalRef *);
 void walRefLastVer(SWal *, SWalRef *);
-SWalRef *walRefCommittedVer(SWal *);
 
 SWalRef *walOpenRef(SWal *);
 void     walCloseRef(SWal *pWal, int64_t refId);

include/util/talgo.h

@@ -31,7 +31,7 @@ typedef void *(*__array_item_dup_fn_t)(void *);
 
 typedef void (*FDelete)(void *);
 typedef int32_t (*FEncode)(void **buf, const void *dst);
-typedef void *(*FDecode)(const void *buf, void *dst);
+typedef void *(*FDecode)(const void *buf, void *dst, int8_t sver);
 
 #define TD_EQ 0x1
 #define TD_GT 0x2

include/util/taoserror.h

@@ -704,6 +704,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_TAGS_PC           TAOS_DEF_ERROR_CODE(0, 0x2665)
 #define TSDB_CODE_PAR_INVALID_TIMELINE_QUERY    TAOS_DEF_ERROR_CODE(0, 0x2666)
 #define TSDB_CODE_PAR_INVALID_OPTR_USAGE        TAOS_DEF_ERROR_CODE(0, 0x2667)
+#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2668)
 #define TSDB_CODE_PAR_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x26FF)
 
 //planner

include/util/tarray.h

@@ -244,7 +244,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
 
 int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
-void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
+void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver);
 
 #ifdef __cplusplus
 }

include/util/tlrucache.h

@@ -24,7 +24,8 @@ extern "C" {
 
 typedef struct SLRUCache SLRUCache;
 
-typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value);
+typedef void (*_taos_lru_deleter_t)(const void *key, size_t keyLen, void *value, void *ud);
+typedef int (*_taos_lru_functor_t)(const void *key, size_t keyLen, void *value, void *ud);
 
 typedef struct LRUHandle LRUHandle;
 
@@ -41,10 +42,11 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo
 void       taosLRUCacheCleanup(SLRUCache *cache);
 
 LRUStatus  taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge,
-                              _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority);
+                              _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority, void *ud);
 LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen);
 void       taosLRUCacheErase(SLRUCache *cache, const void *key, size_t keyLen);
 
+void taosLRUCacheApply(SLRUCache *cache, _taos_lru_functor_t functor, void *ud);
 void taosLRUCacheEraseUnrefEntries(SLRUCache *cache);
 
 bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle);

packaging/checkPackageRuning.py

@@ -42,8 +42,8 @@ else:
 # os.system("rm -rf /var/lib/taos/*")
 # os.system("systemctl restart taosd ")
 
-# wait a moment ,at least 5 seconds
-time.sleep(5)
+# wait a moment ,at least 10 seconds
+time.sleep(10)
 
 # prepare data by taosBenchmark 
 

packaging/deb/makedeb.sh

@@ -124,12 +124,12 @@ if [ -f ${compile_dir}/build/bin/jemalloc-config ]; then
         cp ${compile_dir}/build/lib/libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/
         ln -sf libjemalloc.so.2 ${pkg_dir}${install_user_local_path}/lib/libjemalloc.so
     fi
-    if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then
-        cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/
-    fi
-    if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then
-        cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/
-    fi
+    # if [ -f ${compile_dir}/build/lib/libjemalloc.a ]; then
+    #     cp ${compile_dir}/build/lib/libjemalloc.a ${pkg_dir}${install_user_local_path}/lib/
+    # fi
+    # if [ -f ${compile_dir}/build/lib/libjemalloc_pic.a ]; then
+    #     cp ${compile_dir}/build/lib/libjemalloc_pic.a ${pkg_dir}${install_user_local_path}/lib/
+    # fi
     if [ -f ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ]; then
         cp ${compile_dir}/build/lib/pkgconfig/jemalloc.pc ${pkg_dir}${install_user_local_path}/lib/pkgconfig/
     fi

packaging/rpm/tdengine.spec

@@ -123,12 +123,12 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
         cp %{_compiledir}/build/lib/libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib
         ln -sf libjemalloc.so.2 %{buildroot}%{homepath}/jemalloc/lib/libjemalloc.so
     fi
-    if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then
-        cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib
-    fi
-    if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then
-        cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib
-    fi
+#    if [ -f %{_compiledir}/build/lib/libjemalloc.a ]; then
+#        cp %{_compiledir}/build/lib/libjemalloc.a %{buildroot}%{homepath}/jemalloc/lib
+#    fi
+#    if [ -f %{_compiledir}/build/lib/libjemalloc_pic.a ]; then
+#        cp %{_compiledir}/build/lib/libjemalloc_pic.a %{buildroot}%{homepath}/jemalloc/lib
+#    fi
     if [ -f %{_compiledir}/build/lib/pkgconfig/jemalloc.pc ]; then
         cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
     fi

packaging/tools/install.sh

@@ -315,13 +315,13 @@ function install_jemalloc() {
       ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
       ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
       ${csudo}/usr/bin/install -c -d /usr/local/lib
-      if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
-        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
-      fi
-      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
-        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
-      fi
-      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+      # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
+      #   ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
+      # fi
+      # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+      #   ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
+      # fi
+      if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then
         ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
         ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
       fi

packaging/tools/install_client.sh

@@ -214,13 +214,13 @@ function install_jemalloc() {
             ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
             ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
             ${csudo}/usr/bin/install -c -d /usr/local/lib
-            if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
-                ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
-            fi
-            if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
-                ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
-            fi
-            if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+            # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
+            #     ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
+            # fi
+            # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+            #     ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
+            # fi
+            if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then
                 ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
                 ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
             fi

packaging/tools/make_install.sh

@@ -241,10 +241,10 @@ function install_jemalloc() {
       ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib
       ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1
       ${csudo}/usr/bin/install -c -d /usr/local/lib
-      [ -f ${binary_dir}/build/lib/libjemalloc.a ] &&
-        ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib
-      [ -f ${binary_dir}/build/lib/libjemalloc_pic.a ] &&
-        ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib
+      # [ -f ${binary_dir}/build/lib/libjemalloc.a ] &&
+      #   ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib
+      # [ -f ${binary_dir}/build/lib/libjemalloc_pic.a ] &&
+      #   ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib
       if [ -f "${binary_dir}/build/lib/pkgconfig/jemalloc.pc" ]; then
         ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
         ${csudo}/usr/bin/install -c -m 644 ${binary_dir}/build/lib/pkgconfig/jemalloc.pc \

packaging/tools/makeclient.sh

@@ -118,12 +118,12 @@ if [ -f ${build_dir}/bin/jemalloc-config ]; then
     cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib
     ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so
   fi
-  if [ -f ${build_dir}/lib/libjemalloc.a ]; then
-    cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
-  fi
-  if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
-    cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
-  fi
+  # if [ -f ${build_dir}/lib/libjemalloc.a ]; then
+  #   cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
+  # fi
+  # if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
+  #   cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
+  # fi
   if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then
     cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig
   fi

packaging/tools/makepkg.sh

@@ -217,12 +217,12 @@ if [ -f ${build_dir}/bin/jemalloc-config ]; then
     cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib
     ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so
   fi
-  if [ -f ${build_dir}/lib/libjemalloc.a ]; then
-    cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
-  fi
-  if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
-    cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
-  fi
+  # if [ -f ${build_dir}/lib/libjemalloc.a ]; then
+  #   cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
+  # fi
+  # if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
+  #   cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
+  # fi
   if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then
     cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig
   fi

packaging/tools/post.sh

@@ -169,13 +169,13 @@ function install_jemalloc() {
       ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib
       ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so
       ${csudo}/usr/bin/install -c -d /usr/local/lib
-      if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
-        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
-      fi
-      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
-        ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
-      fi
-      if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+      # if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then
+      #   ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib
+      # fi
+      # if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then
+      #   ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib
+      # fi
+      if [ -f ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc ]; then
         ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig
         ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
       fi

source/common/src/tdatablock.c

@@ -1590,18 +1590,35 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int
   int32_t nRows = payloadSize / rowSize;
   ASSERT(nRows >= 1);
 
-  // the true value must be less than the value of nRows
-  int32_t additional = 0;
+  int32_t numVarCols = 0;
+  int32_t numFixCols = 0;
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      additional += nRows * sizeof(int32_t);
+      ++numVarCols;
+    } else {
+      ++numFixCols;
+    }
+  }
+
+  // find the data payload whose size is greater than payloadSize
+  int result = -1;
+  int start = 1;
+  int end = nRows;
+  while (start <= end) {
+    int mid = start + (end - start) / 2;
+    //data size + var data type columns offset + fixed data type columns bitmap len 
+    int midSize = rowSize * mid + numVarCols * sizeof(int32_t) * mid + numFixCols * BitmapLen(mid); 
+    if (midSize > payloadSize) {
+      result = mid;
+      end = mid - 1;
     } else {
-      additional += BitmapLen(nRows);
+      start = mid + 1;
     }
   }
 
-  int32_t newRows = (payloadSize - additional) / rowSize;
+  int32_t newRows = (result != -1) ? result - 1 : nRows;
+  // the true value must be less than the value of nRows
   ASSERT(newRows <= nRows && newRows >= 1);
 
   return newRows;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -712,6 +712,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME_PUSH, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_WALINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -23,10 +23,6 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
   SEpSet epSet = {0};
   dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet);
 
-  if (epSet.numOfEps == 1) {
-    return;
-  }
-
   const int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
   pMsg->pCont = rpcMallocCont(contLen);
   if (pMsg->pCont == NULL) {

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -105,6 +105,7 @@ typedef struct {
   SHashObj      *dnodeHash;
   TdThreadRwlock lock;
   SMsgCb         msgCb;
+  bool           validMnodeEps;
 } SDnodeData;
 
 typedef struct {

source/dnode/mgmt/node_util/src/dmEps.c

@@ -288,6 +288,8 @@ static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) {
     taosHashPut(pData->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
   }
 
+  pData->validMnodeEps = true;
+
   dmPrintEps(pData);
 }
 
@@ -348,6 +350,7 @@ void dmRotateMnodeEpSet(SDnodeData *pData) {
 }
 
 void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) {
+  if(!pData->validMnodeEps) return;
   dmGetMnodeEpSet(pData, pEpSet);
   dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse);
   for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {

source/dnode/mnode/impl/inc/mndDef.h

@@ -467,6 +467,7 @@ typedef struct {
   int8_t         replica;
   int16_t        numOfColumns;
   int32_t        numOfRows;
+  int32_t        curIterPackedRows;
   void*          pIter;
   SMnode*        pMnode;
   STableMetaRsp* pMeta;
@@ -559,24 +560,25 @@ void*           tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer
 
 typedef struct {
   int32_t vgId;
-  char*   qmsg;  // SubPlanToString
+//  char*   qmsg;  // SubPlanToString
   SEpSet  epSet;
 } SMqVgEp;
 
 SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp);
 void     tDeleteSMqVgEp(SMqVgEp* pVgEp);
 int32_t  tEncodeSMqVgEp(void** buf, const SMqVgEp* pVgEp);
-void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
+void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp, int8_t sver);
 
 typedef struct {
   int64_t consumerId;  // -1 for unassigned
   SArray* vgs;         // SArray<SMqVgEp*>
+  SArray* offsetRows; // SArray<OffsetRows*>
 } SMqConsumerEp;
 
 SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp);
 void           tDeleteSMqConsumerEp(void* pEp);
 int32_t        tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp);
-void*          tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp);
+void*          tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp, int8_t sver);
 
 typedef struct {
   char      key[TSDB_SUBSCRIBE_KEY_LEN];
@@ -588,34 +590,36 @@ typedef struct {
   int64_t   stbUid;
   SHashObj* consumerHash;   // consumerId -> SMqConsumerEp
   SArray*   unassignedVgs;  // SArray<SMqVgEp*>
+  SArray*   offsetRows;
   char      dbName[TSDB_DB_FNAME_LEN];
+  char*     qmsg; // SubPlanToString
 } SMqSubscribeObj;
 
 SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
 SMqSubscribeObj* tCloneSubscribeObj(const SMqSubscribeObj* pSub);
 void             tDeleteSubscribeObj(SMqSubscribeObj* pSub);
 int32_t          tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub);
-void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
-
-typedef struct {
-  int32_t epoch;
-  SArray* consumers;  // SArray<SMqConsumerEp*>
-} SMqSubActionLogEntry;
-
-SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
-void                  tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
-int32_t               tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry);
-void*                 tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry);
-
-typedef struct {
-  char    key[TSDB_SUBSCRIBE_KEY_LEN];
-  SArray* logs;  // SArray<SMqSubActionLogEntry*>
-} SMqSubActionLogObj;
-
-SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog);
-void                tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog);
-int32_t             tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog);
-void*               tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog);
+void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub, int8_t sver);
+
+//typedef struct {
+//  int32_t epoch;
+//  SArray* consumers;  // SArray<SMqConsumerEp*>
+//} SMqSubActionLogEntry;
+
+//SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+//void                  tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
+//int32_t               tEncodeSMqSubActionLogEntry(void** buf, const SMqSubActionLogEntry* pEntry);
+//void*                 tDecodeSMqSubActionLogEntry(const void* buf, SMqSubActionLogEntry* pEntry);
+//
+//typedef struct {
+//  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+//  SArray* logs;  // SArray<SMqSubActionLogEntry*>
+//} SMqSubActionLogObj;
+//
+//SMqSubActionLogObj* tCloneSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+//void                tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog);
+//int32_t             tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog);
+//void*               tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog);
 
 typedef struct {
   int32_t           oldConsumerNum;
@@ -634,7 +638,7 @@ typedef struct {
   SArray*               removedConsumers;  // SArray<int64_t>
   SArray*               modifyConsumers;   // SArray<int64_t>
   SMqSubscribeObj*      pSub;
-  SMqSubActionLogEntry* pLogEntry;
+//  SMqSubActionLogEntry* pLogEntry;
 } SMqRebOutputObj;
 
 typedef struct {

source/dnode/mnode/impl/src/mndConsumer.c

@@ -644,7 +644,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
   SCMSubscribeReq subscribe = {0};
   tDeserializeSCMSubscribeReq(msgStr, &subscribe);
 
-  uint64_t        consumerId = subscribe.consumerId;
+  int64_t        consumerId = subscribe.consumerId;
   char           *cgroup = subscribe.cgroup;
   SMqConsumerObj *pExistedConsumer = NULL;
   SMqConsumerObj *pConsumerNew = NULL;

source/dnode/mnode/impl/src/mndDef.c

@@ -187,14 +187,14 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) {
   SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp));
   if (pVgEpNew == NULL) return NULL;
   pVgEpNew->vgId = pVgEp->vgId;
-  pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
+//  pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg);
   pVgEpNew->epSet = pVgEp->epSet;
   return pVgEpNew;
 }
 
 void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
   if (pVgEp) {
-    taosMemoryFreeClear(pVgEp->qmsg);
+//    taosMemoryFreeClear(pVgEp->qmsg);
     taosMemoryFree(pVgEp);
   }
 }
@@ -202,14 +202,18 @@ void tDeleteSMqVgEp(SMqVgEp *pVgEp) {
 int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
-  tlen += taosEncodeString(buf, pVgEp->qmsg);
+//  tlen += taosEncodeString(buf, pVgEp->qmsg);
   tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
   return tlen;
 }
 
-void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
+void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp, int8_t sver) {
   buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
-  buf = taosDecodeString(buf, &pVgEp->qmsg);
+  if(sver == 1){
+    uint64_t size = 0;
+    buf = taosDecodeVariantU64(buf, &size);
+    buf = POINTER_SHIFT(buf, size);
+  }
   buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
   return (void *)buf;
 }
@@ -395,6 +399,22 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
   tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
+  int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows);
+  tlen += taosEncodeFixedI32(buf, szVgs);
+  for (int32_t j= 0; j < szVgs; ++j) {
+    OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j);
+    tlen += taosEncodeFixedI32(buf, offRows->vgId);
+    tlen += taosEncodeFixedI64(buf, offRows->rows);
+    tlen += taosEncodeFixedI8(buf, offRows->offset.type);
+    if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
+      tlen += taosEncodeFixedI64(buf, offRows->offset.uid);
+      tlen += taosEncodeFixedI64(buf, offRows->offset.ts);
+    } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
+      tlen += taosEncodeFixedI64(buf, offRows->offset.version);
+    } else {
+      // do nothing
+    }
+  }
 #if 0
   int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
   tlen += taosEncodeFixedI32(buf, sz);
@@ -406,9 +426,9 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
   return tlen;
 }
 
-void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) {
+void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) {
   buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
-  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp));
+  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver);
 #if 0
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
@@ -419,7 +439,28 @@ void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) {
     taosArrayPush(pConsumerEp->vgs, &pVgEp);
   }
 #endif
-
+  if (sver > 1){
+    int32_t szVgs = 0;
+    buf = taosDecodeFixedI32(buf, &szVgs);
+    if(szVgs > 0){
+      pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
+      if (NULL == pConsumerEp->offsetRows) return NULL;
+      for (int32_t j= 0; j < szVgs; ++j) {
+        OffsetRows* offRows = taosArrayReserve(pConsumerEp->offsetRows, 1);
+        buf = taosDecodeFixedI32(buf, &offRows->vgId);
+        buf = taosDecodeFixedI64(buf, &offRows->rows);
+        buf = taosDecodeFixedI8(buf, &offRows->offset.type);
+        if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
+          buf = taosDecodeFixedI64(buf, &offRows->offset.uid);
+          buf = taosDecodeFixedI64(buf, &offRows->offset.ts);
+        } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
+          buf = taosDecodeFixedI64(buf, &offRows->offset.version);
+        } else {
+          // do nothing
+        }
+      }
+    }
+  }
   return (void *)buf;
 }
 
@@ -468,7 +509,9 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
     taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp));
   }
   pSubNew->unassignedVgs = taosArrayDup(pSub->unassignedVgs, (__array_item_dup_fn_t)tCloneSMqVgEp);
+  pSubNew->offsetRows = taosArrayDup(pSub->offsetRows, NULL);
   memcpy(pSubNew->dbName, pSub->dbName, TSDB_DB_FNAME_LEN);
+  pSubNew->qmsg = taosStrdup(pSub->qmsg);
   return pSubNew;
 }
 
@@ -482,6 +525,8 @@ void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
   }
   taosHashCleanup(pSub->consumerHash);
   taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp);
+  taosMemoryFreeClear(pSub->qmsg);
+  taosArrayDestroy(pSub->offsetRows);
 }
 
 int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
@@ -508,10 +553,27 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
   if (cnt != sz) return -1;
   tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
   tlen += taosEncodeString(buf, pSub->dbName);
+  int32_t szVgs = taosArrayGetSize(pSub->offsetRows);
+  tlen += taosEncodeFixedI32(buf, szVgs);
+  for (int32_t j= 0; j < szVgs; ++j) {
+    OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j);
+    tlen += taosEncodeFixedI32(buf, offRows->vgId);
+    tlen += taosEncodeFixedI64(buf, offRows->rows);
+    tlen += taosEncodeFixedI8(buf, offRows->offset.type);
+    if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
+      tlen += taosEncodeFixedI64(buf, offRows->offset.uid);
+      tlen += taosEncodeFixedI64(buf, offRows->offset.ts);
+    } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
+      tlen += taosEncodeFixedI64(buf, offRows->offset.version);
+    } else {
+      // do nothing
+    }
+  }
+  tlen += taosEncodeString(buf, pSub->qmsg);
   return tlen;
 }
 
-void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
+void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) {
   //
   buf = taosDecodeStringTo(buf, pSub->key);
   buf = taosDecodeFixedI64(buf, &pSub->dbUid);
@@ -526,74 +588,99 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
   pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
   for (int32_t i = 0; i < sz; i++) {
     SMqConsumerEp consumerEp = {0};
-    buf = tDecodeSMqConsumerEp(buf, &consumerEp);
+    buf = tDecodeSMqConsumerEp(buf, &consumerEp, sver);
     taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp));
   }
 
-  buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp));
+  buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver);
   buf = taosDecodeStringTo(buf, pSub->dbName);
+  if (sver > 1){
+    int32_t szVgs = 0;
+    buf = taosDecodeFixedI32(buf, &szVgs);
+    if(szVgs > 0){
+      pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
+      if (NULL == pSub->offsetRows) return NULL;
+      for (int32_t j= 0; j < szVgs; ++j) {
+        OffsetRows* offRows = taosArrayReserve(pSub->offsetRows, 1);
+        buf = taosDecodeFixedI32(buf, &offRows->vgId);
+        buf = taosDecodeFixedI64(buf, &offRows->rows);
+        buf = taosDecodeFixedI8(buf, &offRows->offset.type);
+        if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
+          buf = taosDecodeFixedI64(buf, &offRows->offset.uid);
+          buf = taosDecodeFixedI64(buf, &offRows->offset.ts);
+        } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
+          buf = taosDecodeFixedI64(buf, &offRows->offset.version);
+        } else {
+          // do nothing
+        }
+      }
+    }
+    buf = taosDecodeString(buf, &pSub->qmsg);
+  }else{
+    pSub->qmsg = taosStrdup("");
+  }
   return (void *)buf;
 }
 
-SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
-  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
-  if (pEntryNew == NULL) return NULL;
-  pEntryNew->epoch = pEntry->epoch;
-  pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
-  return pEntryNew;
-}
-
-void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
-  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
-}
-
-int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
-  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
-  return tlen;
-}
-
-void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
-  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
-  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
-  return (void *)buf;
-}
-
-SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
-  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
-  if (pLogNew == NULL) return pLogNew;
-  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
-  pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
-  return pLogNew;
-}
-
-void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
-  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
-}
-
-int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pLog->key);
-  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
-  return tlen;
-}
-
-void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
-  buf = taosDecodeStringTo(buf, pLog->key);
-  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
-  return (void *)buf;
-}
-
-int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pOffset->key);
-  tlen += taosEncodeFixedI64(buf, pOffset->offset);
-  return tlen;
-}
-
-void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) {
-  buf = taosDecodeStringTo(buf, pOffset->key);
-  buf = taosDecodeFixedI64(buf, &pOffset->offset);
-  return buf;
-}
+//SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+//  SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
+//  if (pEntryNew == NULL) return NULL;
+//  pEntryNew->epoch = pEntry->epoch;
+//  pEntryNew->consumers = taosArrayDup(pEntry->consumers, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
+//  return pEntryNew;
+//}
+//
+//void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
+//  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
+//}
+//
+//int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
+//  int32_t tlen = 0;
+//  tlen += taosEncodeFixedI32(buf, pEntry->epoch);
+//  tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
+//  return tlen;
+//}
+//
+//void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
+//  buf = taosDecodeFixedI32(buf, &pEntry->epoch);
+//  buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+//  return (void *)buf;
+//}
+//
+//SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+//  SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
+//  if (pLogNew == NULL) return pLogNew;
+//  memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
+//  pLogNew->logs = taosArrayDup(pLog->logs, (__array_item_dup_fn_t)tCloneSMqConsumerEp);
+//  return pLogNew;
+//}
+//
+//void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
+//  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
+//}
+//
+//int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
+//  int32_t tlen = 0;
+//  tlen += taosEncodeString(buf, pLog->key);
+//  tlen += taosEncodeArray(buf, pLog->logs, (FEncode)tEncodeSMqSubActionLogEntry);
+//  return tlen;
+//}
+//
+//void *tDecodeSMqSubActionLogObj(const void *buf, SMqSubActionLogObj *pLog) {
+//  buf = taosDecodeStringTo(buf, pLog->key);
+//  buf = taosDecodeArray(buf, &pLog->logs, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
+//  return (void *)buf;
+//}
+//
+//int32_t tEncodeSMqOffsetObj(void **buf, const SMqOffsetObj *pOffset) {
+//  int32_t tlen = 0;
+//  tlen += taosEncodeString(buf, pOffset->key);
+//  tlen += taosEncodeFixedI64(buf, pOffset->offset);
+//  return tlen;
+//}
+//
+//void *tDecodeSMqOffsetObj(void *buf, SMqOffsetObj *pOffset) {
+//  buf = taosDecodeStringTo(buf, pOffset->key);
+//  buf = taosDecodeFixedI64(buf, &pOffset->offset);
+//  return buf;
+//}

source/dnode/mnode/impl/src/mndProfile.c

@@ -764,104 +764,129 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
   return numOfRows;
 }
 
-static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
-  SMnode   *pMnode = pReq->info.node;
-  SSdb     *pSdb = pMnode->pSdb;
-  int32_t   numOfRows = 0;
-  int32_t   cols = 0;
-  SConnObj *pConn = NULL;
-
-  if (pShow->pIter == NULL) {
-    SProfileMgmt *pMgmt = &pMnode->profileMgmt;
-    pShow->pIter = taosCacheCreateIter(pMgmt->connCache);
+/**
+ * @param pConn the conn queries pack from
+ * @param[out] pBlock the block data packed into
+ * @param offset skip [offset] queries in pConn
+ * @param rowsToPack at most rows to pack
+ * @return rows packed
+*/
+static int32_t packQueriesIntoBlock(SShowObj* pShow, SConnObj* pConn, SSDataBlock* pBlock, uint32_t offset, uint32_t rowsToPack) {
+  int32_t cols = 0;
+  taosRLockLatch(&pConn->queryLock);
+  int32_t numOfQueries = taosArrayGetSize(pConn->pQueries);
+  if (NULL == pConn->pQueries || numOfQueries <= offset) {
+    taosRUnLockLatch(&pConn->queryLock);
+    return 0;
   }
 
-  while (numOfRows < rows) {
-    pConn = mndGetNextConn(pMnode, pShow->pIter);
-    if (pConn == NULL) {
-      pShow->pIter = NULL;
-      break;
-    }
+  int32_t i = offset;
+  for (; i < numOfQueries && (i - offset) < rowsToPack; ++i) {
+    int32_t curRowIndex = pBlock->info.rows;
+    SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i);
+    cols = 0;
 
-    taosRLockLatch(&pConn->queryLock);
-    if (NULL == pConn->pQueries || taosArrayGetSize(pConn->pQueries) <= 0) {
-      taosRUnLockLatch(&pConn->queryLock);
-      continue;
-    }
+    char queryId[26 + VARSTR_HEADER_SIZE] = {0};
+    sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid);
+    varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)queryId, false);
 
-    int32_t numOfQueries = taosArrayGetSize(pConn->pQueries);
-    for (int32_t i = 0; i < numOfQueries && numOfRows < rows; ++i) {
-      SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i);
-      cols = 0;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->queryId, false);
 
-      char queryId[26 + VARSTR_HEADER_SIZE] = {0};
-      sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid);
-      varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]);
-      SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)queryId, false);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pConn->id, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->queryId, false);
+    char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE];
+    STR_TO_VARSTR(app, pConn->app);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)app, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->id, false);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pConn->pid, false);
 
-      char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE];
-      STR_TO_VARSTR(app, pConn->app);
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)app, false);
+    char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(user, pConn->user);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)user, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pConn->pid, false);
+    char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
+    sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port);
+    varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)endpoint, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stime, false);
 
-      char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
-      STR_TO_VARSTR(user, pConn->user);
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)user, false);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->useconds, false);
 
-      char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0};
-      sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port);
-      varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]);
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)endpoint, false);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->stableQuery, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stime, false);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)&pQuery->subPlanNum, false);
+
+    char    subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0};
+    int32_t strSize = sizeof(subStatus);
+    int32_t offset = VARSTR_HEADER_SIZE;
+    for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) {
+      if (i) {
+        offset += snprintf(subStatus + offset, strSize - offset - 1, ",");
+      }
+      SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i);
+      offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status);
+    }
+    varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, subStatus, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->useconds, false);
+    char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
+    STR_TO_VARSTR(sql, pQuery->sql);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataSetVal(pColInfo, curRowIndex, (const char *)sql, false);
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false);
+    pBlock->info.rows++;
+  }
 
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false);
+  taosRUnLockLatch(&pConn->queryLock);
+  return i - offset;
+}
 
-      char    subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0};
-      int32_t strSize = sizeof(subStatus);
-      int32_t offset = VARSTR_HEADER_SIZE;
-      for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) {
-        if (i) {
-          offset += snprintf(subStatus + offset, strSize - offset - 1, ",");
-        }
-        SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i);
-        offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status);
-      }
-      varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]);
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, subStatus, false);
+static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
+  SMnode *  pMnode = pReq->info.node;
+  SSdb *    pSdb = pMnode->pSdb;
+  int32_t   numOfRows = 0;
+  SConnObj *pConn = NULL;
 
-      char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
-      STR_TO_VARSTR(sql, pQuery->sql);
-      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-      colDataSetVal(pColInfo, numOfRows, (const char *)sql, false);
+  if (pShow->pIter == NULL) {
+    SProfileMgmt *pMgmt = &pMnode->profileMgmt;
+    pShow->pIter = taosCacheCreateIter(pMgmt->connCache);
+  }
 
-      numOfRows++;
+  // means fetched some data last time for this conn
+  if (pShow->curIterPackedRows > 0) {
+    size_t len = 0;
+    pConn = taosCacheIterGetData(pShow->pIter, &len);
+    if (pConn && (taosArrayGetSize(pConn->pQueries) > pShow->curIterPackedRows)) {
+      numOfRows = packQueriesIntoBlock(pShow, pConn, pBlock, pShow->curIterPackedRows, rows);
+      pShow->curIterPackedRows += numOfRows;
     }
-
-    taosRUnLockLatch(&pConn->queryLock);
   }
 
+  while (numOfRows < rows) {
+    pConn = mndGetNextConn(pMnode, pShow->pIter);
+    if (pConn == NULL) {
+      pShow->pIter = NULL;
+      break;
+    }
+
+    int32_t packedRows = packQueriesIntoBlock(pShow, pConn, pBlock, 0, rows - numOfRows);
+    pShow->curIterPackedRows = packedRows;
+    numOfRows += packedRows;
+  }
   pShow->numOfRows += numOfRows;
   return numOfRows;
 }

source/dnode/mnode/impl/src/mndScheduler.c

@@ -570,23 +570,19 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
 
     mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
 
-    if (pSubplan) {
-      int32_t msgLen;
+    sdbRelease(pSdb, pVgroup);
+  }
 
-      pSubplan->execNode.epSet = pVgEp->epSet;
-      pSubplan->execNode.nodeId = pVgEp->vgId;
+  if (pSubplan) {
+    int32_t msgLen;
 
-      if (qSubPlanToString(pSubplan, &pVgEp->qmsg, &msgLen) < 0) {
-        sdbRelease(pSdb, pVgroup);
-        qDestroyQueryPlan(pPlan);
-        terrno = TSDB_CODE_QRY_INVALID_INPUT;
-        return -1;
-      }
-    } else {
-      pVgEp->qmsg = taosStrdup("");
+    if (qSubPlanToString(pSubplan, &pSub->qmsg, &msgLen) < 0) {
+      qDestroyQueryPlan(pPlan);
+      terrno = TSDB_CODE_QRY_INVALID_INPUT;
+      return -1;
     }
-
-    sdbRelease(pSdb, pVgroup);
+  } else {
+    pSub->qmsg = taosStrdup("");
   }
 
   qDestroyQueryPlan(pPlan);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -24,7 +24,7 @@
 #include "tcompare.h"
 #include "tname.h"
 
-#define MND_SUBSCRIBE_VER_NUMBER   1
+#define MND_SUBSCRIBE_VER_NUMBER   2
 #define MND_SUBSCRIBE_RESERVE_SIZE 64
 
 #define MND_SUBSCRIBE_REBALANCE_CNT 3
@@ -99,13 +99,23 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
   return pSub;
 }
 
-static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub,
-                                    const SMqRebOutputVg *pRebVg) {
+static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub,
+                                    const SMqRebOutputVg *pRebVg, SSubplan* pPlan) {
   SMqRebVgReq req = {0};
   req.oldConsumerId = pRebVg->oldConsumerId;
   req.newConsumerId = pRebVg->newConsumerId;
   req.vgId = pRebVg->pVgEp->vgId;
-  req.qmsg = pRebVg->pVgEp->qmsg;
+  if(pPlan){
+    pPlan->execNode.epSet = pRebVg->pVgEp->epSet;
+    pPlan->execNode.nodeId = pRebVg->pVgEp->vgId;
+    int32_t msgLen;
+    if (qSubPlanToString(pPlan, &req.qmsg, &msgLen) < 0) {
+      terrno = TSDB_CODE_QRY_INVALID_INPUT;
+      return -1;
+    }
+  }else{
+    req.qmsg = taosStrdup("");
+  }
   req.subType = pSub->subType;
   req.withMeta = pSub->withMeta;
   req.suid = pSub->stbUid;
@@ -115,6 +125,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
   int32_t ret = 0;
   tEncodeSize(tEncodeSMqRebVgReq, &req, tlen, ret);
   if (ret < 0) {
+    taosMemoryFree(req.qmsg);
     return -1;
   }
 
@@ -122,6 +133,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
   void   *buf = taosMemoryMalloc(tlen);
   if (buf == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(req.qmsg);
     return -1;
   }
 
@@ -135,17 +147,19 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
   if (tEncodeSMqRebVgReq(&encoder, &req) < 0) {
     taosMemoryFreeClear(buf);
     tEncoderClear(&encoder);
+    taosMemoryFree(req.qmsg);
     return -1;
   }
   tEncoderClear(&encoder);
   *pBuf = buf;
   *pLen = tlen;
 
+  taosMemoryFree(req.qmsg);
   return 0;
 }
 
-static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub,
-                                        const SMqRebOutputVg *pRebVg) {
+static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub,
+                                        const SMqRebOutputVg *pRebVg, SSubplan* pPlan) {
 //  if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
 //    terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
 //    return -1;
@@ -153,7 +167,7 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM
 
   void   *buf;
   int32_t tlen;
-  if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) {
+  if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg, pPlan) < 0) {
     return -1;
   }
 
@@ -255,7 +269,7 @@ static void doAddNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pI
   for (int32_t i = 0; i < numOfNewConsumers; i++) {
     int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
 
-    SMqConsumerEp newConsumerEp;
+    SMqConsumerEp newConsumerEp = {0};
     newConsumerEp.consumerId = consumerId;
     newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
 
@@ -483,14 +497,25 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
 }
 
 static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) {
+  struct SSubplan* pPlan = NULL;
+  if(strcmp(pOutput->pSub->qmsg, "") != 0){
+    int32_t code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan);
+    if (code != TSDB_CODE_SUCCESS) {
+      terrno = code;
+      return -1;
+    }
+  }
+
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
   if (pTrans == NULL) {
+    nodesDestroyNode((SNode*)pPlan);
     return -1;
   }
 
   mndTransSetDbName(pTrans, pOutput->pSub->dbName, NULL);
   if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
     mndTransDrop(pTrans);
+    nodesDestroyNode((SNode*)pPlan);
     return -1;
   }
 
@@ -500,11 +525,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
   int32_t       vgNum = taosArrayGetSize(rebVgs);
   for (int32_t i = 0; i < vgNum; i++) {
     SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i);
-    if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) {
+    if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg, pPlan) < 0) {
       mndTransDrop(pTrans);
+      nodesDestroyNode((SNode*)pPlan);
       return -1;
     }
   }
+  nodesDestroyNode((SNode*)pPlan);
 
   // 2. redo log: subscribe and vg assignment
   // subscribe
@@ -809,7 +836,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
   int8_t sver = 0;
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER;
 
-  if (sver != MND_SUBSCRIBE_VER_NUMBER) {
+  if (sver > MND_SUBSCRIBE_VER_NUMBER || sver < 1) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
     goto SUB_DECODE_OVER;
   }
@@ -828,7 +855,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) {
   SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER);
   SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER);
 
-  if (tDecodeSubscribeObj(buf, pSub) == NULL) {
+  if (tDecodeSubscribeObj(buf, pSub, sver) == NULL) {
     goto SUB_DECODE_OVER;
   }
 

source/dnode/vnode/src/inc/tq.h

@@ -139,6 +139,7 @@ static STqMgmt tqMgmt = {0};
 
 int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
 int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
+void    tqDestroyTqHandle(void* data);
 
 // tqRead
 int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
@@ -161,6 +162,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq);
 int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
 int32_t tqMetaRestoreCheckInfo(STQ* pTq);
+int32_t tqMetaGetHandle(STQ* pTq, const char* key);
+int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle);
 
 STqOffsetStore* tqOffsetOpen(STQ* pTq);
 void            tqOffsetClose(STqOffsetStore*);

source/dnode/vnode/src/inc/tsdb.h

@@ -357,19 +357,25 @@ typedef struct {
   STSchema                            *pTSchema;
 } SRocksCache;
 
+typedef struct {
+  STsdb *pTsdb;
+  int    flush_count;
+} SCacheFlushState;
+
 struct STsdb {
-  char          *path;
-  SVnode        *pVnode;
-  STsdbKeepCfg   keepCfg;
-  TdThreadRwlock rwLock;
-  SMemTable     *mem;
-  SMemTable     *imem;
-  STsdbFS        fs;
-  SLRUCache     *lruCache;
-  TdThreadMutex  lruMutex;
-  SLRUCache     *biCache;
-  TdThreadMutex  biMutex;
-  SRocksCache    rCache;
+  char            *path;
+  SVnode          *pVnode;
+  STsdbKeepCfg     keepCfg;
+  TdThreadRwlock   rwLock;
+  SMemTable       *mem;
+  SMemTable       *imem;
+  STsdbFS          fs;
+  SLRUCache       *lruCache;
+  SCacheFlushState flushState;
+  TdThreadMutex    lruMutex;
+  SLRUCache       *biCache;
+  TdThreadMutex    biMutex;
+  SRocksCache      rCache;
 };
 
 struct TSDBKEY {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -226,6 +226,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
 int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
 int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen);
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg);
 
 // tq-stream

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2846,18 +2846,18 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
       setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order);
       return code;
     }
-    
+
     pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr);
     if (pBlockScanInfo == NULL) {
       goto _end;
     }
 
-    TSDBKEY   keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
+    TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
 
     // it is a clean block, load it directly
     if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) &&
         pBlock->nRow <= pReader->resBlockInfo.capacity) {
-      if (asc || (!hasDataInLastBlock(pLastBlockReader))) {
+      if (asc || (!hasDataInLastBlock(pLastBlockReader) && (pBlock->maxKey.ts > keyInBuf.ts))) {
         code = copyBlockDataToSDataBlock(pReader);
         if (code) {
           goto _end;