other: merge 3.0

.gitignore

@@ -16,7 +16,6 @@ debug/
 release/
 target/
 debs/
-deps/
 rpms/
 mac/
 *.pyc

cmake/cmake.define

 cmake_minimum_required(VERSION 3.0)
 
-set(CMAKE_VERBOSE_MAKEFILE OFF)
+set(CMAKE_VERBOSE_MAKEFILE ON)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
 
 #set output directory

cmake/cmake.platform

@@ -172,5 +172,15 @@ ENDIF()
 
 MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR})
 
+set(TD_DEPS_DIR "x86")
+if (TD_LINUX)
+  IF (TD_ARM_64 OR TD_ARM_32)
+    set(TD_DEPS_DIR "arm")
+  ELSE() 
+    set(TD_DEPS_DIR "x86")
+  ENDIF()
+endif()
+MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR})
+
 MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
 MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/rocksdb_CMakeLists.txt.in

 
 # rocksdb
+IF (NOT ${TD_LINUX})
 ExternalProject_Add(rocksdb
         GIT_REPOSITORY https://github.com/facebook/rocksdb.git
         GIT_TAG v8.1.1
@@ -9,3 +10,5 @@ ExternalProject_Add(rocksdb
         INSTALL_COMMAND ""
         TEST_COMMAND ""
         )
+ENDIF(NOT ${TD_LINUX})
+

contrib/CMakeLists.txt

@@ -78,10 +78,18 @@ if(${BUILD_WITH_LEVELDB})
 endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
+IF (NOT ${TD_LINUX})
 if(${BUILD_WITH_ROCKSDB})
     cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_ROCKSDB)
 endif(${BUILD_WITH_ROCKSDB})
+ELSE()
+if(${BUILD_WITH_ROCKSDB})
+    #cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    add_definitions(-DUSE_ROCKSDB)
+endif(${BUILD_WITH_ROCKSDB})
+    
+ENDIF(NOT ${TD_LINUX})
 
 # canonical-raft
 if(${BUILD_WITH_CRAFT})
@@ -175,8 +183,8 @@ if(${BUILD_TEST})
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
         )
     endif(${TD_DARWIN})
-    
-    
+
+
 endif(${BUILD_TEST})
 
 # cJson
@@ -227,6 +235,8 @@ endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
 # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev
+IF (NOT ${TD_LINUX})
+
 if(${BUILD_WITH_ROCKSDB})
     if(${TD_LINUX})
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result")
@@ -245,7 +255,7 @@ if(${BUILD_WITH_ROCKSDB})
     endif(${TD_DARWIN_ARM64})
 
     if (${TD_WINDOWS})
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")  
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
     endif(${TD_WINDOWS})
 
 
@@ -263,8 +273,8 @@ if(${BUILD_WITH_ROCKSDB})
         option(WITH_MD_LIBRARY "build with MD" OFF)
         set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
     endif(${TD_WINDOWS})
-    
-    
+
+
     option(WITH_FALLOCATE "" OFF)
     option(WITH_JEMALLOC "" OFF)
     option(WITH_GFLAGS "" OFF)
@@ -276,7 +286,7 @@ if(${BUILD_WITH_ROCKSDB})
     option(WITH_BENCHMARK_TOOLS "" OFF)
     option(WITH_TOOLS "" OFF)
     option(WITH_LIBURING "" OFF)
-    IF (TD_LINUX) 
+    IF (TD_LINUX)
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
     ELSE()
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
@@ -288,16 +298,17 @@ if(${BUILD_WITH_ROCKSDB})
     )
 endif(${BUILD_WITH_ROCKSDB})
 
+ENDIF(NOT ${TD_LINUX})
 # lucene
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
     add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
-      lucene++ 
+      lucene++
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
-    ) 
-     
+    )
+
 endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
@@ -357,7 +368,7 @@ if(${BUILD_MSVCREGEX})
     target_include_directories(msvcregex
         PRIVATE "msvcregex"
     )
-    target_link_libraries(msvcregex 
+    target_link_libraries(msvcregex
         INTERFACE Shell32
     )
     SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
@@ -417,8 +428,8 @@ if(${BUILD_WITH_BDB})
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb"
     )
-    target_link_libraries(bdb 
-        INTERFACE pthread 
+    target_link_libraries(bdb
+        INTERFACE pthread
     )
 endif(${BUILD_WITH_BDB})
 
@@ -430,12 +441,12 @@ if(${BUILD_WITH_SQLITE})
         IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/.libs/libsqlite3.a"
         INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/sqlite"
     )
-    target_link_libraries(sqlite 
-        INTERFACE m 
-        INTERFACE pthread 
+    target_link_libraries(sqlite
+        INTERFACE m
+        INTERFACE pthread
     )
     if(NOT TD_WINDOWS)
-        target_link_libraries(sqlite 
+        target_link_libraries(sqlite
             INTERFACE dl
         )
     endif(NOT TD_WINDOWS)
@@ -443,22 +454,22 @@ endif(${BUILD_WITH_SQLITE})
 
 # addr2line
 if(${BUILD_ADDR2LINE})
-    if(NOT ${TD_WINDOWS})    
-        check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H) 
-        check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H ) 
-        check_include_file( "inttypes.h"      HAVE_INTTYPES_H   ) 
-        check_include_file( "stddef.h"        HAVE_STDDEF_H   ) 
-        check_include_file( "stdlib.h"        HAVE_STDLIB_H   ) 
-        check_include_file( "string.h"        HAVE_STRING_H   ) 
-        check_include_file( "memory.h"        HAVE_MEMORY_H   ) 
-        check_include_file( "strings.h"       HAVE_STRINGS_H  ) 
+    if(NOT ${TD_WINDOWS})
+        check_include_file( "sys/types.h"     HAVE_SYS_TYPES_H)
+        check_include_file( "sys/stat.h"      HAVE_SYS_STAT_H )
+        check_include_file( "inttypes.h"      HAVE_INTTYPES_H   )
+        check_include_file( "stddef.h"        HAVE_STDDEF_H   )
+        check_include_file( "stdlib.h"        HAVE_STDLIB_H   )
+        check_include_file( "string.h"        HAVE_STRING_H   )
+        check_include_file( "memory.h"        HAVE_MEMORY_H   )
+        check_include_file( "strings.h"       HAVE_STRINGS_H  )
         check_include_file( "stdint.h"        HAVE_STDINT_H   )
         check_include_file( "unistd.h"        HAVE_UNISTD_H   )
         check_include_file( "sgidefs.h"       HAVE_SGIDEFS_H  )
         check_include_file( "stdafx.h"        HAVE_STDAFX_H   )
-        check_include_file( "elf.h"           HAVE_ELF_H      ) 
-        check_include_file( "libelf.h"        HAVE_LIBELF_H   ) 
-        check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H) 
+        check_include_file( "elf.h"           HAVE_ELF_H      )
+        check_include_file( "libelf.h"        HAVE_LIBELF_H   )
+        check_include_file( "libelf/libelf.h" HAVE_LIBELF_LIBELF_H)
         check_include_file( "alloca.h"        HAVE_ALLOCA_H   )
         check_include_file( "elfaccess.h"     HAVE_ELFACCESS_H)
         check_include_file( "sys/elf_386.h"   HAVE_SYS_ELF_386_H  )
@@ -466,7 +477,7 @@ if(${BUILD_ADDR2LINE})
         check_include_file( "sys/elf_sparc.h" HAVE_SYS_ELF_SPARC_H)
         check_include_file( "sys/ia64/elf.h"  HAVE_SYS_IA64_ELF_H )
         set(VERSION 0.3.1)
-        set(PACKAGE_VERSION "\"${VERSION}\"") 
+        set(PACKAGE_VERSION "\"${VERSION}\"")
         configure_file(libdwarf/cmake/config.h.cmake config.h)
         file(GLOB_RECURSE LIBDWARF_SOURCES "libdwarf/src/lib/libdwarf/*.c")
         add_library(libdwarf STATIC ${LIBDWARF_SOURCES})
@@ -497,6 +508,7 @@ if(${BUILD_GEOS})
     endif(${TD_LINUX})
     option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF)
     add_subdirectory(geos EXCLUDE_FROM_ALL)
+    unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
     target_include_directories(
         geos_c
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/geos/include>

docs/en/14-reference/03-connector/07-python.mdx

@@ -667,6 +667,137 @@ Insert with req_id argument
 </TabItem>
 </Tabs>
 
+### Parameter Binding
+
+The Python connector provides a parameter binding api for inserting data. Similar to most databases, TDengine currently only supports the question mark `?` to indicate the parameters to be bound.
+
+<Tabs>
+<TabItem value="native" label="native connection">
+
+#### Create Stmt
+
+Call the `statement` method in `Connection` to create the `stmt` for parameter binding.
+
+```
+import taos
+
+conn = taos.connect()
+stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+```
+
+#### parameter binding
+
+Call the `new_multi_binds` function to create the parameter list for parameter bindings.
+
+```
+params = new_multi_binds(16)
+params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
+params[1].bool((True, None, False))
+params[2].tinyint([-128, -128, None])  # -128 is tinyint null
+params[3].tinyint([0, 127, None])
+params[4].smallint([3, None, 2])
+params[5].int([3, 4, None])
+params[6].bigint([3, 4, None])
+params[7].tinyint_unsigned([3, 4, None])
+params[8].smallint_unsigned([3, 4, None])
+params[9].int_unsigned([3, 4, None])
+params[10].bigint_unsigned([3, 4, None])
+params[11].float([3, None, 1])
+params[12].double([3, None, 1.2])
+params[13].binary(["abc", "dddafadfadfadfadfa", None])
+params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
+params[15].timestamp([None, None, 1626861392591])
+```
+
+Call the `bind_param` (for a single row) method or the `bind_param_batch` (for multiple rows) method to set the values.
+
+```
+stmt.bind_param_batch(params)
+```
+
+#### execute sql
+
+Call `execute` method to execute sql.
+
+```
+stmt.execute()
+```
+
+#### Close Stmt
+
+```
+stmt.close()
+```
+
+#### Example
+
+```python
+{{#include docs/examples/python/stmt_example.py}}
+```
+</TabItem>
+
+<TabItem value="websocket" label="WebSocket connection">
+
+#### Create Stmt
+
+Call the `statement` method in `Connection` to create the `stmt` for parameter binding.
+
+```
+import taosws
+
+conn = taosws.connect('taosws://localhost:6041/test')
+stmt = conn.statement()
+```
+
+#### Prepare sql
+
+Call `prepare` method in stmt to prepare sql.
+
+```
+stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+```
+
+#### parameter binding
+
+Call the `bind_param` method to bind parameters.
+
+```
+stmt.bind_param([
+    taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+    taosws.ints_to_column([1, 2, 3, 4]),
+    taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+    taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+])
+```
+
+Call the `add_batch` method to add parameters to the batch.
+
+```
+stmt.add_batch()
+```
+
+#### execute sql
+
+Call `execute` method to execute sql.
+
+```
+stmt.execute()
+```
+
+#### Close Stmt
+
+```
+stmt.close()
+```
+
+#### Example
+
+```python
+{{#include docs/examples/python/stmt_websocket_example.py}}
+```
+</TabItem>
+</Tabs>
+
 ### Other sample programs
 
 | Example program links | Example program content |

docs/examples/python/stmt_example.py

+#!
+
+import taosws
+
+import taos
+
+db_name = 'test_ws_stmt'
+
+
+def before():
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.execute("create database %s" % db_name)
+    taos_conn.select_db(db_name)
+    taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
+    taos_conn.execute(
+        "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))")
+    taos_conn.close()
+
+
+def stmt_insert():
+    before()
+
+    conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name)
+
+    while True:
+        try:
+            stmt = conn.statement()
+            stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+
+            stmt.bind_param([
+                taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+                taosws.ints_to_column([1, 2, 3, 4]),
+                taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+                taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+            ])
+
+            stmt.add_batch()
+            rows = stmt.execute()
+            print(rows)
+            stmt.close()
+        except Exception as e:
+            if 'Retry needed' in e.args[0]:  # deal with [0x0125] Retry needed
+                continue
+            else:
+                raise e
+
+        break
+
+
+def stmt_insert_into_stable():
+    before()
+
+    conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name)
+
+    while True:
+        try:
+            stmt = conn.statement()
+            stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)")
+            stmt.set_tbname('stb1_1')
+            stmt.set_tags([
+                taosws.int_to_tag(1),
+                taosws.varchar_to_tag('aaa'),
+            ])
+            stmt.bind_param([
+                taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+                taosws.ints_to_column([1, 2, 3, 4]),
+                taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+                taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+            ])
+
+            stmt.add_batch()
+            rows = stmt.execute()
+            print(rows)
+            stmt.close()
+        except Exception as e:
+            if 'Retry needed' in e.args[0]:  # deal with [0x0125] Retry needed
+                continue
+            else:
+                raise e
+
+        break

docs/examples/python/stmt_websocket_example.py

+#!
+import time
+
+import taosws
+
+import taos
+
+
+def before_test(db_name):
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.execute("create database %s" % db_name)
+    taos_conn.select_db(db_name)
+    taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
+    taos_conn.execute(
+        "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))")
+    taos_conn.close()
+
+
+def after_test(db_name):
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.close()
+
+
+def stmt_insert():
+    db_name = 'test_ws_stmt_{}'.format(int(time.time()))
+    before_test(db_name)
+
+    conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name)
+
+    stmt = conn.statement()
+    stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+
+    stmt.bind_param([
+        taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+        taosws.ints_to_column([1, 2, 3, 4]),
+        taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+        taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+    ])
+
+    stmt.add_batch()
+    rows = stmt.execute()
+    assert rows == 4
+    stmt.close()
+    after_test(db_name)
+
+
+def stmt_insert_into_stable():
+    db_name = 'test_ws_stmt_{}'.format(int(time.time()))
+    before_test(db_name)
+
+    conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name)
+
+    stmt = conn.statement()
+    stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)")
+    stmt.set_tbname('stb1_1')
+    stmt.set_tags([
+        taosws.int_to_tag(1),
+        taosws.varchar_to_tag('aaa'),
+    ])
+    stmt.bind_param([
+        taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+        taosws.ints_to_column([1, 2, 3, 4]),
+        taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+        taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+    ])
+
+    stmt.add_batch()
+    rows = stmt.execute()
+    assert rows == 4
+    stmt.close()
+    after_test(db_name)
+
+
+if __name__ == '__main__':
+    stmt_insert()
+    stmt_insert_into_stable()

docs/zh/08-connector/30-python.mdx

@@ -672,6 +672,141 @@ consumer.close()
 </TabItem>
 </Tabs>
 
+### 通过参数绑定写入数据
+
+TDengine 的 Python 连接器支持参数绑定风格的 Prepare API 方式写入数据，和大多数数据库类似，目前仅支持用 `?` 来代表待绑定的参数。
+
+<Tabs>
+<TabItem value="native" label="原生连接">
+
+#### 创建 stmt
+
+Python 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。
+
+```
+import taos
+
+conn = taos.connect()
+stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+```
+
+#### 参数绑定
+
+调用 `new_multi_binds` 函数创建 params 列表，用于参数绑定。
+
+```
+params = new_multi_binds(16)
+params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
+params[1].bool((True, None, False))
+params[2].tinyint([-128, -128, None])  # -128 is tinyint null
+params[3].tinyint([0, 127, None])
+params[4].smallint([3, None, 2])
+params[5].int([3, 4, None])
+params[6].bigint([3, 4, None])
+params[7].tinyint_unsigned([3, 4, None])
+params[8].smallint_unsigned([3, 4, None])
+params[9].int_unsigned([3, 4, None])
+params[10].bigint_unsigned([3, 4, None])
+params[11].float([3, None, 1])
+params[12].double([3, None, 1.2])
+params[13].binary(["abc", "dddafadfadfadfadfa", None])
+params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
+params[15].timestamp([None, None, 1626861392591])
+```
+
+调用 stmt 的 `bind_param` 以单行的方式设置 values 或 `bind_param_batch` 以多行的方式设置 values 方法绑定参数。
+
+```
+stmt.bind_param_batch(params)
+```
+
+#### 执行 sql
+
+调用 stmt 的 `execute` 方法执行 sql
+
+```
+stmt.execute()
+```
+
+#### 关闭 stmt
+
+最后需要关闭 stmt。
+
+```
+stmt.close()
+```
+
+#### 示例代码
+
+```python
+{{#include docs/examples/python/stmt_example.py}}
+```
+</TabItem>
+
+<TabItem value="websocket" label="WebSocket 连接">
+
+#### 创建 stmt
+
+Python WebSocket 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。
+
+```
+import taosws
+
+conn = taosws.connect('taosws://localhost:6041/test')
+stmt = conn.statement()
+```
+
+#### 解析 sql
+
+调用 stmt 的 `prepare` 方法来解析 insert 语句。
+
+```
+stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+```
+
+#### 参数绑定
+
+调用 stmt 的 `bind_param` 方法绑定参数。
+
+```
+stmt.bind_param([
+    taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+    taosws.ints_to_column([1, 2, 3, 4]),
+    taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+    taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+])
+```
+
+调用 stmt 的 `add_batch` 方法，将参数加入批处理。
+
+```
+stmt.add_batch()
+```
+
+#### 执行 sql
+
+调用 stmt 的 `execute` 方法执行 sql
+
+```
+stmt.execute()
+```
+
+#### 关闭 stmt
+
+最后需要关闭 stmt。
+
+```
+stmt.close()
+```
+
+#### 示例代码
+
+```python
+{{#include docs/examples/python/stmt_websocket_example.py}}
+```
+</TabItem>
+</Tabs>
+
 ### 其它示例程序
 
 | 示例程序链接                                                                                                  | 示例程序内容            |

include/common/tglobal.h

@@ -120,6 +120,7 @@ extern bool    tsQueryUseNodeAllocator;
 extern bool    tsKeepColumnName;
 extern bool    tsEnableQueryHb;
 extern bool    tsEnableScience;
+extern bool    tsTtlChangeOnWrite;
 extern int32_t tsRedirectPeriod;
 extern int32_t tsRedirectFactor;
 extern int32_t tsRedirectMaxPeriod;

include/common/tmsg.h

@@ -945,7 +945,7 @@ int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
 int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
 
 typedef struct {
-  int32_t timestamp;
+  int32_t timestampSec;
 } SVDropTtlTableReq;
 
 int32_t tSerializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq);
@@ -2278,7 +2278,7 @@ typedef struct SVCreateTbReq {
   int32_t  flags;
   char*    name;
   tb_uid_t uid;
-  int64_t  ctime;
+  int64_t  btime;
   int32_t  ttl;
   int32_t  commentLen;
   char*    comment;
@@ -2415,10 +2415,12 @@ typedef struct {
   int32_t newTTL;
   int32_t newCommentLen;
   char*   newComment;
+  int64_t ctimeMs; // fill by vnode
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
 int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq);
+int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs);
 
 typedef struct {
   int32_t        code;
@@ -3436,6 +3438,7 @@ typedef struct SDeleteRes {
   int64_t  affectedRows;
   char     tableFName[TSDB_TABLE_NAME_LEN];
   char     tsColName[TSDB_COL_NAME_LEN];
+  int64_t  ctimeMs; // fill by vnode
 } SDeleteRes;
 
 int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);
@@ -3454,10 +3457,12 @@ int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq);
 typedef struct {
   int64_t suid;
   SArray* deleteReqs;  // SArray<SSingleDeleteReq>
+  int64_t ctimeMs;     // fill by vnode
 } SBatchDeleteReq;
 
 int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq);
 int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq);
+int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder* pDecoder, SBatchDeleteReq* pReq, int64_t ctimeMs);
 
 typedef struct {
   int32_t msgIdx;
@@ -3525,6 +3530,7 @@ typedef struct {
     SArray* aRowP;
     SArray* aCol;
   };
+  int64_t        ctimeMs;
 } SSubmitTbData;
 
 typedef struct {

include/libs/executor/storageapi.h

@@ -54,7 +54,7 @@ typedef struct SMetaEntry {
       SRSmaParam     rsmaParam;
     } stbEntry;
     struct {
-      int64_t  ctime;
+      int64_t  btime;
       int32_t  ttlDays;
       int32_t  commentLen;
       char*    comment;
@@ -62,7 +62,7 @@ typedef struct SMetaEntry {
       uint8_t* pTags;
     } ctbEntry;
     struct {
-      int64_t        ctime;
+      int64_t        btime;
       int32_t        ttlDays;
       int32_t        commentLen;
       char*          comment;

source/common/src/tglobal.c

@@ -110,6 +110,7 @@ int32_t tsQueryRspPolicy = 0;
 int64_t tsQueryMaxConcurrentTables = 200;  // unit is TSDB_TABLE_NUM_UNIT
 bool    tsEnableQueryHb = false;
 bool    tsEnableScience = false;  // on taos-cli show float and doulbe with scientific notation if true
+bool    tsTtlChangeOnWrite = false; // ttl delete time changes on last write if true
 int32_t tsQuerySmaOptimize = 0;
 int32_t tsQueryRsmaTolerance = 1000;  // the tolerance time (ms) to judge from which level to query rsma data.
 bool    tsQueryPlannerTrace = false;
@@ -513,6 +514,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1;
+  if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1;
 
@@ -873,6 +875,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
   tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
+  tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
   tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;
   tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
   tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
@@ -978,6 +981,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
         taosSetCoreDump(enableCore);
       } else if (strcasecmp("enableQueryHb", name) == 0) {
         tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
+      }  else if (strcasecmp("ttlChangeOnWrite", name) == 0) {
+        tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
       }
       break;
     }

source/common/src/tmsg.c

@@ -30,6 +30,9 @@
 
 #include "tlog.h"
 
+static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
+static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
+
 int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL) {
     terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
@@ -1725,7 +1728,7 @@ int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq
   } else {
     pReq->unsafe = false;
   }
-  
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -3161,7 +3164,7 @@ int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq
   tEncoderInit(&encoder, buf, bufLen);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->timestampSec) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -3174,7 +3177,7 @@ int32_t tDeserializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableR
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->timestampSec) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -4671,7 +4674,7 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode
       if (tDecodeSReplica(&decoder, pReplica) < 0) return -1;
     }
   }
-  
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -6409,7 +6412,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
   if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1;
   if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
   if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
-  if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
+  if (tEncodeI64(pCoder, pReq->btime) < 0) return -1;
   if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
   if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
   if (tEncodeI32(pCoder, pReq->commentLen) < 0) return -1;
@@ -6444,7 +6447,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
   if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1;
   if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
   if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
-  if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
+  if (tDecodeI64(pCoder, &pReq->btime) < 0) return -1;
   if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
   if (tDecodeI32(pCoder, &pReq->commentLen) < 0) return -1;
@@ -6909,14 +6912,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
     default:
       break;
   }
+  if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
 
   tEndEncode(pEncoder);
   return 0;
 }
 
-int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-
+static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq) {
   if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1;
   if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->colId) < 0) return -1;
@@ -6960,6 +6962,28 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
     default:
       break;
   }
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1;
+
+  pReq->ctimeMs = 0;
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  }
+
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1;
+
+  *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs;
+  if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
 
   tEndDecode(pDecoder);
   return 0;
@@ -7238,6 +7262,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
 
   if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1;
   if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1;
+  if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1;
   return 0;
 }
 
@@ -7257,6 +7282,11 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
 
   if (tDecodeCStrTo(pCoder, pRes->tableFName) < 0) return -1;
   if (tDecodeCStrTo(pCoder, pRes->tsColName) < 0) return -1;
+
+  pRes->ctimeMs = 0;
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1;
+  }
   return 0;
 }
 
@@ -7480,10 +7510,11 @@ int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq)
     SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i);
     if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1;
   }
+  if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
   return 0;
 }
 
-int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
+static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
   if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1;
   int32_t sz;
   if (tDecodeI32(pDecoder, &sz) < 0) return -1;
@@ -7497,6 +7528,24 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
   return 0;
 }
 
+int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
+  if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1;
+
+  pReq->ctimeMs = 0;
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  }
+  return 0;
+}
+
+int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq, int64_t ctimeMs) {
+  if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1;
+
+  *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs;
+  if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  return 0;
+}
+
 static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) {
   if (tStartEncode(pCoder) < 0) return -1;
 
@@ -7531,6 +7580,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
       pCoder->pos += rows[iRow]->len;
     }
   }
+  if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -7611,6 +7661,14 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
     }
   }
 
+  pSubmitTbData->ctimeMs = 0;
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI64(pCoder, &pSubmitTbData->ctimeMs) < 0) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+  }
+
   tEndDecode(pCoder);
 
 _exit:

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -46,6 +46,7 @@ typedef struct {
   int32_t vgId;
   int32_t vgVersion;
   int8_t  dropped;
+  int32_t toVgId;
   char    path[PATH_MAX + 20];
 } SWrapperCfg;
 
@@ -55,6 +56,7 @@ typedef struct {
   int32_t       refCount;
   int8_t        dropped;
   int8_t        disable;
+  int32_t       toVgId;
   char         *path;
   SVnode       *pImpl;
   SMultiWorker  pWriteW;
@@ -70,6 +72,7 @@ typedef struct {
   int32_t      vnodeNum;
   int32_t      opened;
   int32_t      failed;
+  bool         updateVnodesList;
   int32_t      threadIndex;
   TdThread     thread;
   SVnodeMgmt  *pMgmt;

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -71,6 +71,8 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
     if (code < 0) goto _OVER;
     tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
     if (code < 0) goto _OVER;
+    tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
+    if (code < 0) goto _OVER;
 
     snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
   }
@@ -165,6 +167,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num
     if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1;
     if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1;
     if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1;
+    if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1;
     if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1;
   }
 
@@ -179,7 +182,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
   SVnodeObj **ppVnodes = NULL;
   char        file[PATH_MAX] = {0};
   char        realfile[PATH_MAX] = {0};
-  snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
+  snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
   snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
 
   int32_t numOfVnodes = 0;
@@ -226,4 +229,4 @@ _OVER:
     dError("failed to write vnodes file:%s since %s, vnodes:%d", realfile, terrstr(), numOfVnodes);
   }
   return code;
-}
+}
\ No newline at end of file

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -484,10 +484,18 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   int32_t srcVgId = req.srcVgId;
   int32_t dstVgId = req.dstVgId;
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, dstVgId);
+  if (pVnode != NULL) {
+    dError("vgId:%d, vnode already exist", dstVgId);
+    vmReleaseVnode(pMgmt, pVnode);
+    terrno = TSDB_CODE_VND_ALREADY_EXIST;
+    return -1;
+  }
+
   dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd,
         req.dstVgId);
-
-  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, srcVgId);
+  pVnode = vmAcquireVnode(pMgmt, srcVgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
@@ -501,6 +509,13 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   };
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
 
+  // prepare alter
+  pVnode->toVgId = dstVgId;
+  if (vmWriteVnodeListToFile(pMgmt) != 0) {
+    dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr());
+    return -1;
+  }
+
   dInfo("vgId:%d, close vnode", srcVgId);
   vmCloseVnode(pMgmt, pVnode, true);
 
@@ -532,6 +547,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
+  // complete alter
   if (vmWriteVnodeListToFile(pMgmt) != 0) {
     dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr());
     return -1;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -158,6 +158,28 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   taosMemoryFree(pVnode);
 }
 
+static int32_t vmRestoreVgroupId(SWrapperCfg *pCfg, STfs *pTfs) {
+  int32_t srcVgId = pCfg->vgId;
+  int32_t dstVgId = pCfg->toVgId;
+  if (dstVgId == 0) return 0;
+
+  char srcPath[TSDB_FILENAME_LEN];
+  char dstPath[TSDB_FILENAME_LEN];
+
+  snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId);
+  snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId);
+
+  int32_t vgId = vnodeRestoreVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs);
+  if (vgId <= 0) {
+    dError("vgId:%d, failed to restore vgroup id. srcPath: %s", pCfg->vgId, srcPath);
+    return -1;
+  }
+
+  pCfg->vgId = vgId;
+  pCfg->toVgId = 0;
+  return 0;
+}
+
 static void *vmOpenVnodeInThread(void *param) {
   SVnodeThread *pThread = param;
   SVnodeMgmt   *pMgmt = pThread->pMgmt;
@@ -174,17 +196,33 @@ static void *vmOpenVnodeInThread(void *param) {
              pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
     tmsgReportStartup("vnode-open", stepDesc);
 
+    if (pCfg->toVgId) {
+      if (vmRestoreVgroupId(pCfg, pMgmt->pTfs) != 0) {
+        dError("vgId:%d, failed to restore vgroup id by thread:%d", pCfg->vgId, pThread->threadIndex);
+        pThread->failed++;
+        continue;
+      }
+      pThread->updateVnodesList = true;
+    }
+
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
+
     SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
     if (pImpl == NULL) {
       dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
       pThread->failed++;
-    } else {
-      vmOpenVnode(pMgmt, pCfg, pImpl);
-      dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
-      pThread->opened++;
-      atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
+      continue;
     }
+
+    if (vmOpenVnode(pMgmt, pCfg, pImpl) != 0) {
+      dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
+      pThread->failed++;
+      continue;
+    }
+
+    dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
+    pThread->opened++;
+    atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
   }
 
   dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
@@ -242,6 +280,8 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
     taosThreadAttrDestroy(&thAttr);
   }
 
+  bool updateVnodesList = false;
+
   for (int32_t t = 0; t < threadNum; ++t) {
     SVnodeThread *pThread = &threads[t];
     if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
@@ -249,6 +289,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
       taosThreadClear(&pThread->thread);
     }
     taosMemoryFree(pThread->pCfgs);
+    if (pThread->updateVnodesList) updateVnodesList = true;
   }
   taosMemoryFree(threads);
   taosMemoryFree(pCfgs);
@@ -256,10 +297,15 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
   if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) {
     dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes);
     return -1;
-  } else {
-    dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes);
-    return 0;
   }
+
+  if (updateVnodesList && vmWriteVnodeListToFile(pMgmt) != 0) {
+    dError("failed to write vnode list since %s", terrstr());
+    return -1;
+  }
+
+  dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes);
+  return 0;
 }
 
 static void *vmCloseVnodeInThread(void *param) {

source/dnode/mnode/impl/src/mndStb.c

@@ -888,7 +888,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
   SSdb             *pSdb = pMnode->pSdb;
   SVgObj           *pVgroup = NULL;
   void             *pIter = NULL;
-  SVDropTtlTableReq ttlReq = {.timestamp = taosGetTimestampSec()};
+  SVDropTtlTableReq ttlReq = {.timestampSec = taosGetTimestampSec()};
   int32_t           reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq);
   int32_t           contLen = reqLen + sizeof(SMsgHead);
 
@@ -914,7 +914,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
     if (code != 0) {
       mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code);
     } else {
-      mInfo("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp);
+      mInfo("vgId:%d, send drop ttl table request to vnode, time:%" PRId32, pVgroup->vgId, ttlReq.timestampSec);
     }
     sdbRelease(pSdb, pVgroup);
   }
@@ -1188,7 +1188,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
- 
+
   if(pNew->nextColId < 0 || pNew->nextColId >= 0x7fff - ntags){
     terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW;
     return -1;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1217,6 +1217,7 @@ static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, i
   action.pCont = pReq;
   action.contLen = contLen;
   action.msgType = TDMT_VND_ALTER_HASHRANGE;
+  action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;
 
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);

source/dnode/vnode/CMakeLists.txt

@@ -14,16 +14,17 @@ set(
   "src/vnd/vnodeRetention.c"
   "src/vnd/vnodeInitApi.c"
 
-  # meta
-  "src/meta/metaOpen.c"
-  "src/meta/metaIdx.c"
-  "src/meta/metaTable.c"
-  "src/meta/metaSma.c"
-  "src/meta/metaQuery.c"
-  "src/meta/metaCommit.c"
-  "src/meta/metaEntry.c"
-  "src/meta/metaSnapshot.c"
-  "src/meta/metaCache.c"
+    # meta
+    "src/meta/metaOpen.c"
+    "src/meta/metaIdx.c"
+    "src/meta/metaTable.c"
+    "src/meta/metaSma.c"
+    "src/meta/metaQuery.c"
+    "src/meta/metaCommit.c"
+    "src/meta/metaEntry.c"
+    "src/meta/metaSnapshot.c"
+    "src/meta/metaCache.c"
+    "src/meta/metaTtl.c"
 
   # sma
   "src/sma/smaEnv.c"
@@ -90,6 +91,7 @@ IF (TD_VNODE_PLUGINS)
   )
 ENDIF ()
 
+IF (NOT ${TD_LINUX})
 target_include_directories(
     vnode
     PUBLIC "inc"
@@ -97,7 +99,25 @@ target_include_directories(
     PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
     PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include"
 )
+ELSE()
+target_include_directories(
+    vnode
+    PUBLIC "inc"
+    PUBLIC "src/inc"
+    PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
+)
+ENDIF (NOT ${TD_LINUX})
+
 IF (TD_LINUX)
+target_include_directories(
+      vnode
+      PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+    )
+
+    target_link_directories(
+        vnode
+        PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+    )
 target_link_libraries(
     vnode
     PUBLIC os

source/dnode/vnode/inc/vnode.h

@@ -54,6 +54,7 @@ void    vnodeCleanup();
 int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
 int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs);
 int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs);
+int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs);
 void    vnodeDestroy(const char *path, STfs *pTfs);
 SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
 void    vnodePreClose(SVnode *pVnode);
@@ -114,6 +115,7 @@ int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
 int      metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
 int      metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid);
 int      metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+int      metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays);
 bool     metaIsTableExist(void* pVnode, tb_uid_t uid);
 int32_t  metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
                                    bool *acquired);

source/dnode/vnode/src/inc/meta.h

@@ -17,6 +17,7 @@
 #define _TD_VNODE_META_H_
 
 #include "index.h"
+#include "metaTtl.h"
 #include "vnodeInt.h"
 
 #ifdef __cplusplus
@@ -89,10 +90,10 @@ struct SMeta {
   // ivt idx and idx
   void* pTagIvtIdx;
 
-  TTB* pTagIdx;
-  TTB* pTtlIdx;
+  TTB*        pTagIdx;
+  STtlManger* pTtlMgr;
 
-  TTB* pCtimeIdx;  // table created time idx
+  TTB* pBtimeIdx;  // table created time idx
   TTB* pNcolIdx;   // ncol of table idx, normal table only
 
   TTB* pSmaIdx;
@@ -138,20 +139,15 @@ typedef struct {
 } STagIdxKey;
 #pragma pack(pop)
 
-typedef struct {
-  int64_t  dtime;
-  tb_uid_t uid;
-} STtlIdxKey;
-
 typedef struct {
   tb_uid_t uid;
   int64_t  smaUid;
 } SSmaIdxKey;
 
 typedef struct {
-  int64_t  ctime;
+  int64_t  btime;
   tb_uid_t uid;
-} SCtimeIdxKey;
+} SBtimeIdxKey;
 
 typedef struct {
   int64_t  ncol;

source/dnode/vnode/src/inc/metaTtl.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_TTL_H_
+#define _TD_VNODE_TTL_H_
+
+#include "taosdef.h"
+#include "thash.h"
+
+#include "tdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum DirtyEntryType {
+  ENTRY_TYPE_DEL = 1,
+  ENTRY_TYPE_UPSERT = 2,
+} DirtyEntryType;
+
+typedef struct STtlManger {
+  TdThreadRwlock lock;
+
+  TTB* pOldTtlIdx;       // btree<{deleteTime, tuid}, NULL>
+
+  SHashObj* pTtlCache;   // key: tuid, value: {ttl, ctime}
+  SHashObj* pDirtyUids;  // dirty tuid
+  TTB*      pTtlIdx;     // btree<{deleteTime, tuid}, ttl>
+} STtlManger;
+
+typedef struct {
+  int64_t ttlDays;
+  int64_t changeTimeMs;
+} STtlCacheEntry;
+
+typedef struct {
+  DirtyEntryType type;
+} STtlDirtyEntry;
+
+typedef struct {
+  int64_t  deleteTimeSec;
+  tb_uid_t uid;
+} STtlIdxKey;
+
+typedef struct {
+  int64_t  deleteTimeMs;
+  tb_uid_t uid;
+} STtlIdxKeyV1;
+
+typedef struct {
+  int64_t ttlDays;
+} STtlIdxValue;
+
+typedef struct {
+  tb_uid_t uid;
+  int64_t  changeTimeMs;
+} STtlUpdCtimeCtx;
+
+typedef struct {
+  tb_uid_t uid;
+  int64_t  changeTimeMs;
+  int64_t  ttlDays;
+} STtlUpdTtlCtx;
+
+typedef struct {
+  tb_uid_t uid;
+  TXN*     pTxn;
+} STtlDelTtlCtx;
+
+int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback);
+int ttlMgrClose(STtlManger* pTtlMgr);
+int ttlMgrBegin(STtlManger* pTtlMgr, void* pMeta);
+
+int ttlMgrConvert(TTB* pOldTtlIdx, TTB* pNewTtlIdx, void* pMeta);
+int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn);
+
+int ttlMgrInsertTtl(STtlManger* pTtlMgr, const STtlUpdTtlCtx* pUpdCtx);
+int ttlMgrDeleteTtl(STtlManger* pTtlMgr, const STtlDelTtlCtx* pDelCtx);
+int ttlMgrUpdateChangeTime(STtlManger* pTtlMgr, const STtlUpdCtimeCtx* pUpdCtimeCtx);
+int ttlMgrFindExpired(STtlManger* pTtlMgr, int64_t timePointMs, SArray* pTbUids);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_TTL_H_*/

source/dnode/vnode/src/inc/vnodeInt.h

@@ -149,8 +149,9 @@ int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp);
 int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid);
 int32_t         metaTrimTables(SMeta* pMeta);
-int             metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
+int             metaTtlDropTable(SMeta* pMeta, int64_t timePointMs, SArray* tbUids);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
+int             metaUpdateChangeTime(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
 int32_t         metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema);
@@ -176,7 +177,6 @@ SArray*       metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*       metaGetSmaTbUids(SMeta* pMeta);
 void*         metaGetIdx(SMeta* pMeta);
 void*         metaGetIvtIdx(SMeta* pMeta);
-int           metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
 
 void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags);
 

source/dnode/vnode/src/meta/metaCommit.c

@@ -40,6 +40,12 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
     return -1;
   }
 
+  if (ttlMgrBegin(pMeta->pTtlMgr, pMeta) < 0) {
+    return -1;
+  }
+
+  tdbCommit(pMeta->pEnv, pMeta->txn);
+
   return 0;
 }
 
@@ -50,6 +56,7 @@ int  metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv
 int  metaPrepareAsyncCommit(SMeta *pMeta) {
    // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
   int code = 0;
+  code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
   code = tdbCommit(pMeta->pEnv, pMeta->txn);
 
   return code;

source/dnode/vnode/src/meta/metaEntry.c

@@ -31,7 +31,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
       if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
     }
   } else if (pME->type == TSDB_CHILD_TABLE) {
-    if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
+    if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
     if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0) {
@@ -40,7 +40,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
     if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
     if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
+    if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
     if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0) {
@@ -76,7 +76,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
       if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
     }
   } else if (pME->type == TSDB_CHILD_TABLE) {
-    if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
+    if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
     if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0) {
@@ -85,7 +85,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
     if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
     if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1;  // (TODO)
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
+    if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
     if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0) {

source/dnode/vnode/src/meta/metaOpen.c

@@ -19,12 +19,11 @@ static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen
 static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
-static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int taskIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
-static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
 static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
@@ -128,8 +127,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
     goto _err;
   }
 
-  // open pTtlIdx
-  ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx, 0);
+  // open pTtlMgr ("ttlv1.idx")
+  ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
@@ -143,7 +142,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
   }
 
   // idx table create time
-  ret = tdbTbOpen("ctime.idx", sizeof(SCtimeIdxKey), 0, ctimeIdxCmpr, pMeta->pEnv, &pMeta->pCtimeIdx, 0);
+  ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
@@ -184,9 +183,9 @@ _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
   if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
   if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-  if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx);
+  if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
   if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-  if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+  if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
   if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
   if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
   if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
@@ -209,9 +208,9 @@ int metaClose(SMeta **ppMeta) {
     if (pMeta->pIdx) metaCloseIdx(pMeta);
     if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
     if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-    if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx);
+    if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
     if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-    if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+    if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
     if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
     if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
     if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
@@ -399,37 +398,18 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
   return 0;
 }
 
-static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
-  STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
-  STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
-
-  if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) {
-    return 1;
-  } else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) {
-    return -1;
-  }
-
-  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
-    return 1;
-  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
-  SCtimeIdxKey *pCtimeIdxKey1 = (SCtimeIdxKey *)pKey1;
-  SCtimeIdxKey *pCtimeIdxKey2 = (SCtimeIdxKey *)pKey2;
-  if (pCtimeIdxKey1->ctime > pCtimeIdxKey2->ctime) {
+static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  SBtimeIdxKey *pBtimeIdxKey1 = (SBtimeIdxKey *)pKey1;
+  SBtimeIdxKey *pBtimeIdxKey2 = (SBtimeIdxKey *)pKey2;
+  if (pBtimeIdxKey1->btime > pBtimeIdxKey2->btime) {
     return 1;
-  } else if (pCtimeIdxKey1->ctime < pCtimeIdxKey2->ctime) {
+  } else if (pBtimeIdxKey1->btime < pBtimeIdxKey2->btime) {
     return -1;
   }
 
-  if (pCtimeIdxKey1->uid > pCtimeIdxKey2->uid) {
+  if (pBtimeIdxKey1->uid > pBtimeIdxKey2->uid) {
     return 1;
-  } else if (pCtimeIdxKey1->uid < pCtimeIdxKey2->uid) {
+  } else if (pBtimeIdxKey1->uid < pBtimeIdxKey2->uid) {
     return -1;
   }
 

source/dnode/vnode/src/meta/metaQuery.c

@@ -212,6 +212,29 @@ int metaReadNext(SMetaReader *pReader) {
   return 0;
 }
 
+int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays) {
+  int         code = -1;
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, (SMeta *)meta, 0);
+  code = metaReaderGetTableEntryByUid(&mr, uid);
+  if (code < 0) {
+    goto _exit;
+  }
+  if (mr.me.type == TSDB_CHILD_TABLE) {
+    *ttlDays = mr.me.ctbEntry.ttlDays;
+  } else if (mr.me.type == TSDB_NORMAL_TABLE) {
+    *ttlDays = mr.me.ntbEntry.ttlDays;
+  } else {
+    goto _exit;
+  }
+
+  code = 0;
+
+_exit:
+  metaReaderClear(&mr);
+  return code;
+}
+
 #if 1  // ===================================================
 SMTbCursor *metaOpenTbCursor(void *pVnode) {
   SMTbCursor *pTbCur = NULL;
@@ -387,37 +410,6 @@ _err:
   return NULL;
 }
 
-int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) {
-  TBC *pCur;
-  int  ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL);
-  if (ret < 0) {
-    return ret;
-  }
-
-  STtlIdxKey ttlKey = {0};
-  ttlKey.dtime = ttl;
-  ttlKey.uid = INT64_MAX;
-  int c = 0;
-  tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c);
-  if (c < 0) {
-    tdbTbcMoveToPrev(pCur);
-  }
-
-  void *pKey = NULL;
-  int   kLen = 0;
-  while (1) {
-    ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL);
-    if (ret < 0) {
-      break;
-    }
-    ttlKey = *(STtlIdxKey *)pKey;
-    taosArrayPush(uidList, &ttlKey.uid);
-  }
-  tdbFree(pKey);
-  tdbTbcClose(pCur);
-  return 0;
-}
-
 struct SMCtbCursor {
   SMeta   *pMeta;
   TBC     *pCur;
@@ -1018,17 +1010,17 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
   pCursor->type = param->type;
 
   metaRLock(pMeta);
-  ret = tdbTbcOpen(pMeta->pCtimeIdx, &pCursor->pCur, NULL);
+  ret = tdbTbcOpen(pMeta->pBtimeIdx, &pCursor->pCur, NULL);
   if (ret != 0) {
     goto END;
   }
   int64_t uidLimit = param->reverse ? INT64_MAX : 0;
 
-  SCtimeIdxKey  ctimeKey = {.ctime = *(int64_t *)(param->val), .uid = uidLimit};
-  SCtimeIdxKey *pCtimeKey = &ctimeKey;
+  SBtimeIdxKey  btimeKey = {.btime = *(int64_t *)(param->val), .uid = uidLimit};
+  SBtimeIdxKey *pBtimeKey = &btimeKey;
 
   int cmp = 0;
-  if (tdbTbcMoveTo(pCursor->pCur, &ctimeKey, sizeof(ctimeKey), &cmp) < 0) {
+  if (tdbTbcMoveTo(pCursor->pCur, &btimeKey, sizeof(btimeKey), &cmp) < 0) {
     goto END;
   }
 
@@ -1042,10 +1034,10 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
     valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, NULL, NULL);
     if (valid < 0) break;
 
-    SCtimeIdxKey *p = entryKey;
+    SBtimeIdxKey *p = entryKey;
     if (count > TRY_ERROR_LIMIT) break;
 
-    int32_t cmp = (*param->filterFunc)((void *)&p->ctime, (void *)&pCtimeKey->ctime, param->type);
+    int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type);
     if (cmp == 0)
       taosArrayPush(pUids, &p->uid);
     else {
@@ -1149,7 +1141,7 @@ int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
   pCursor->type = param->type;
 
   metaRLock(pMeta);
-  ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
+  //ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
 
 END:
   if (pCursor->pMeta) metaULock(pCursor->pMeta);

source/dnode/vnode/src/meta/metaTable.c

@@ -20,7 +20,7 @@ static int  metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, con
 static int  metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
-static int  metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int  metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -28,8 +28,8 @@ static int  metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
 static int  metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
 static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
 // opt ins_tables query
-static int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
-static int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 
@@ -734,7 +734,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
   me.uid = pReq->uid;
   me.name = pReq->name;
   if (me.type == TSDB_CHILD_TABLE) {
-    me.ctbEntry.ctime = pReq->ctime;
+    me.ctbEntry.btime = pReq->btime;
     me.ctbEntry.ttlDays = pReq->ttl;
     me.ctbEntry.commentLen = pReq->commentLen;
     me.ctbEntry.comment = pReq->comment;
@@ -770,7 +770,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
     metaTbGroupCacheClear(pMeta, me.ctbEntry.suid);
     metaULock(pMeta);
   } else {
-    me.ntbEntry.ctime = pReq->ctime;
+    me.ntbEntry.btime = pReq->btime;
     me.ntbEntry.ttlDays = pReq->ttl;
     me.ntbEntry.commentLen = pReq->commentLen;
     me.ntbEntry.comment = pReq->comment;
@@ -923,50 +923,40 @@ end:
   return code;
 }
 
-int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
-  int ret = metaTtlSmaller(pMeta, ttl, tbUids);
+int metaTtlDropTable(SMeta *pMeta, int64_t timePointMs, SArray *tbUids) {
+  int ret = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
   if (ret != 0) {
+    metaError("ttl failed to flush, ret:%d", ret);
+    return ret;
+  }
+
+  ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids);
+  if (ret != 0) {
+    metaError("ttl failed to find expired table, ret:%d", ret);
     return ret;
   }
   if (TARRAY_SIZE(tbUids) == 0) {
     return 0;
   }
 
+  metaInfo("ttl find expired table count: %zu" , TARRAY_SIZE(tbUids));
+
   metaDropTables(pMeta, tbUids);
   return 0;
 }
 
-static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) {
-  int64_t ttlDays = 0;
-  int64_t ctime = 0;
+static int metaBuildBtimeIdxKey(SBtimeIdxKey *btimeKey, const SMetaEntry *pME) {
+  int64_t btime;
   if (pME->type == TSDB_CHILD_TABLE) {
-    ctime = pME->ctbEntry.ctime;
-    ttlDays = pME->ctbEntry.ttlDays;
+    btime = pME->ctbEntry.btime;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    ctime = pME->ntbEntry.ctime;
-    ttlDays = pME->ntbEntry.ttlDays;
-  } else {
-    metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type);
-    return;
-  }
-
-  if (ttlDays <= 0) return;
-
-  ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit;
-  ttlKey->uid = pME->uid;
-}
-static int metaBuildCtimeIdxKey(SCtimeIdxKey *ctimeKey, const SMetaEntry *pME) {
-  int64_t ctime;
-  if (pME->type == TSDB_CHILD_TABLE) {
-    ctime = pME->ctbEntry.ctime;
-  } else if (pME->type == TSDB_NORMAL_TABLE) {
-    ctime = pME->ntbEntry.ctime;
+    btime = pME->ntbEntry.btime;
   } else {
     return -1;
   }
 
-  ctimeKey->ctime = ctime;
-  ctimeKey->uid = pME->uid;
+  btimeKey->btime = btime;
+  btimeKey->uid = pME->uid;
   return 0;
 }
 
@@ -980,11 +970,9 @@ static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) {
   return 0;
 }
 
-static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  STtlIdxKey ttlKey = {0};
-  metaBuildTtlIdxKey(&ttlKey, pME);
-  if (ttlKey.dtime == 0) return 0;
-  return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn);
+static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
+  STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn};
+  return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
 }
 
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
@@ -1066,10 +1054,10 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn);
   tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn);
 
-  if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e);
+  if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteBtimeIdx(pMeta, &e);
   if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e);
 
-  if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
+  if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e);
 
   if (e.type == TSDB_CHILD_TABLE) {
     tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn);
@@ -1102,23 +1090,23 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   return 0;
 }
 // opt ins_tables
-int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  SCtimeIdxKey ctimeKey = {0};
-  if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
+int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  SBtimeIdxKey btimeKey = {0};
+  if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) {
     return 0;
   }
-  metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode),
-            pME->version, pME->uid, ctimeKey.ctime);
+  metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode),
+            pME->version, pME->uid, btimeKey.btime);
 
-  return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn);
 }
 
-int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  SCtimeIdxKey ctimeKey = {0};
-  if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
+int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
+  SBtimeIdxKey btimeKey = {0};
+  if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) {
     return 0;
   }
-  return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn);
+  return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn);
 }
 int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SNcolIdxKey ncolKey = {0};
@@ -1328,6 +1316,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs);
+
   metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp);
 
   if (entry.pBuf) taosMemoryFree(entry.pBuf);
@@ -1515,6 +1505,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
 
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs);
+
   tDecoderClear(&dc1);
   tDecoderClear(&dc2);
   taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
@@ -1603,9 +1595,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   // build SMetaEntry
   if (entry.type == TSDB_CHILD_TABLE) {
     if (pAlterTbReq->updateTTL) {
-      metaDeleteTtlIdx(pMeta, &entry);
+      metaDeleteTtl(pMeta, &entry);
       entry.ctbEntry.ttlDays = pAlterTbReq->newTTL;
-      metaUpdateTtlIdx(pMeta, &entry);
+      metaUpdateTtl(pMeta, &entry);
     }
     if (pAlterTbReq->newCommentLen >= 0) {
       entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen;
@@ -1613,9 +1605,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
     }
   } else {
     if (pAlterTbReq->updateTTL) {
-      metaDeleteTtlIdx(pMeta, &entry);
+      metaDeleteTtl(pMeta, &entry);
       entry.ntbEntry.ttlDays = pAlterTbReq->newTTL;
-      metaUpdateTtlIdx(pMeta, &entry);
+      metaUpdateTtl(pMeta, &entry);
     }
     if (pAlterTbReq->newCommentLen >= 0) {
       entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen;
@@ -1628,6 +1620,8 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   metaUpdateUidIdx(pMeta, &entry);
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs);
+
   tdbTbcClose(pTbDbc);
   tdbTbcClose(pUidIdxc);
   tDecoderClear(&dc);
@@ -1967,11 +1961,28 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
   return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
-static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  STtlIdxKey ttlKey = {0};
-  metaBuildTtlIdxKey(&ttlKey, pME);
-  if (ttlKey.dtime == 0) return 0;
-  return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
+static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) {
+  if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0;
+
+  STtlUpdTtlCtx ctx = {.uid = pME->uid};
+
+  if (pME->type == TSDB_CHILD_TABLE) {
+    ctx.ttlDays = pME->ctbEntry.ttlDays;
+    ctx.changeTimeMs = pME->ctbEntry.btime;
+  } else {
+    ctx.ttlDays = pME->ntbEntry.ttlDays;
+    ctx.changeTimeMs = pME->ntbEntry.btime;
+  }
+
+  return ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx);
+}
+
+int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) {
+  if (!tsTtlChangeOnWrite) return 0;
+
+  STtlUpdCtimeCtx ctx = {.uid = uid, .changeTimeMs = changeTimeMs};
+
+  return ttlMgrUpdateChangeTime(pMeta->pTtlMgr, &ctx);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -2182,7 +2193,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
     }
   }
 
-  code = metaUpdateCtimeIdx(pMeta, pME);
+  code = metaUpdateBtimeIdx(pMeta, pME);
   VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_NORMAL_TABLE) {
@@ -2191,7 +2202,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
   }
 
   if (pME->type != TSDB_SUPER_TABLE) {
-    code = metaUpdateTtlIdx(pMeta, pME);
+    code = metaUpdateTtl(pMeta, pME);
     VND_CHECK_CODE(code, line, _err);
   }
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -136,14 +136,13 @@ static int32_t vnodeVgroupIdLen(int32_t vgId) {
 }
 
 int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) {
-  int32_t ret = tfsRename(pTfs, srcPath, dstPath);
-  if (ret != 0) return ret;
+  int32_t ret = 0;
 
   char oldRname[TSDB_FILENAME_LEN] = {0};
   char newRname[TSDB_FILENAME_LEN] = {0};
   char tsdbPath[TSDB_FILENAME_LEN] = {0};
   char tsdbFilePrefix[TSDB_FILENAME_LEN] = {0};
-  snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", dstPath, TD_DIRSEP);
+  snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", srcPath, TD_DIRSEP);
   snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP);
 
   STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath);
@@ -168,7 +167,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr
 
       ret = tfsRename(pTfs, tsdbFile->rname, newRname);
       if (ret != 0) {
-        vInfo("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr());
+        vError("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr());
         tfsClosedir(tsdbDir);
         return ret;
       }
@@ -176,6 +175,21 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr
   }
 
   tfsClosedir(tsdbDir);
+
+  vInfo("vgId:%d, rename dir from %s to %s", dstVgId, srcPath, dstPath);
+  ret = tfsRename(pTfs, srcPath, dstPath);
+  if (ret != 0) {
+    vError("vgId:%d, failed to rename dir from %s to %s since %s", dstVgId, srcPath, dstPath, terrstr());
+  }
+  return ret;
+}
+
+int32_t vnodeGetAbsDir(const char *relPath, STfs *pTfs, char *buf, size_t bufLen) {
+  if (pTfs) {
+    snprintf(buf, bufLen, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, relPath);
+  } else {
+    snprintf(buf, bufLen, "%s", relPath);
+  }
   return 0;
 }
 
@@ -184,13 +198,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
   char       dir[TSDB_FILENAME_LEN] = {0};
   int32_t    ret = 0;
 
-  if (pTfs) {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, srcPath);
-  } else {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s", srcPath);
-  }
-
-  // todo add stat file to handle exception while vnode open
+  vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN);
 
   ret = vnodeLoadInfo(dir, &info);
   if (ret < 0) {
@@ -245,6 +253,42 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
   return 0;
 }
 
+int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) {
+  SVnodeInfo info = {0};
+  char       dir[TSDB_FILENAME_LEN] = {0};
+
+  vnodeGetAbsDir(dstPath, pTfs, dir, TSDB_FILENAME_LEN);
+  if (vnodeLoadInfo(dir, &info) == 0) {
+    if (info.config.vgId != dstVgId) {
+      vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId);
+      return -1;
+    }
+    return dstVgId;
+  }
+
+  vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN);
+  if (vnodeLoadInfo(dir, &info) < 0) {
+    vError("vgId:%d, failed to read vnode config from %s since %s", srcVgId, srcPath, tstrerror(terrno));
+    return -1;
+  }
+
+  if (info.config.vgId == srcVgId) {
+    vInfo("vgId:%d, rollback alter hashrange", srcVgId);
+    return srcVgId;
+  } else if (info.config.vgId != dstVgId) {
+    vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId);
+    return -1;
+  }
+
+  vInfo("vgId:%d, rename %s to %s", dstVgId, srcPath, dstPath);
+  if (vnodeRenameVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs) < 0) {
+    vError("vgId:%d, failed to rename vnode from %s to %s since %s", dstVgId, srcPath, dstPath, tstrerror(terrno));
+    return -1;
+  }
+
+  return dstVgId;
+}
+
 void vnodeDestroy(const char *path, STfs *pTfs) {
   vInfo("path:%s is removed while destroy vnode", path);
   tfsRmdir(pTfs, path);

source/libs/executor/src/sysscanoperator.c

@@ -1154,7 +1154,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     int32_t tableType = mr.me.type;
     if (tableType == TSDB_CHILD_TABLE) {
       // create time
-      int64_t ts = mr.me.ctbEntry.ctime;
+      int64_t ts = mr.me.ctbEntry.btime;
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
@@ -1206,7 +1206,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     } else if (tableType == TSDB_NORMAL_TABLE) {
       // create time
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
-      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false);
+      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false);
 
       // number of columns
       pColInfoData = taosArrayGet(p->pDataBlock, 3);
@@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
     int32_t tableType = pInfo->pCur->mr.me.type;
     if (tableType == TSDB_CHILD_TABLE) {
       // create time
-      int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime;
+      int64_t ts = pInfo->pCur->mr.me.ctbEntry.btime;
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
@@ -1392,7 +1392,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
     } else if (tableType == TSDB_NORMAL_TABLE) {
       // create time
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
-      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false);
+      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false);
 
       // number of columns
       pColInfoData = taosArrayGet(p->pDataBlock, 3);

source/libs/parser/src/parTranslater.c

@@ -2361,7 +2361,7 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect)
   int32_t code = TSDB_CODE_SUCCESS;
   if (NULL == getGroupByList(pCxt) && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow) {
     return code;
-  }  
+  }
   if (NULL != pSelect->pHaving) {
     code = checkExprForGroupBy(pCxt, &pSelect->pHaving);
   }
@@ -2372,7 +2372,7 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect)
   if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) {
     code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList);
   }
-*/  
+*/
   return code;
 }
 
@@ -2680,7 +2680,7 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) {
   SNode** pNode = NULL;
   SRewriteTbNameContext pRewriteCxt = {0};
   pRewriteCxt.pTbName = pTable->table.tableName;
-  
+
   nodesRewriteExprPostOrder(&pSelect->pWhere, doTranslateTbName, &pRewriteCxt);
 
   return pRewriteCxt.errCode;
@@ -3177,7 +3177,7 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList);
-  }  
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = translateFillValues(pCxt, pSelect);
   }
@@ -4965,6 +4965,14 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
   if (TSDB_CODE_SUCCESS == code) {
     code = checkTableSchema(pCxt, pStmt);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    if (createStable && pStmt->pOptions->ttl != 0) {
+      code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION,
+                                     "Only supported for create non-super table in databases "
+                                     "configured with the 'TTL' option");
+    }
+  }
+
   return code;
 }
 

source/libs/parser/test/parInitialCTest.cpp

@@ -765,7 +765,7 @@ TEST_F(ParserInitialCTest, createStable) {
       "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, "
       "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, "
       "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) "
-      "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m "
+      "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m "
       "DELETE_MARK 1000s,200m");
   clearCreateStbReq();
 }
@@ -1005,7 +1005,7 @@ TEST_F(ParserInitialCTest, createTable) {
         ASSERT_EQ(pReq->flags, pExpect->flags);
         ASSERT_EQ(std::string(pReq->name), std::string(pExpect->name));
         ASSERT_EQ(pReq->uid, pExpect->uid);
-        ASSERT_EQ(pReq->ctime, pExpect->ctime);
+        ASSERT_EQ(pReq->btime, pExpect->btime);
         ASSERT_EQ(pReq->ttl, pExpect->ttl);
         ASSERT_EQ(pReq->commentLen, pExpect->commentLen);
         ASSERT_EQ(std::string(pReq->comment), std::string(pExpect->comment));
@@ -1038,7 +1038,7 @@ TEST_F(ParserInitialCTest, createTable) {
       "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), "
       "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, "
       "a14 NCHAR(30), a15 VARCHAR(50)) "
-      "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)");
+      "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)");
 
   run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW)");
 

source/libs/planner/src/planOptimizer.c

@@ -167,25 +167,25 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) {
     return false;
   }
-  if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) &&
-                                 QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) &&
-                                 QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) {
-    return false;
-  }
-  if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) &&
-       WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) ||
-      (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && pNode->pParent->pParent &&
-       QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent) &&
+  return true;
+}
+
+static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
+  if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&
+       WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode)->winType) ||
+      (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) && pNode->pParent &&
+       QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) &&
        WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) {
     return true;
   }
-  if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode->pParent)) {
-    return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
+  if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode)) {
+    return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode)->pGroupKeys);
   }
   return false;
 }
 
 static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) {
+  if (!scanPathOptShouldGetFuncs(pNode)) return NULL;
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_WINDOW:
       return ((SWindowLogicNode*)pNode)->pFuncs;

source/libs/qworker/src/qworker.c

@@ -427,6 +427,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
 
   pRes->suid = pDelRes->suid;
   pRes->uidList = pDelRes->uidList;
+  pRes->ctimeMs = taosGetTimestampMs();
   pRes->skey = pDelRes->skey;
   pRes->ekey = pDelRes->ekey;
   pRes->affectedRows = pDelRes->affectedRows;

source/libs/stream/CMakeLists.txt

@@ -9,6 +9,14 @@ target_include_directories(
 
 if(${BUILD_WITH_ROCKSDB})
   IF (TD_LINUX)
+  target_include_directories(
+    stream
+    PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+  )
+  target_link_directories(
+    stream 
+    PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+  )
   target_link_libraries(
     stream
     PUBLIC rocksdb tdb

tests/parallel_test/cases.task

@@ -152,6 +152,7 @@
 ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
 ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
 ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py
+,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py