Merge branch '3.0' into feat/sangshuduo/TD-14141-update-taostools-for3.0

cmake/cmake.platform

@@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "")
     SET(TD_ARM_32 TRUE)
     ADD_DEFINITIONS("-D_TD_ARM_")
     ADD_DEFINITIONS("-D_TD_ARM_32")
-  ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+  ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
     MESSAGE(STATUS "The current platform is aarch64")
     SET(PLATFORM_ARCH_STR "arm64")
     SET(TD_ARM_64 TRUE)

cmake/cmake.version

@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.0.2")
+  SET(TD_VER_NUMBER "3.0.1.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG abed566
+        GIT_TAG 71e7ccf
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

cmake/taosws_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosws-rs
 ExternalProject_Add(taosws-rs
         GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
-        GIT_TAG 0609b50
+        GIT_TAG e771403
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/zh/05-get-started/01-docker.md

@@ -62,7 +62,7 @@ taos>
 
 ## 体验查询
 
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。
+使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。
 
 查询超级表下记录总条数：
 

docs/zh/07-develop/09-udf.md

@@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。
 
 参数的具体含义是：
   - inputDataBlock: 输入的数据块
-  - resultColumn: 输出列。输出列 
+  - resultColumn: 输出列 
 
 ### 聚合接口函数
 

docs/zh/12-taos-sql/03-table.md

@@ -23,10 +23,7 @@ create_subtable_clause: {
 }
 
 create_definition:
-    col_name column_definition
-
-column_definition:
-    type_name [comment 'string_value']
+    col_name column_type
 
 table_options:
     table_option ...

docs/zh/28-releases/01-tdengine.md

@@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接
 
 import Release from "/components/ReleaseV3";
 
-## 3.0.0.1
+## 3.0.1.0
 
-<Release type="tdengine" version="3.0.0.1" />
-
-<!-- ## 3.0.0.0
-
-<Release type="tdengine" version="3.0.0.0" /> -->
+<Release type="tdengine" version="3.0.1.0" />
 

docs/zh/28-releases/02-tools.md

@@ -6,6 +6,6 @@ description: taosTools 的发布历史、Release Notes 和下载链接
 
 import Release from "/components/ReleaseV3";
 
-## 2.1.2
+## 2.1.3
 
-<Release type="tools" version="2.1.2" />
+<Release type="tools" version="2.1.3" />
\ No newline at end of file

include/common/tmsgdef.h

@@ -272,6 +272,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   
 #if defined(TD_MSG_NUMBER_)

include/libs/nodes/nodes.h

@@ -320,6 +320,9 @@ int32_t     nodesStringToNode(const char* pStr, SNode** pNode);
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
+int32_t nodesNodeToMsg(const SNode* pNode, char** pMsg, int32_t* pLen);
+int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode);
+
 int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len);
 char*   nodesGetNameFromColumnNode(SNode* pNode);
 int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);

include/libs/stream/tstream.h

@@ -554,6 +554,8 @@ typedef struct {
 int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
 int32_t streamStateDel(SStreamState* pState, const SWinKey* key);
+int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen);
+int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal);
 void    streamFreeVal(void* val);
 
 SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key);

include/libs/sync/syncTools.h

@@ -444,6 +444,70 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
 void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
 void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
 
+// ---------------------------------------------
+typedef struct SyncHeartbeat {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm  term;
+  SyncIndex commitIndex;
+  SyncTerm  privateTerm;
+} SyncHeartbeat;
+
+SyncHeartbeat* syncHeartbeatBuild(int32_t vgId);
+void           syncHeartbeatDestroy(SyncHeartbeat* pMsg);
+void           syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen);
+void           syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg);
+char*          syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len);
+SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len);
+void           syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg);
+void           syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg);
+SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncHeartbeat2Json(const SyncHeartbeat* pMsg);
+char*          syncHeartbeat2Str(const SyncHeartbeat* pMsg);
+
+// for debug ----------------------
+void syncHeartbeatPrint(const SyncHeartbeat* pMsg);
+void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg);
+void syncHeartbeatLog(const SyncHeartbeat* pMsg);
+void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncHeartbeatReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+
+  // private data
+  SyncTerm term;
+  SyncTerm privateTerm;
+  int64_t  startTime;
+} SyncHeartbeatReply;
+
+SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId);
+void                syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg);
+void                syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen);
+void                syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg);
+char*               syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len);
+SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len);
+void                syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg);
+void                syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg);
+SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*              syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg);
+char*               syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg);
+
+// for debug ----------------------
+void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg);
+void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg);
+
 // ---------------------------------------------
 typedef struct SyncApplyMsg {
   uint32_t   bytes;

include/util/tdef.h

@@ -359,8 +359,8 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SCHEMALESS_ON           1
 #define TSDB_DB_SCHEMALESS_OFF          0
 #define TSDB_DEFAULT_DB_SCHEMALESS      TSDB_DB_SCHEMALESS_OFF
-#define TSDB_MIN_SST_TRIGGER            1
-#define TSDB_MAX_SST_TRIGGER            128
+#define TSDB_MIN_STT_TRIGGER            1
+#define TSDB_MAX_STT_TRIGGER            16
 #define TSDB_DEFAULT_SST_TRIGGER        8
 #define TSDB_MIN_HASH_PREFIX            0
 #define TSDB_MAX_HASH_PREFIX            128

include/util/tutil.h

@@ -69,7 +69,18 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
   memcpy(target, buf, TSDB_PASSWORD_LEN);
 }
 
-#define taosGetTbHashVal(tbname, tblen, method, prefix, suffix) MurmurHash3_32((tbname), (tblen))
+static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
+                                      int32_t suffix) {
+  if (prefix == 0 && suffix == 0) {
+    return MurmurHash3_32(tbname, tblen);
+  } else {
+    if (tblen <= (prefix + suffix)) {
+      return MurmurHash3_32(tbname, tblen);
+    } else {
+      return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
+    }
+  }
+}
 
 #ifdef __cplusplus
 }

packaging/MPtestJenkinsfile

@@ -5,13 +5,6 @@ def sync_source(branch_name) {
         echo ''' + branch_name + '''
     '''
     sh '''
-        cd ${TDINTERNAL_ROOT_DIR}
-        git reset --hard
-        git fetch || git fetch
-        git checkout ''' + branch_name + ''' -f
-        git branch
-        git pull || git pull
-        git log | head -n 20
         cd ${TDENGINE_ROOT_DIR}
         git reset --hard
         git fetch || git fetch
@@ -64,17 +57,12 @@ pipeline {
             defaultValue:'2.1.2', 
             description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1'
         )
-        string (
-            name:'nasPassword', 
-            defaultValue:'password', 
-            description: 'the pasword of the NAS server which has installPackage-192.168.1.131'
-        )
     }
     environment{
         WORK_DIR = '/var/lib/jenkins/workspace'
         TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal'
         TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community'
-        BRANCH_NAME = '3.0'
+        BRANCH_NAME = 'test/chr/TD-14699'
 
         TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz"
         BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz"
@@ -107,7 +95,7 @@ pipeline {
 
     }
     stages {
-        stage ('RUN') {
+        stage ('Test Server') {
             parallel {
                 stage('ubuntu16') {
                     agent{label " ubuntu16 "}
@@ -116,17 +104,17 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -139,24 +127,21 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
-                            python3 checkPackageRuning.py  
-                            '''
-                            sh '''
-                            cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server  
                             python3 checkPackageRuning.py  
+                            dpkg -r tdengine
                             '''
+
                         }
                     }
                 }
@@ -167,17 +152,17 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                         }
@@ -190,28 +175,23 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging 
-                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} 
+                            bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server  
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword}
-                            python3 checkPackageRuning.py  
-                            '''            
-                            sh '''
-                            cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
-                            '''         
+                            sudo rpm -e tdengine 
+                            '''              
                         }
                     }
-                }
-                                    
+                }                  
                 stage('arm64') {
                     agent{label 'linux_arm64'}
                     steps {
@@ -219,18 +199,53 @@ pipeline {
                             sync_source("${BRANCH_NAME}")
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${nasPassword}
+                            bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server 
                             python3 checkPackageRuning.py  
                             '''
+                        }
+                    }
+                }
+            }
+        }
+        stage ('Test Client') {
+            parallel {
+                stage('ubuntu18') {
+                    agent{label " ubuntu18 "}
+                    steps {
+                        timeout(time: 30, unit: 'MINUTES'){
                             sh '''
                             cd ${TDENGINE_ROOT_DIR}/packaging
-                            bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client  ${nasPassword}
-                            python3 checkPackageRuning.py  
-                            '''                     
+                            bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client 
+                            python3 checkPackageRuning.py  192.168.0.21
+                            '''
                         }
                     }
                 }
+                stage('centos8') {
+                    agent{label " centos8_3 "}
+                    steps {
+                        timeout(time: 30, unit: 'MINUTES'){
+                            sh '''
+                            cd ${TDENGINE_ROOT_DIR}/packaging
+                            bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client 
+                            python3 checkPackageRuning.py  192.168.0.24
+                            '''         
+                        }
+                    }
+                }      
             }
-        }
-    }
+        }  
+        stage('arm64-client') {
+            agent{label " linux_arm64 "}
+            steps {
+                timeout(time: 30, unit: 'MINUTES'){
+                    sh '''
+                    cd ${TDENGINE_ROOT_DIR}/packaging
+                    bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client  
+                    python3 checkPackageRuning.py  192.168.0.21
+                    '''  
+                }
+            }
+        } 
+    }    
 }
\ No newline at end of file

packaging/checkPackageRuning.py

@@ -19,12 +19,19 @@ import subprocess
 # from this import d
 import time
 
+
+if( len(sys.argv)>1 ):
+    serverHost=sys.argv[1]
+else:
+    serverHost="localhost"
+
+
 # install taospy
 
 out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ")
 print("taospy version %s "%out)
 if (out == "" ):
-    os.system("pip install git+https://github.com/taosdata/taos-connector-python.git")
+    os.system("pip3 install git+https://github.com/taosdata/taos-connector-python.git")
     print("install taos python connector")
 else:
     os.system("pip3 install --upgrade  taospy  ")
@@ -32,19 +39,19 @@ else:
 
 
 # start taosd prepare 
-os.system("rm -rf /var/lib/taos/*")
-os.system("systemctl restart taosd ")
+# os.system("rm -rf /var/lib/taos/*")
+# os.system("systemctl restart taosd ")
 
 # wait a moment ,at least 5 seconds
 time.sleep(5)
 
 # prepare data by taosBenchmark 
 
-os.system("taosBenchmark -y -n 100 -t 100")
+os.system("taosBenchmark -y -n 100 -t 100 -h %s "%serverHost )
 
 import taos
 
-conn = taos.connect(host="localhost",
+conn = taos.connect(host="%s"%serverHost,
                                          user="root",
                                          password="taosdata",
                                          database="test",
@@ -80,15 +87,15 @@ os.system("rm -rf /tmp/dumpdata/*")
 # dump data out 
 print("taosdump dump out data")
 
-os.system("taosdump -o /tmp/dumpdata -D test -y ")
+os.system("taosdump -o /tmp/dumpdata -D test -y -h %s  "%serverHost)
 
 # drop database of test
 print("drop database test")
-os.system(" taos -s ' drop database test ;' ")
+os.system(" taos -s ' drop database test ;'  -h %s  "%serverHost)
 
 # dump data in 
 print("taosdump dump data in")
-os.system("taosdump -i /tmp/dumpdata  -y ")
+os.system("taosdump -i /tmp/dumpdata  -y  -h %s  "%serverHost)
 
 result = conn.query("SELECT count(*) from test.meters")
 

packaging/debRpmAutoInstall.sh

@@ -11,3 +11,5 @@ expect "*one:"
 send  "\r"
 expect "*skip:"
 send  "\r" 
+
+expect eof
\ No newline at end of file

packaging/testpackage.sh

@@ -7,7 +7,6 @@ originPackageName=$3
 originversion=$4
 testFile=$5
 subFile="taos.tar.gz"
-password=$6
 
 # Color setting
 RED='\033[41;30m'
@@ -68,11 +67,37 @@ fi
 }
 
 
+function wgetFile {
+
+file=$1
+
+if [ ! -f  ${file}  ];then
+    echoColor  BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
+    wget https://www.taosdata.com/assets-download/3.0/${file}
+else
+    echoColor  YD "${file} already exists "
+fi
+}
+
+function newPath {
+
+buildPath=$1
+
+if [ ! -d ${buildPath} ] ;then
+    echoColor BD "mkdir -p ${buildPath}"
+    mkdir -p ${buildPath}
+else  
+    echoColor YD "${buildPath} already exists"
+fi
+
+}
+
+
 echoColor G "===== install basesoft ====="
 
 cmdInstall tree
 cmdInstall wget
-cmdInstall sshpass
+cmdInstall expect
 
 echoColor G "===== Uninstall all components of TDeingne ====="
 
@@ -97,45 +122,28 @@ echoColor G "===== new workroom path ====="
 installPath="/usr/local/src/packageTest"
 oriInstallPath="/usr/local/src/packageTest/3.1"
 
-if [ ! -d ${installPath} ] ;then
-    echoColor BD "mkdir -p ${installPath}"
-    mkdir -p ${installPath}
-else  
-    echoColor YD "${installPath} already exists"
-fi
+newPath ${installPath}
 
-if [ -d ${installPath}/${tdPath} ] ;then
-    echoColor BD "rm -rf ${installPath}/${tdPath}/*"
-    rm -rf ${installPath}/${tdPath}/*
-fi
+newPath ${oriInstallPath}
 
-if [ ! -d ${oriInstallPath} ] ;then
-    echoColor BD "mkdir -p ${oriInstallPath}"
-    mkdir -p ${oriInstallPath}
-else  
-    echoColor YD "${oriInstallPath} already exists"
-fi
 
 if [ -d ${oriInstallPath}/${originTdpPath} ] ;then
     echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*"
     rm -rf  ${oriInstallPath}/${originTdpPath}/*  
 fi
 
+if [ -d ${installPath}/${tdPath} ] ;then
+    echoColor BD "rm -rf ${installPath}/${tdPath}/*"
+    rm -rf ${installPath}/${tdPath}/*
+fi
 
 echoColor G "===== download  installPackage ====="
-# cd ${installPath}
-# wget https://www.taosdata.com/assets-download/3.0/${packgeName}
-# cd  ${oriInstallPath}
-# wget https://www.taosdata.com/assets-download/3.0/${originPackageName}
+cd ${installPath} && wgetFile ${packgeName}
+cd  ${oriInstallPath}  && wgetFile ${originPackageName}
 
 cd ${installPath}
 cp -r ${scriptDir}/debRpmAutoInstall.sh   . 
 
-if [ ! -f  {packgeName}  ];then
-    echoColor  BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  ."
-    sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName}  .
-fi
-
 packageSuffix=$(echo ${packgeName}  | awk -F '.' '{print $NF}')
 
 
@@ -181,8 +189,7 @@ elif [[ ${packgeName} =~ "tar" ]];then
     cd  ${oriInstallPath}
     if [ ! -f  {originPackageName}  ];then
         echoColor YD "download  base installPackage"
-        echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} ."
-        sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} .
+        wgetFile ${originPackageName}
     fi
     echoColor YD "unzip the base installation package" 
     echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} 
@@ -222,24 +229,45 @@ fi
 
 cd ${installPath}
 
-if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
-    echoColor G "===== install taos-tools when package is lite or client ====="
-    cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
-    # wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz
-    tar xf taosTools-2.1.2-Linux-x64.tar.gz
-    cd taosTools-2.1.2 && bash install-taostools.sh
-elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "deb" ]] ;then
+if [[ ${packgeName} =~ "Lite" ]]  ||   ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) ||  ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]])  || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then
     echoColor G "===== install taos-tools when package is lite or client ====="
     cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
-    tar xf taosTools-2.1.2-Linux-x64.tar.gz
-    cd taosTools-2.1.2 && bash install-taostools.sh
-elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "rpm" ]]  ;then
-    echoColor G "===== install taos-tools when package is lite or client ====="
+    wgetFile taosTools-2.1.3-Linux-x64.tar.gz .
+    tar xf taosTools-2.1.3-Linux-x64.tar.gz
+    cd taosTools-2.1.3 && bash install-taostools.sh
+elif  ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then
+    echoColor G "===== install taos-tools arm when package is arm64-client ====="
     cd ${installPath}
-    sshpass -p ${password}   scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz .
-    tar xf taosTools-2.1.2-Linux-x64.tar.gz
-    cd taosTools-2.1.2 && bash install-taostools.sh
+    wgetFile taosTools-2.1.3-Linux-arm64.tar.gz .
+    tar xf taosTools-2.1.3-Linux-arm64.tar.gz
+    cd taosTools-2.1.3 && bash install-taostools.sh
 fi
 
+echoColor G  "===== start TDengine ====="
+
+if [[ ${packgeName} =~ "server" ]] ;then
+    echoColor BD " rm -rf /var/lib/taos/* &&  systemctl restart taosd "
+    rm -rf /var/lib/taos/*
+    systemctl restart taosd
+fi
+
+# if ([[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "tar" ]]) ||   [[ ${packgeName} =~ "client" ]] ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "deb" ]] ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# elif [[ ${packgeName} =~ "Lite" ]] &&  [[ ${packgeName} =~ "rpm" ]]  ;then
+#     echoColor G "===== install taos-tools when package is lite or client ====="
+#     cd ${installPath}
+#     wgetFile taosTools-2.1.2-Linux-x64.tar.gz .
+#     tar xf taosTools-2.1.2-Linux-x64.tar.gz
+#     cd taosTools-2.1.2 && bash install-taostools.sh
+# fi
+

source/common/src/systable.c

@@ -103,6 +103,8 @@ static const SSysDbTableSchema userDBSchema[] = {
     {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true},
     {.name = "sst_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
+    {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
+    {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
 };
 
 static const SSysDbTableSchema userFuncSchema[] = {

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -167,7 +167,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->walCfg.segSize = pCreate->walSegmentSize;
   pCfg->walCfg.level = pCreate->walLevel;
 
-  pCfg->sstTrigger = pCreate->sstTrigger;
+  pCfg->sttTrigger = pCreate->sstTrigger;
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
   pCfg->hashMethod = pCreate->hashMethod;
@@ -223,10 +223,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d",
-         createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
-         createReq.sstTrigger);
+        createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
+        createReq.sstTrigger);
   dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod,
-         createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
+        createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
   vmGenerateVnodeCfg(&createReq, &vnodeCfg);
 
   if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {

source/dnode/mnode/impl/src/mndDb.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndDb.h"
+#include "mndCluster.h"
 #include "mndDnode.h"
 #include "mndOffset.h"
 #include "mndPrivilege.h"
@@ -337,7 +338,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1;
   if (pCfg->walRollPeriod < TSDB_DB_MIN_WAL_ROLL_PERIOD) return -1;
   if (pCfg->walSegmentSize < TSDB_DB_MIN_WAL_SEGMENT_SIZE) return -1;
-  if (pCfg->sstTrigger < TSDB_MIN_SST_TRIGGER || pCfg->sstTrigger > TSDB_MAX_SST_TRIGGER) return -1;
+  if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
   if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
 
@@ -512,6 +513,12 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
     return -1;
   }
 
+  if (dbObj.cfg.hashPrefix > 0) {
+    int32_t dbLen = strlen(dbObj.name) + 1;
+    mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
+    dbObj.cfg.hashPrefix += dbLen;
+  }
+
   SVgObj *pVgroups = NULL;
   if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) {
     mError("db:%s, failed to create since %s", pCreate->db, terrstr());
@@ -1709,23 +1716,33 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    int16_t hashPrefix = pDb->cfg.hashPrefix;
+    if (hashPrefix > 0) {
+      hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
+    }
+    colDataAppend(pColInfo, rows, (const char *)&hashPrefix, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false);
   }
 
   taosMemoryFree(buf);
 }
 
-static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
+static void setInformationSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) {
   tstrncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name));
-  pDbObj->createdTime = 0;
+  pDbObj->createdTime = mndGetClusterCreateTime(pMnode);
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
   pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
 }
 
-static void setPerfSchemaDbCfg(SDbObj *pDbObj) {
+static void setPerfSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) {
   tstrncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name));
-  pDbObj->createdTime = 0;
+  pDbObj->createdTime = mndGetClusterCreateTime(pMnode);
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
@@ -1756,7 +1773,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
   // Append the information_schema database into the result.
   if (!pShow->sysDbRsp) {
     SDbObj infoschemaDb = {0};
-    setInformationSchemaDbCfg(&infoschemaDb);
+    setInformationSchemaDbCfg(pMnode, &infoschemaDb);
     size_t numOfTables = 0;
     getVisibleInfosTablesNum(sysinfo, &numOfTables);
     mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1);
@@ -1764,7 +1781,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
     numOfRows += 1;
 
     SDbObj perfschemaDb = {0};
-    setPerfSchemaDbCfg(&perfschemaDb);
+    setPerfSchemaDbCfg(pMnode, &perfschemaDb);
     numOfTables = 0;
     getPerfDbMeta(NULL, &numOfTables);
     mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1);

source/dnode/vnode/inc/vnode.h

@@ -125,6 +125,9 @@ int32_t     metaTbCursorNext(SMTbCursor *pTbCur);
 // typedef struct STsdb STsdb;
 typedef struct STsdbReader STsdbReader;
 
+#define TSDB_DEFAULT_STT_FILE  8
+#define TSDB_DEFAULT_PAGE_SIZE 4096
+
 #define TIMEWINDOW_RANGE_CONTAINED 1
 #define TIMEWINDOW_RANGE_EXTERNAL  2
 
@@ -288,9 +291,10 @@ struct SVnodeCfg {
   SVnodeStats vndStats;
   uint32_t    hashBegin;
   uint32_t    hashEnd;
-  int16_t     sstTrigger;
+  int16_t     sttTrigger;
   int16_t     hashPrefix;
   int16_t     hashSuffix;
+  int32_t     tsdbPageSize;
 };
 
 typedef struct {

source/dnode/vnode/src/inc/tsdb.h

@@ -67,12 +67,9 @@ typedef struct SBlockCol     SBlockCol;
 typedef struct SVersionRange SVersionRange;
 typedef struct SLDataIter    SLDataIter;
 
-#define TSDB_FILE_DLMT         ((uint32_t)0xF00AFA0F)
-#define TSDB_MAX_SUBBLOCKS     8
-#define TSDB_MAX_STT_FILE      16
-#define TSDB_DEFAULT_STT_FILE  8
-#define TSDB_FHDR_SIZE         512
-#define TSDB_DEFAULT_PAGE_SIZE 4096
+#define TSDB_FILE_DLMT     ((uint32_t)0xF00AFA0F)
+#define TSDB_MAX_SUBBLOCKS 8
+#define TSDB_FHDR_SIZE     512
 
 #define HAS_NONE  ((int8_t)0x1)
 #define HAS_NULL  ((int8_t)0x2)
@@ -84,13 +81,26 @@ typedef struct SLDataIter    SLDataIter;
 #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
 #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})
 
+#define TABLE_SAME_SCHEMA(SUID1, UID1, SUID2, UID2) ((SUID1) ? (SUID1) == (SUID2) : (UID1) == (UID2))
+
 #define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
 #define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
   ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
 #define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
 #define PAGE_OFFSET(PGNO, PAGE)            (((PGNO)-1) * (PAGE))
 #define OFFSET_PGNO(OFFSET, PAGE)          ((OFFSET) / (PAGE) + 1)
-#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE)    OFFSET_PGNO(LOGIC_TO_FILE_OFFSET(LSIZE, PAGE), PAGE) * (PAGE)
+
+static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
+  int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage);
+  int64_t pgno = OFFSET_PGNO(fOffSet, szPage);
+  int32_t szPageCont = PAGE_CONTENT_SIZE(szPage);
+
+  if (fOffSet % szPageCont == 0) {
+    pgno--;
+  }
+
+  return pgno * szPage;
+}
 
 // tsdbUtil.c ==============================================================================================
 // TSDBROW
@@ -262,7 +272,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
 int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
 int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
 int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
-int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx);
+int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx);
 int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
 int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
                            int8_t cmprAlg, int8_t toLast);
@@ -272,7 +282,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
 int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet);
 int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
 int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
-int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData);
+int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk);
 int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
 int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
 int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
@@ -572,7 +582,7 @@ struct SDFileSet {
   SDataFile *pDataF;
   SSmaFile  *pSmaF;
   uint8_t    nSttF;
-  SSttFile  *aSttF[TSDB_MAX_STT_FILE];
+  SSttFile  *aSttF[TSDB_MAX_STT_TRIGGER];
 };
 
 struct SRowIter {
@@ -622,7 +632,7 @@ struct SDataFWriter {
   SHeadFile fHead;
   SDataFile fData;
   SSmaFile  fSma;
-  SSttFile  fStt[TSDB_MAX_STT_FILE];
+  SSttFile  fStt[TSDB_MAX_STT_TRIGGER];
 
   uint8_t *aBuf[4];
 };
@@ -633,7 +643,7 @@ struct SDataFReader {
   STsdbFD   *pHeadFD;
   STsdbFD   *pDataFD;
   STsdbFD   *pSmaFD;
-  STsdbFD   *aSttFD[TSDB_MAX_STT_FILE];
+  STsdbFD   *aSttFD[TSDB_MAX_STT_TRIGGER];
   uint8_t   *aBuf[3];
 };
 
@@ -644,31 +654,41 @@ typedef struct {
 } SRowInfo;
 
 typedef struct SSttBlockLoadInfo {
-    SBlockData blockData[2];
-    SArray    *aSttBlk;
-    int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
-    int32_t    currentLoadBlockIndex;
+  SBlockData blockData[2];
+  SArray    *aSttBlk;
+  int32_t    blockIndex[2];  // to denote the loaded block in the corresponding position.
+  int32_t    currentLoadBlockIndex;
+  int32_t    loadBlocks;
+  double     elapsedTime;
 } SSttBlockLoadInfo;
 
 typedef struct SMergeTree {
-  int8_t      backward;
-  SRBTree     rbt;
-  SArray     *pIterList;
-  SLDataIter *pIter;
-  bool        destroyLoadInfo;
-  SSttBlockLoadInfo* pLoadInfo;
+  int8_t             backward;
+  SRBTree            rbt;
+  SArray            *pIterList;
+  SLDataIter        *pIter;
+  bool               destroyLoadInfo;
+  SSttBlockLoadInfo *pLoadInfo;
+  const char        *idStr;
 } SMergeTree;
 
+typedef struct {
+  int64_t   suid;
+  int64_t   uid;
+  STSchema *pTSchema;
+} SSkmInfo;
+
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
-                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pLoadInfo);
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr);
 void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 
-SSttBlockLoadInfo* tCreateLastBlockLoadInfo();
-void  resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo);
-void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo);
+SSttBlockLoadInfo *tCreateLastBlockLoadInfo();
+void               resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
+void               getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el);
+void              *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
 
 // ========== inline functions ==========
 static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -104,7 +104,7 @@ int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq*
 int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp);
-int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t *tbUid);
+int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid);
 int             metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -457,7 +457,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
 
       tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX},
-                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL);
+                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL);
       bool hasVal = tMergeTreeNext(&state->mergeTree);
       if (!hasVal) {
         state->state = SFSLASTNEXTROW_FILESET;
@@ -589,7 +589,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       }
 
       tMapDataReset(&state->blockMap);
-      code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap);
+      code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap);
       if (code) goto _err;
 
       state->nBlock = state->blockMap.nItem;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -14,13 +14,8 @@
  */
 
 #include "tsdb.h"
-typedef struct {
-  int64_t   suid;
-  int64_t   uid;
-  STSchema *pTSchema;
-} SSkmInfo;
 
-typedef enum { MEMORY_DATA_ITER = 0, LAST_DATA_ITER } EDataIterT;
+typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT;
 
 typedef struct {
   SRBTreeNode n;
@@ -50,7 +45,7 @@ typedef struct {
   int32_t minRow;
   int32_t maxRow;
   int8_t  cmprAlg;
-  int8_t  maxLast;
+  int8_t  sttTrigger;
   SArray *aTbDataP;  // memory
   STsdbFS fs;        // disk
   // --------------
@@ -71,7 +66,7 @@ typedef struct {
     SDataIter *pIter;
     SRBTree    rbt;
     SDataIter  dataIter;
-    SDataIter  aDataIter[TSDB_MAX_STT_FILE];
+    SDataIter  aDataIter[TSDB_MAX_STT_TRIGGER];
     int8_t     toLastOnly;
   };
   struct {
@@ -99,7 +94,7 @@ static int32_t tsdbCommitCache(SCommitter *pCommitter);
 static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno);
 static int32_t tsdbNextCommitRow(SCommitter *pCommitter);
 
-static int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
+int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
   SRowInfo *pInfo1 = (SRowInfo *)p1;
   SRowInfo *pInfo2 = (SRowInfo *)p2;
 
@@ -325,22 +320,22 @@ _err:
   return code;
 }
 
-static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) {
+int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) {
   int32_t code = 0;
 
   if (suid) {
-    if (pCommitter->skmTable.suid == suid) {
-      pCommitter->skmTable.uid = uid;
+    if (pSkmInfo->suid == suid) {
+      pSkmInfo->uid = uid;
       goto _exit;
     }
   } else {
-    if (pCommitter->skmTable.uid == uid) goto _exit;
+    if (pSkmInfo->uid == uid) goto _exit;
   }
 
-  pCommitter->skmTable.suid = suid;
-  pCommitter->skmTable.uid = uid;
-  tTSchemaDestroy(pCommitter->skmTable.pTSchema);
-  code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema);
+  pSkmInfo->suid = suid;
+  pSkmInfo->uid = uid;
+  tTSchemaDestroy(pSkmInfo->pTSchema);
+  code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema);
   if (code) goto _exit;
 
 _exit:
@@ -382,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
     pCommitter->dReader.pBlockIdx =
         (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
 
-    code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
+    code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
     if (code) goto _exit;
 
     ASSERT(pCommitter->dReader.mBlock.nItem > 0);
@@ -428,11 +423,11 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
   pCommitter->toLastOnly = 0;
   SDataFReader *pReader = pCommitter->dReader.pReader;
   if (pReader) {
-    if (pReader->pSet->nSttF >= pCommitter->maxLast) {
+    if (pReader->pSet->nSttF >= pCommitter->sttTrigger) {
       int8_t iIter = 0;
       for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
         pIter = &pCommitter->aDataIter[iIter];
-        pIter->type = LAST_DATA_ITER;
+        pIter->type = STT_DATA_ITER;
         pIter->iStt = iStt;
 
         code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk);
@@ -498,7 +493,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
     pCommitter->dReader.iBlockIdx = 0;
     if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) {
       pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0);
-      code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
+      code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
       if (code) goto _err;
     } else {
       pCommitter->dReader.pBlockIdx = NULL;
@@ -515,11 +510,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   SSttFile  fStt = {.commitID = pCommitter->commitID};
   SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
   if (pRSet) {
-    ASSERT(pRSet->nSttF <= pCommitter->maxLast);
+    ASSERT(pRSet->nSttF <= pCommitter->sttTrigger);
     fData = *pRSet->pDataF;
     fSma = *pRSet->pSmaF;
     wSet.diskId = pRSet->diskId;
-    if (pRSet->nSttF < pCommitter->maxLast) {
+    if (pRSet->nSttF < pCommitter->sttTrigger) {
       for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) {
         wSet.aSttF[iStt] = pRSet->aSttF[iStt];
       }
@@ -556,46 +551,45 @@ _err:
   return code;
 }
 
-static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) {
-  int32_t     code = 0;
-  SBlockData *pBlockData = &pCommitter->dWriter.bData;
-  SDataBlk    block;
+int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) {
+  int32_t code = 0;
 
-  ASSERT(pBlockData->nRow > 0);
+  if (pBlockData->nRow == 0) return code;
 
-  tDataBlkReset(&block);
+  SDataBlk dataBlk;
+  tDataBlkReset(&dataBlk);
 
   // info
-  block.nRow += pBlockData->nRow;
+  dataBlk.nRow += pBlockData->nRow;
   for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
     TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
 
     if (iRow == 0) {
-      if (tsdbKeyCmprFn(&block.minKey, &key) > 0) {
-        block.minKey = key;
+      if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) {
+        dataBlk.minKey = key;
       }
     } else {
       if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
-        block.hasDup = 1;
+        dataBlk.hasDup = 1;
       }
     }
 
-    if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) {
-      block.maxKey = key;
+    if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) {
+      dataBlk.maxKey = key;
     }
 
-    block.minVer = TMIN(block.minVer, key.version);
-    block.maxVer = TMAX(block.maxVer, key.version);
+    dataBlk.minVer = TMIN(dataBlk.minVer, key.version);
+    dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version);
   }
 
   // write
-  block.nSubBlock++;
-  code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1],
-                            ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0);
+  dataBlk.nSubBlock++;
+  code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1],
+                            ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0);
   if (code) goto _err;
 
   // put SDataBlk
-  code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutDataBlk);
+  code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk);
   if (code) goto _err;
 
   // clear
@@ -604,39 +598,38 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter) {
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
-  int32_t     code = 0;
-  SSttBlk     blockL;
-  SBlockData *pBlockData = &pCommitter->dWriter.bDatal;
+int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) {
+  int32_t code = 0;
+  SSttBlk sstBlk;
 
-  ASSERT(pBlockData->nRow > 0);
+  if (pBlockData->nRow == 0) return code;
 
   // info
-  blockL.suid = pBlockData->suid;
-  blockL.nRow = pBlockData->nRow;
-  blockL.minKey = TSKEY_MAX;
-  blockL.maxKey = TSKEY_MIN;
-  blockL.minVer = VERSION_MAX;
-  blockL.maxVer = VERSION_MIN;
+  sstBlk.suid = pBlockData->suid;
+  sstBlk.nRow = pBlockData->nRow;
+  sstBlk.minKey = TSKEY_MAX;
+  sstBlk.maxKey = TSKEY_MIN;
+  sstBlk.minVer = VERSION_MAX;
+  sstBlk.maxVer = VERSION_MIN;
   for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
-    blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]);
-    blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]);
-    blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]);
-    blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]);
+    sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]);
+    sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]);
+    sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]);
+    sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]);
   }
-  blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0];
-  blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1];
+  sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0];
+  sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1];
 
   // write
-  code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1);
+  code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1);
   if (code) goto _err;
 
   // push SSttBlk
-  if (taosArrayPush(pCommitter->dWriter.aSttBlk, &blockL) == NULL) {
+  if (taosArrayPush(aSttBlk, &sstBlk) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
@@ -647,7 +640,7 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -692,7 +685,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
 
   while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) {
     SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
-    code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
+    code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
     if (code) goto _err;
 
     if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
@@ -757,7 +750,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
   pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
   pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
-  pCommitter->maxLast = TSDB_DEFAULT_STT_FILE;  // TODO: make it as a config
+  pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger;
   pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
   if (pCommitter->aTbDataP == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -787,7 +780,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
   if (code) goto _exit;
 
   // merger
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
     SDataIter *pIter = &pCommitter->aDataIter[iStt];
     pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
     if (pIter->aSttBlk == NULL) {
@@ -829,7 +822,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   tBlockDataDestroy(&pCommitter->dReader.bData, 1);
 
   // merger
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
     SDataIter *pIter = &pCommitter->aDataIter[iStt];
     taosArrayDestroy(pIter->aSttBlk);
     tBlockDataDestroy(&pIter->bData, 1);
@@ -1046,7 +1039,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
           break;
         }
       }
-    } else if (pCommitter->pIter->type == LAST_DATA_ITER) {  // last file
+    } else if (pCommitter->pIter->type == STT_DATA_ITER) {  // last file
       pIter->iRow++;
       if (pIter->iRow < pIter->bData.nRow) {
         pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
@@ -1124,15 +1117,14 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
     }
 
     if (pBlockData->nRow >= pCommitter->maxRow) {
-      code = tsdbCommitDataBlock(pCommitter);
+      code =
+          tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
       if (code) goto _err;
     }
   }
 
-  if (pBlockData->nRow) {
-    code = tsdbCommitDataBlock(pCommitter);
-    if (code) goto _err;
-  }
+  code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+  if (code) goto _err;
 
   return code;
 
@@ -1193,7 +1185,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
     }
 
     if (pBDataW->nRow >= pCommitter->maxRow) {
-      code = tsdbCommitDataBlock(pCommitter);
+      code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
       if (code) goto _err;
     }
   }
@@ -1210,15 +1202,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
     }
 
     if (pBDataW->nRow >= pCommitter->maxRow) {
-      code = tsdbCommitDataBlock(pCommitter);
+      code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
       if (code) goto _err;
     }
   }
 
-  if (pBDataW->nRow) {
-    code = tsdbCommitDataBlock(pCommitter);
-    if (code) goto _err;
-  }
+  code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+  if (code) goto _err;
 
   return code;
 
@@ -1306,10 +1296,8 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) {
   SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
   if (pBDatal->suid || pBDatal->uid) {
     if ((pBDatal->suid != id.suid) || (id.suid == 0)) {
-      if (pBDatal->nRow) {
-        code = tsdbCommitLastBlock(pCommitter);
-        if (code) goto _exit;
-      }
+      code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
+      if (code) goto _exit;
       tBlockDataReset(pBDatal);
     }
   }
@@ -1341,7 +1329,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
     if (code) goto _err;
 
     if (pBDatal->nRow >= pCommitter->maxRow) {
-      code = tsdbCommitLastBlock(pCommitter);
+      code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
       if (code) goto _err;
     }
   }
@@ -1393,10 +1381,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
 
     if (pBData->nRow >= pCommitter->maxRow) {
       if (pCommitter->toLastOnly) {
-        code = tsdbCommitLastBlock(pCommitter);
+        code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
         if (code) goto _err;
       } else {
-        code = tsdbCommitDataBlock(pCommitter);
+        code =
+            tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
         if (code) goto _err;
       }
     }
@@ -1404,7 +1393,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
 
   if (!pCommitter->toLastOnly && pBData->nRow) {
     if (pBData->nRow > pCommitter->minRow) {
-      code = tsdbCommitDataBlock(pCommitter);
+      code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
       if (code) goto _err;
     } else {
       code = tsdbAppendLastBlock(pCommitter);
@@ -1437,7 +1426,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
     tMapDataReset(&pCommitter->dWriter.mBlock);
 
     // impl
-    code = tsdbCommitterUpdateTableSchema(pCommitter, id.suid, id.uid);
+    code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable);
     if (code) goto _err;
     code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema);
     if (code) goto _err;
@@ -1455,7 +1444,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
     // end
     if (pCommitter->dWriter.mBlock.nItem > 0) {
       SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid};
-      code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
+      code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
       if (code) goto _err;
 
       if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
@@ -1470,10 +1459,9 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
   code = tsdbMoveCommitData(pCommitter, id);
   if (code) goto _err;
 
-  if (pCommitter->dWriter.bDatal.nRow > 0) {
-    code = tsdbCommitLastBlock(pCommitter);
-    if (code) goto _err;
-  }
+  code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
+                           pCommitter->cmprAlg);
+  if (code) goto _err;
 
   return code;
 

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -295,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size != LOGIC_TO_FILE_SIZE(pSet->pHeadF->size, TSDB_DEFAULT_PAGE_SIZE)) {
+    if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
     }
@@ -306,10 +306,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
+    if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
+    } else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE);
       if (code) goto _err;
     }
@@ -320,10 +320,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
+    if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
+    } else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
       if (code) goto _err;
     }
@@ -335,7 +335,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
         code = TAOS_SYSTEM_ERROR(errno);
         goto _err;
       }
-      if (size != LOGIC_TO_FILE_SIZE(pSet->aSttF[iStt]->size, TSDB_DEFAULT_PAGE_SIZE)) {
+      if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) {
         code = TSDB_CODE_FILE_CORRUPTED;
         goto _err;
       }

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
   }
 
   // ftruncate
-  if (taosFtruncateFile(pFD, LOGIC_TO_FILE_SIZE(size, TSDB_DEFAULT_PAGE_SIZE)) < 0) {
+  if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -67,6 +67,16 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) {
     pLoadInfo[i].blockIndex[1] = -1;
 
     taosArrayClear(pLoadInfo[i].aSttBlk);
+
+    pLoadInfo[i].elapsedTime = 0;
+    pLoadInfo[i].loadBlocks = 0;
+  }
+}
+
+void getLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo, int64_t* blocks, double* el) {
+  for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) {
+    *el += pLoadInfo[i].elapsedTime;
+    *blocks += pLoadInfo[i].loadBlocks;
   }
 }
 
@@ -86,7 +96,7 @@ void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) {
   return NULL;
 }
 
-static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
+static SBlockData* loadLastBlock(SLDataIter *pIter, const char* idStr) {
   int32_t code = 0;
 
   SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo;
@@ -100,8 +110,13 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
 
   pInfo->currentLoadBlockIndex ^= 1;
   if (pIter->pSttBlk != NULL) {  // current block not loaded yet
+    int64_t st = taosGetTimestampUs();
     code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]);
-    tsdbDebug("read last block, index:%d, last file index:%d", pIter->iSttBlk, pIter->iStt);
+    double el = (taosGetTimestampUs() - st)/ 1000.0;
+    pInfo->elapsedTime += el;
+    pInfo->loadBlocks += 1;
+
+    tsdbDebug("read last block, index:%d, last file index:%d, elapsed time:%.2f ms, %s", pIter->iSttBlk, pIter->iStt, el, idStr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
     }
@@ -120,6 +135,92 @@ static SBlockData* loadBlockIfMissing(SLDataIter *pIter) {
   return NULL;
 }
 
+// find the earliest block that contains the required records
+static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) {
+  int32_t i = index;
+  int32_t step = backward? 1:-1;
+  while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) {
+    i += step;
+  }
+  return i - step;
+}
+
+static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64_t uid, int32_t backward) {
+  int32_t midPos = -1;
+  if (num <= 0) {
+    return -1;
+  }
+
+  int32_t firstPos = 0;
+  int32_t lastPos = num - 1;
+
+  // find the first position which is bigger than the key
+  if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) {
+    return -1;
+  }
+
+  while (1) {
+    if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) {
+      return findEarliestIndex(firstPos, uid, pBlockList, num, backward);
+    }
+
+    if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) {
+      return -1;
+    }
+
+    int32_t numOfRows = lastPos - firstPos + 1;
+    midPos = (numOfRows >> 1u) + firstPos;
+
+    if (uid < pBlockList[midPos].minUid) {
+      lastPos = midPos - 1;
+    } else if (uid > pBlockList[midPos].maxUid) {
+      firstPos = midPos + 1;
+    } else {
+      return findEarliestIndex(midPos, uid, pBlockList, num, backward);
+    }
+  }
+}
+
+static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) {
+  int32_t i = index;
+  int32_t step = backward? 1:-1;
+  while (i >= 0 && i < num && uid == uidList[i]) {
+    i += step;
+  }
+  return i - step;
+}
+
+static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint64_t uid, int32_t backward) {
+  int32_t firstPos = 0;
+  int32_t lastPos = num - 1;
+
+  // find the first position which is bigger than the key
+  if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) {
+    return -1;
+  }
+
+  while (1) {
+    if (uid == uidList[firstPos]) {
+      return findEarliestRow(firstPos, uid, uidList, num, backward);
+    }
+
+    if (uid > uidList[lastPos] || uid < uidList[firstPos]) {
+      return -1;
+    }
+
+    int32_t numOfRows = lastPos - firstPos + 1;
+    int32_t midPos = (numOfRows >> 1u) + firstPos;
+
+    if (uid < uidList[midPos]) {
+      lastPos = midPos - 1;
+    } else if (uid > uidList[midPos]) {
+      firstPos = midPos + 1;
+    } else {
+      return findEarliestRow(midPos, uid, uidList, num, backward);
+    }
+  }
+}
+
 int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
                        uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) {
   int32_t code = 0;
@@ -141,41 +242,26 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
       goto _exit;
+    } else {
+      size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+      SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk));
+      for(int32_t i = 0; i < size; ++i) {
+        SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
+        if (p->suid == suid) {
+          taosArrayPush(pTmp, p);
+        }
+      }
+
+      taosArrayDestroy(pBlockLoadInfo->aSttBlk);
+      pBlockLoadInfo->aSttBlk = pTmp;
     }
   }
 
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
-  int32_t index = -1;
-  if (!backward) {  // asc
-    for (int32_t i = 0; i < size; ++i) {
-      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
-      if (p->suid != suid) {
-        continue;
-      }
-
-      if (p->minUid <= uid && p->maxUid >= uid) {
-        index = i;
-        break;
-      }
-    }
-  } else {  // desc
-    for (int32_t i = size - 1; i >= 0; --i) {
-      SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i);
-      if (p->suid != suid) {
-        continue;
-      }
-
-      if (p->minUid <= uid && p->maxUid >= uid) {
-        index = i;
-        break;
-      }
-    }
-  }
-
-  (*pIter)->iSttBlk = index;
-  if (index != -1) {
+  (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
+  if ((*pIter)->iSttBlk != -1) {
     (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
     (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
   }
@@ -193,7 +279,7 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
   pIter->iSttBlk += step;
 
   int32_t index = -1;
-  size_t  size = taosArrayGetSize(pIter->pBlockLoadInfo->aSttBlk);
+  size_t  size = pIter->pBlockLoadInfo->aSttBlk->size;
   for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) {
     SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i);
     if ((!pIter->backward) && p->minUid > pIter->uid) {
@@ -232,33 +318,41 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
     }
   }
 
-  if (index == -1) {
-    pIter->pSttBlk = NULL;
-  } else {
+  pIter->pSttBlk = NULL;
+  if (index != -1) {
     pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
   }
 }
 
-static void findNextValidRow(SLDataIter *pIter) {
+static void findNextValidRow(SLDataIter *pIter, const char* idStr) {
   int32_t step = pIter->backward ? -1 : 1;
 
   bool        hasVal = false;
   int32_t     i = pIter->iRow;
 
-  SBlockData *pBlockData = loadBlockIfMissing(pIter);
+  SBlockData *pBlockData = loadLastBlock(pIter, idStr);
+
+  // mostly we only need to find the start position for a given table
+  if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) {
+    i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward);
+    if (i == -1) {
+      pIter->iRow = -1;
+      return;
+    }
+  }
 
   for (; i < pBlockData->nRow && i >= 0; i += step) {
     if (pBlockData->aUid != NULL) {
       if (!pIter->backward) {
-        if (pBlockData->aUid[i] < pIter->uid) {
+        /*if (pBlockData->aUid[i] < pIter->uid) {
           continue;
-        } else if (pBlockData->aUid[i] > pIter->uid) {
+        } else */if (pBlockData->aUid[i] > pIter->uid) {
           break;
         }
       } else {
-        if (pBlockData->aUid[i] > pIter->uid) {
+        /*if (pBlockData->aUid[i] > pIter->uid) {
           continue;
-        } else if (pBlockData->aUid[i] < pIter->uid) {
+        } else */if (pBlockData->aUid[i] < pIter->uid) {
           break;
         }
       }
@@ -296,7 +390,7 @@ static void findNextValidRow(SLDataIter *pIter) {
   pIter->iRow = (hasVal) ? i : -1;
 }
 
-bool tLDataIterNextRow(SLDataIter *pIter) {
+bool tLDataIterNextRow(SLDataIter *pIter, const char* idStr) {
   int32_t code = 0;
   int32_t step = pIter->backward ? -1 : 1;
 
@@ -306,11 +400,11 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
   }
 
   int32_t iBlockL = pIter->iSttBlk;
-  SBlockData *pBlockData = loadBlockIfMissing(pIter);
+  SBlockData *pBlockData = loadLastBlock(pIter, idStr);
   pIter->iRow += step;
 
   while (1) {
-    findNextValidRow(pIter);
+    findNextValidRow(pIter, idStr);
 
     if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
       tLDataIterNextBlock(pIter);
@@ -322,7 +416,7 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
     }
 
     if (iBlockL != pIter->iSttBlk) {
-      pBlockData = loadBlockIfMissing(pIter);
+      pBlockData = loadLastBlock(pIter, idStr);
       pIter->iRow += step;
     }
   }
@@ -365,7 +459,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) {
 }
 
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
-                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo) {
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, const char* idStr) {
   pMTree->backward = backward;
   pMTree->pIter = NULL;
   pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
@@ -373,6 +467,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pMTree->idStr = idStr;
+
   tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -395,7 +491,7 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
       goto _end;
     }
 
-    bool hasVal = tLDataIterNextRow(pIter);
+    bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
     if (hasVal) {
       taosArrayPush(pMTree->pIterList, &pIter);
       tMergeTreeAddIter(pMTree, pIter);
@@ -418,7 +514,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
   if (pMTree->pIter) {
     SLDataIter *pIter = pMTree->pIter;
 
-    bool hasVal = tLDataIterNextRow(pIter);
+    bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
     if (!hasVal) {
       pMTree->pIter = NULL;
     }

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -209,7 +209,7 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS
   int32_t       code = 0;
   int32_t       flag;
   int64_t       n;
-  int32_t       szPage = TSDB_DEFAULT_PAGE_SIZE;
+  int32_t       szPage = pTsdb->pVnode->config.tsdbPageSize;
   SDataFWriter *pWriter = NULL;
   char          fname[TSDB_FILENAME_LEN];
   char          hdr[TSDB_FHDR_SIZE] = {0};
@@ -418,21 +418,21 @@ _err:
   return code;
 }
 
-int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) {
+int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) {
   int32_t    code = 0;
   SHeadFile *pHeadFile = &pWriter->fHead;
   int64_t    size;
   int64_t    n;
 
-  ASSERT(mBlock->nItem > 0);
+  ASSERT(mDataBlk->nItem > 0);
 
   // alloc
-  size = tPutMapData(NULL, mBlock);
+  size = tPutMapData(NULL, mDataBlk);
   code = tRealloc(&pWriter->aBuf[0], size);
   if (code) goto _err;
 
   // build
-  n = tPutMapData(pWriter->aBuf[0], mBlock);
+  n = tPutMapData(pWriter->aBuf[0], mDataBlk);
 
   // write
   code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size);
@@ -446,7 +446,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBloc
   tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
             " size:%" PRId64 " nItem:%d",
             TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
-            pBlockIdx->offset, pBlockIdx->size, mBlock->nItem);
+            pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem);
   return code;
 
 _err:
@@ -611,28 +611,26 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   int32_t   code = 0;
   int64_t   n;
   int64_t   size;
-  TdFilePtr pOutFD = NULL;  // TODO
-  TdFilePtr PInFD = NULL;   // TODO
+  TdFilePtr pOutFD = NULL;
+  TdFilePtr PInFD = NULL;
+  int32_t   szPage = pTsdb->pVnode->config.szPage;
   char      fNameFrom[TSDB_FILENAME_LEN];
   char      fNameTo[TSDB_FILENAME_LEN];
 
   // head
   tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
   tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
-
   pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
   PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
   if (PInFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size);
+  n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage));
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -643,44 +641,17 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   // data
   tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
   tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
-
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
-  if (pOutFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
-  if (PInFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  taosCloseFile(&pOutFD);
-  taosCloseFile(&PInFD);
-
-  // stt
-  tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[0], fNameFrom);
-  tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[0], fNameTo);
-
   pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
   PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
   if (PInFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSttF[0]->size);
+  n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage));
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -691,20 +662,17 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   // sma
   tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
   tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
-
   pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
   PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
   if (PInFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size);
+  n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage));
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -712,6 +680,29 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   taosCloseFile(&pOutFD);
   taosCloseFile(&PInFD);
 
+  // stt
+  for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) {
+    tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom);
+    tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo);
+    pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+    if (pOutFD == NULL) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+    PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+    if (PInFD == NULL) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+    n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage));
+    if (n < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+    taosCloseFile(&pOutFD);
+    taosCloseFile(&PInFD);
+  }
+
   return code;
 
 _err:
@@ -723,7 +714,7 @@ _err:
 int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
   int32_t       code = 0;
   SDataFReader *pReader;
-  int32_t       szPage = TSDB_DEFAULT_PAGE_SIZE;
+  int32_t       szPage = pTsdb->pVnode->config.tsdbPageSize;
   char          fname[TSDB_FILENAME_LEN];
 
   // alloc
@@ -780,7 +771,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
   tsdbCloseFile(&(*ppReader)->pSmaFD);
 
   // stt
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
     if ((*ppReader)->aSttFD[iStt]) {
       tsdbCloseFile(&(*ppReader)->aSttFD[iStt]);
     }
@@ -872,7 +863,7 @@ _err:
   return code;
 }
 
-int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) {
+int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) {
   int32_t code = 0;
   int64_t offset = pBlockIdx->offset;
   int64_t size = pBlockIdx->size;
@@ -886,7 +877,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl
   if (code) goto _err;
 
   // decode
-  int64_t n = tGetMapData(pReader->aBuf[0], mBlock);
+  int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk);
   if (n < 0) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -1053,6 +1044,29 @@ _err:
   return code;
 }
 
+int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
+  int32_t     code = 0;
+  SBlockInfo *pBlockInfo = &pDataBlk->aSubBlock[0];
+
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], pBlockInfo->szBlock);
+  if (code) goto _err;
+
+  // read
+  code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock);
+  if (code) goto _err;
+
+  // decmpr
+  code = tDecmprBlockData(pReader->aBuf[0], pBlockInfo->szBlock, pBlockData, &pReader->aBuf[1]);
+  if (code) goto _err;
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb read data block ex failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
 int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
   int32_t code = 0;
 
@@ -1147,8 +1161,8 @@ int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb
   pDelFWriter->fDel = *pFile;
 
   tsdbDelFileName(pTsdb, pFile, fname);
-  code =
-      tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH);
+  code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE,
+                      &pDelFWriter->pWriteH);
   if (code) goto _err;
 
   // update header
@@ -1315,7 +1329,7 @@ int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb
   pDelFReader->fDel = *pFile;
 
   tsdbDelFileName(pTsdb, pFile, fname);
-  code = tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ, &pDelFReader->pReadH);
+  code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH);
   if (code) goto _err;
 
   *ppReader = pDelFReader;

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -82,8 +82,6 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
       code = tsdbFSUpsertFSet(&fs, &fSet);
       if (code) goto _err;
     }
-
-    /* code */
   }
 
   // do change fs

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
 #include "tutil.h"
+#include "vnd.h"
 
 const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
                                    .dbname = "",
@@ -48,7 +48,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
                                        },
                                    .hashBegin = 0,
                                    .hashEnd = 0,
-                                   .hashMethod = 0};
+                                   .hashMethod = 0,
+                                   .sttTrigger = TSDB_DEFAULT_STT_FILE,
+                                   .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE};
 
 int vnodeCheckCfg(const SVnodeCfg *pCfg) {
   // TODO
@@ -107,7 +109,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sstTrigger) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
@@ -132,6 +134,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
     tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
   }
 
+  // add tsdb page size config
+  if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
+
   return 0;
 }
 
@@ -209,7 +214,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
   if (code < 0) return -1;
-  tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sstTrigger, code);
+  tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
   if (code < 0) return -1;
@@ -249,6 +254,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
     tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
   }
 
+  tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code);
+
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -60,6 +60,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
 
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
 
+  info.config = vnodeCfgDefault;
+
   // load vnode info
   ret = vnodeLoadInfo(dir, &info);
   if (ret < 0) {

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -354,7 +354,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
       code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
       if (code) goto _err;
     } break;
-    case SNAP_DATA_TSDB: {
+    case SNAP_DATA_TSDB:
+    case SNAP_DATA_DEL: {
       // tsdb
       if (pWriter->pTsdbSnapWriter == NULL) {
         code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);

source/libs/executor/inc/executil.h

@@ -126,6 +126,7 @@ SArray*  extractPartitionColInfo(SNodeList* pNodeList);
 SArray*  extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
                              int32_t type);
 
+void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);
 SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs);
 
 SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset);

source/libs/executor/inc/executorimpl.h

@@ -411,7 +411,7 @@ typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_READERHANDLE = 1,
   STREAM_SCAN_FROM_RES,
   STREAM_SCAN_FROM_UPDATERES,
-  STREAM_SCAN_FROM_DELETERES,
+  STREAM_SCAN_FROM_DELETE_DATA,
   STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
   STREAM_SCAN_FROM_DATAREADER_RANGE,
 } EStreamScanMode;
@@ -794,6 +794,7 @@ typedef struct SStreamPartitionOperatorInfo {
   void*                 parIte;
   SSDataBlock*          pInputDataBlock;
   int32_t               tsColIndex;
+  SSDataBlock*          pDelRes;
 } SStreamPartitionOperatorInfo;
 
 typedef struct STimeSliceOperatorInfo {
@@ -1108,6 +1109,13 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
 bool    groupbyTbname(SNodeList* pGroupList);
 int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
 void*   destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+                                   SGroupResInfo* pGroupResInfo);
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+                           int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+                           SExecTaskInfo* pTaskInfo);
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult);
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
 
 #ifdef __cplusplus
 }

source/libs/executor/src/executil.c

@@ -475,7 +475,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
     qError("failed to create result, reason:%s", tstrerror(code));
-    terrno = code;
     goto end;
   }
 
@@ -1012,6 +1011,100 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa
   return pCol;
 }
 
+void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) {
+  pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode));
+  pExp->pExpr->_function.num = 1;
+  pExp->pExpr->_function.functionId = -1;
+
+  int32_t type = nodeType(pTargetNode->pExpr);
+  // it is a project query, or group by column
+  if (type == QUERY_NODE_COLUMN) {
+    pExp->pExpr->nodeType = QUERY_NODE_COLUMN;
+    SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr;
+
+    pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+    pExp->base.numOfParams = 1;
+
+    SDataType* pType = &pColNode->node.resType;
+    pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+                                           pType->precision, pColNode->colName);
+    pExp->base.pParam[0].pCol =
+        createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType);
+    pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
+  } else if (type == QUERY_NODE_VALUE) {
+    pExp->pExpr->nodeType = QUERY_NODE_VALUE;
+    SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
+
+    pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+    pExp->base.numOfParams = 1;
+
+    SDataType* pType = &pValNode->node.resType;
+    pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+                                           pType->precision, pValNode->node.aliasName);
+    pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
+    nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
+  } else if (type == QUERY_NODE_FUNCTION) {
+    pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
+    SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
+
+    SDataType* pType = &pFuncNode->node.resType;
+    pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+                                           pType->precision, pFuncNode->node.aliasName);
+
+    pExp->pExpr->_function.functionId = pFuncNode->funcId;
+    pExp->pExpr->_function.pFunctNode = pFuncNode;
+
+    strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
+            tListLen(pExp->pExpr->_function.functionName));
+#if 1
+    // todo refactor: add the parameter for tbname function
+    if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
+      pFuncNode->pParameterList = nodesMakeList();
+      ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
+      SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
+      if (NULL == res) {  // todo handle error
+      } else {
+        res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
+        nodesListAppend(pFuncNode->pParameterList, (SNode*)res);
+      }
+    }
+#endif
+
+    int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
+
+    pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam));
+    pExp->base.numOfParams = numOfParam;
+
+    for (int32_t j = 0; j < numOfParam; ++j) {
+      SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j);
+      if (p1->type == QUERY_NODE_COLUMN) {
+        SColumnNode* pcn = (SColumnNode*)p1;
+
+        pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN;
+        pExp->base.pParam[j].pCol =
+            createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType);
+      } else if (p1->type == QUERY_NODE_VALUE) {
+        SValueNode* pvn = (SValueNode*)p1;
+        pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE;
+        nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param);
+      }
+    }
+  } else if (type == QUERY_NODE_OPERATOR) {
+    pExp->pExpr->nodeType = QUERY_NODE_OPERATOR;
+    SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr;
+
+    pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
+    pExp->base.numOfParams = 1;
+
+    SDataType* pType = &pNode->node.resType;
+    pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
+                                           pType->precision, pNode->node.aliasName);
+    pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
+  } else {
+    ASSERT(0);
+  }
+}
+
 SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) {
   int32_t numOfFuncs = LIST_LENGTH(pNodeList);
   int32_t numOfGroupKeys = 0;
@@ -1035,98 +1128,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
     }
 
     SExprInfo* pExp = &pExprs[i];
-
-    pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode));
-    pExp->pExpr->_function.num = 1;
-    pExp->pExpr->_function.functionId = -1;
-
-    int32_t type = nodeType(pTargetNode->pExpr);
-    // it is a project query, or group by column
-    if (type == QUERY_NODE_COLUMN) {
-      pExp->pExpr->nodeType = QUERY_NODE_COLUMN;
-      SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr;
-
-      pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
-      pExp->base.numOfParams = 1;
-
-      SDataType* pType = &pColNode->node.resType;
-      pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
-                                             pType->precision, pColNode->colName);
-      pExp->base.pParam[0].pCol =
-          createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType);
-      pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN;
-    } else if (type == QUERY_NODE_VALUE) {
-      pExp->pExpr->nodeType = QUERY_NODE_VALUE;
-      SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr;
-
-      pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
-      pExp->base.numOfParams = 1;
-
-      SDataType* pType = &pValNode->node.resType;
-      pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
-                                             pType->precision, pValNode->node.aliasName);
-      pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE;
-      nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param);
-    } else if (type == QUERY_NODE_FUNCTION) {
-      pExp->pExpr->nodeType = QUERY_NODE_FUNCTION;
-      SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr;
-
-      SDataType* pType = &pFuncNode->node.resType;
-      pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
-                                             pType->precision, pFuncNode->node.aliasName);
-
-      pExp->pExpr->_function.functionId = pFuncNode->funcId;
-      pExp->pExpr->_function.pFunctNode = pFuncNode;
-
-      strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName,
-              tListLen(pExp->pExpr->_function.functionName));
-#if 1
-      // todo refactor: add the parameter for tbname function
-      if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) {
-        pFuncNode->pParameterList = nodesMakeList();
-        ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0);
-        SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
-        if (NULL == res) {  // todo handle error
-        } else {
-          res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
-          nodesListAppend(pFuncNode->pParameterList, (SNode*)res);
-        }
-      }
-#endif
-
-      int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList);
-
-      pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam));
-      pExp->base.numOfParams = numOfParam;
-
-      for (int32_t j = 0; j < numOfParam; ++j) {
-        SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j);
-        if (p1->type == QUERY_NODE_COLUMN) {
-          SColumnNode* pcn = (SColumnNode*)p1;
-
-          pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN;
-          pExp->base.pParam[j].pCol =
-              createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType);
-        } else if (p1->type == QUERY_NODE_VALUE) {
-          SValueNode* pvn = (SValueNode*)p1;
-          pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE;
-          nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param);
-        }
-      }
-    } else if (type == QUERY_NODE_OPERATOR) {
-      pExp->pExpr->nodeType = QUERY_NODE_OPERATOR;
-      SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr;
-
-      pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam));
-      pExp->base.numOfParams = 1;
-
-      SDataType* pType = &pNode->node.resType;
-      pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale,
-                                             pType->precision, pNode->node.aliasName);
-      pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr;
-    } else {
-      ASSERT(0);
-    }
+    createExprFromTargetNode(pExp, pTargetNode);
   }
 
   return pExprs;

source/libs/executor/src/executorimpl.c

@@ -3439,6 +3439,44 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t
   }
 }
 
+static bool isWstartColumnExist(SFillOperatorInfo* pInfo) {
+  if (pInfo->numOfNotFillExpr == 0) {
+    return false;
+  }
+  for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) {
+    SExprInfo* exprInfo = pInfo->pNotFillExprInfo + i;
+    if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN &&
+        exprInfo->base.numOfParams == 1 &&
+        exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static int32_t createWStartTsAsNotFillExpr(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode) {
+  bool wstartExist = isWstartColumnExist(pInfo);
+  if (wstartExist == false) {
+    if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) {
+      qError("pWStartTs of fill physical node is not a target node");
+      return TSDB_CODE_QRY_SYS_ERROR;
+    }
+
+    SExprInfo* notFillExprs = taosMemoryRealloc(pInfo->pNotFillExprInfo, (pInfo->numOfNotFillExpr + 1) * sizeof(SExprInfo));
+    if (notFillExprs == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    createExprFromTargetNode(notFillExprs + pInfo->numOfNotFillExpr, (STargetNode*)pPhyFillNode->pWStartTs);
+
+    ++pInfo->numOfNotFillExpr;
+    pInfo->pNotFillExprInfo = notFillExprs;
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
                                       SExecTaskInfo* pTaskInfo) {
   SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
@@ -3450,7 +3488,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
   SExprInfo*   pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
   pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr);
-
+  int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
   SInterval* pInterval =
       QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
           ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval
@@ -3471,7 +3512,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
                                                  &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
 
-  int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr,
+  code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr,
                               (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,
                               pTaskInfo->id.str, pInterval, type, order);
   if (code != TSDB_CODE_SUCCESS) {
@@ -3938,7 +3979,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
                                        pTaskInfo, isStream);
 
-  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { 
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
     pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
     SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
@@ -4410,3 +4451,108 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
 
   return code;
 }
+
+int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx,
+                     int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup,
+                     SExecTaskInfo* pTaskInfo) {
+  SWinKey key = {
+      .ts = win->skey,
+      .groupId = tableGroupId,
+  };
+  char*   value = NULL;
+  int32_t size = pAggSup->resultRowSize;
+  if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+  *pResult = (SResultRow*)value;
+  ASSERT(*pResult);
+  // set time window for current result
+  (*pResult)->win = (*win);
+  setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) {
+  streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) {
+  streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup,
+                                   SGroupResInfo* pGroupResInfo) {
+  SExprInfo*      pExprInfo = pSup->pExprInfo;
+  int32_t         numOfExprs = pSup->numOfExprs;
+  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
+  SqlFunctionCtx* pCtx = pSup->pCtx;
+
+  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
+
+  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
+    SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
+    int32_t     size = 0;
+    void*       pVal = NULL;
+    SWinKey     key = {
+            .ts = *(TSKEY*)pPos->key,
+            .groupId = pPos->groupId,
+    };
+    int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size);
+    ASSERT(code == 0);
+    SResultRow* pRow = (SResultRow*)pVal;
+    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
+    // no results, continue to check the next one
+    if (pRow->numOfRows == 0) {
+      pGroupResInfo->index += 1;
+      releaseOutputBuf(pTaskInfo, &key, pRow);
+      continue;
+    }
+
+    if (pBlock->info.groupId == 0) {
+      pBlock->info.groupId = pPos->groupId;
+    } else {
+      // current value belongs to different group, it can't be packed into one datablock
+      if (pBlock->info.groupId != pPos->groupId) {
+        releaseOutputBuf(pTaskInfo, &key, pRow);
+        break;
+      }
+    }
+
+    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
+      ASSERT(pBlock->info.rows > 0);
+      releaseOutputBuf(pTaskInfo, &key, pRow);
+      break;
+    }
+
+    pGroupResInfo->index += 1;
+
+    for (int32_t j = 0; j < numOfExprs; ++j) {
+      int32_t slotId = pExprInfo[j].base.resSchema.slotId;
+
+      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);
+      if (pCtx[j].fpSet.finalize) {
+        int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+        if (TAOS_FAILED(code)) {
+          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
+          T_LONG_JMP(pTaskInfo->env, code);
+        }
+      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
+        // do nothing, todo refactor
+      } else {
+        // expand the result into multiple rows. E.g., _wstart, top(k, 20)
+        // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows.
+        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
+        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);
+        for (int32_t k = 0; k < pRow->numOfRows; ++k) {
+          colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);
+        }
+      }
+    }
+    releaseOutputBuf(pTaskInfo, &key, pRow);
+    pBlock->info.rows += pRow->numOfRows;
+  }
+  blockDataUpdateTsWindow(pBlock, 0);
+  return TSDB_CODE_SUCCESS;
+}

source/libs/function/src/builtins.c

@@ -2175,6 +2175,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .type = FUNCTION_TYPE_AVG_PARTIAL,
     .classification = FUNC_MGT_AGG_FUNC,
     .translateFunc = translateAvgPartial,
+    .dataRequiredFunc = statisDataRequired,
     .getEnvFunc   = getAvgFuncEnv,
     .initFunc     = avgFunctionSetup,
     .processFunc  = avgFunction,

source/libs/function/src/builtinsimpl.c

@@ -4711,7 +4711,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) {
       colDataAppendNULL(pOutput, i);
       // handle selectivity
       if (pCtx->subsidiaries.num > 0) {
-        appendSelectivityValue(pCtx, i, i);
+        appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
       }
       continue;
     }
@@ -4724,11 +4724,11 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) {
     } else {
       pInfo->count = 0;
     }
-    colDataAppend(pOutput, i, (char*)&output, false);
+    colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false);
 
     // handle selectivity
     if (pCtx->subsidiaries.num > 0) {
-      appendSelectivityValue(pCtx, i, i);
+      appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
     }
   }
 

source/libs/parser/src/parTranslater.c

@@ -3770,7 +3770,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
         checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_SST_TRIGGER, TSDB_MAX_SST_TRIGGER);
+    code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
@@ -6604,7 +6604,17 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
   pReq->colId = pSchema->colId;
 
   SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
-  if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
+
+  if (QUERY_NODE_VALUE != pStmt->pVal->node.type) {
+    SValueNode* pVal = NULL;
+    pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal);
+    if (pCxt->errCode) {
+      return pCxt->errCode;
+    }
+
+    nodesDestroyNode((SNode*)pStmt->pVal);
+    pStmt->pVal = pVal;
+  } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) {
     return pCxt->errCode;
   }
 

tests/script/tsim/column/table.sim

@@ -159,6 +159,7 @@ if $data01 != 10 then
   return -1
 endi
 if $data02 != 4.500000000 then
+  print expect 4.500000000, actual: $data02
   return -1
 endi
 if $data03 != 4.500000000 then