Merge branch 'main' into lihui3.0/test

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 0681d8b
+        GIT_TAG 4378702
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

include/common/tcommon.h

@@ -341,6 +341,8 @@ typedef struct {
     float       f;
   };
   size_t length;
+  bool keyEscaped;
+  bool valueEscaped;
 } SSmlKv;
 
 #define QUERY_ASC_FORWARD_STEP  1

include/common/tmsg.h

@@ -689,6 +689,7 @@ typedef struct {
 
 int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
 int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq);
+void    tFreeSAlterUserReq(SAlterUserReq* pReq);
 
 typedef struct {
   char user[TSDB_USER_LEN];

include/libs/catalog/catalog.h

@@ -82,6 +82,7 @@ typedef struct SCatalogReq {
   SArray* pUser;          // element is SUserAuthInfo
   SArray* pTableIndex;    // element is SNAME
   SArray* pTableCfg;      // element is SNAME
+  SArray* pTableTag;      // element is SNAME
   bool    qNodeRequired;  // valid qnode
   bool    dNodeRequired;  // valid dnode
   bool    svrVerRequired;
@@ -105,6 +106,7 @@ typedef struct SMetaData {
   SArray*   pUser;        // pRes = SUserAuthRes*
   SArray*   pQnodeList;   // pRes = SArray<SQueryNodeLoad>*
   SArray*   pTableCfg;    // pRes = STableCfg*
+  SArray*   pTableTag;    // pRes = SArray<STagVal>*
   SArray*   pDnodeList;   // pRes = SArray<SEpSet>*
   SMetaRes* pSvrVer;      // pRes = char*
 } SMetaData;
@@ -122,8 +124,8 @@ typedef struct SSTableVersion {
   char     stbName[TSDB_TABLE_NAME_LEN];
   uint64_t dbId;
   uint64_t suid;
-  int16_t  sversion;
-  int16_t  tversion;
+  int32_t  sversion;
+  int32_t  tversion;
   int32_t  smaVer;
 } SSTableVersion;
 
@@ -312,6 +314,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char*
 
 int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
 
+int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
+
 int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg);
 
 int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp);

include/libs/nodes/querynodes.h

@@ -379,6 +379,8 @@ typedef struct SVnodeModifyOpStmt {
   SName                 usingTableName;
   const char*           pBoundCols;
   struct STableMeta*    pTableMeta;
+  SNode*                pTagCond;
+  SArray*               pTableTag;
   SHashObj*             pVgroupsHashObj;
   SHashObj*             pTableBlockHashObj;  // SHashObj<tuid, STableDataCxt*>
   SHashObj*             pSubTableHashObj;

include/libs/qcom/query.h

@@ -116,8 +116,8 @@ typedef struct STableMeta {
 
   // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
   // info
-  int16_t       sversion;
-  int16_t       tversion;
+  int32_t       sversion;
+  int32_t       tversion;
   STableComInfo tableInfo;
   SSchema       schema[];
 } STableMeta;

include/libs/stream/tstream.h

@@ -273,6 +273,7 @@ typedef struct SStreamId {
 typedef struct SCheckpointInfo {
   int64_t id;
   int64_t version;   // offset in WAL
+  int64_t currentVer;// current offset in WAL, not serialize it
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
@@ -537,6 +538,7 @@ void    streamTaskInputFail(SStreamTask* pTask);
 int32_t streamTryExec(SStreamTask* pTask);
 int32_t streamSchedExec(SStreamTask* pTask);
 int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
+bool    streamTaskShouldStop(const SStreamStatus* pStatus);
 
 int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);
 

include/util/tdef.h

@@ -368,11 +368,11 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_STT_TRIGGER            1
 #define TSDB_MAX_STT_TRIGGER            16
 #define TSDB_DEFAULT_SST_TRIGGER        1
-#define TSDB_MIN_HASH_PREFIX            0
-#define TSDB_MAX_HASH_PREFIX            128
+#define TSDB_MIN_HASH_PREFIX            (2 - TSDB_TABLE_NAME_LEN)
+#define TSDB_MAX_HASH_PREFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_PREFIX        0
-#define TSDB_MIN_HASH_SUFFIX            0
-#define TSDB_MAX_HASH_SUFFIX            128
+#define TSDB_MIN_HASH_SUFFIX            (2 - TSDB_TABLE_NAME_LEN)
+#define TSDB_MAX_HASH_SUFFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_SUFFIX        0
 
 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

include/util/tutil.h

@@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
 
 static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
                                              int32_t suffix) {
-  if (prefix == 0 && suffix == 0) {
+  if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
     return MurmurHash3_32(tbname, tblen);
+  } else if (prefix > 0 || suffix > 0) {
+    return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
   } else {
-    if (tblen <= (prefix + suffix)) {
-      return MurmurHash3_32(tbname, tblen);
-    } else {
-      return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
+    char tbName[TSDB_TABLE_FNAME_LEN];
+    int32_t offset = 0;
+    if (prefix < 0) {
+      offset = -1 * prefix;
+      strncpy(tbName, tbname, offset);
     }
+    if (suffix < 0) {
+      strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
+      offset += -1 *suffix;
+    }
+    return MurmurHash3_32(tbName, offset);
   }
 }
 
@@ -98,6 +106,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
     goto LABEL;                            \
   }
 
+#define VND_CHECK_CODE(CODE, LINO, LABEL) TSDB_CHECK_CODE(CODE, LINO, LABEL)
+
 #ifdef __cplusplus
 }
 #endif

packaging/deb/DEBIAN/preinst

@@ -26,6 +26,38 @@ if pidof taosd &> /dev/null; then
     sleep 1
 fi
 
+# Stop adapter service if running
+if pidof taosadapter &> /dev/null; then
+    if pidof systemd &> /dev/null; then
+        ${csudo}systemctl stop taosadapter || :
+    elif $(which service  &> /dev/null); then
+        ${csudo}service taosadapter stop || :
+    else
+        pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}')
+        if [ -n "$pid" ]; then
+           ${csudo}kill -9 $pid   || :
+        fi
+    fi
+    echo "Stop taosadapter service success!"
+    sleep 1
+fi
+
+# Stop keeper service if running
+if pidof taoskeeper &> /dev/null; then
+    if pidof systemd &> /dev/null; then
+        ${csudo}systemctl stop taoskeeper || :
+    elif $(which service  &> /dev/null); then
+        ${csudo}service taoskeeper stop || :
+    else
+        pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
+        if [ -n "$pid" ]; then
+           ${csudo}kill -9 $pid   || :
+        fi
+    fi
+    echo "Stop taoskeeper service success!"
+    sleep 1
+fi
+
 # if taos.cfg already softlink, remove it
 cfg_install_dir="/etc/taos"
 install_main_dir="/usr/local/taos"
@@ -41,6 +73,11 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then
     ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || :
 fi
 
+if [ -f "${install_main_dir}/taoskeeper.toml" ]; then
+    ${csudo}rm -f ${install_main_dir}/cfg/taoskeeper.toml     || :
+fi
+
+
 # there can not libtaos.so*, otherwise ln -s  error
 ${csudo}rm -f ${install_main_dir}/driver/libtaos.*   || :
 [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so   || :

packaging/deb/DEBIAN/prerm

@@ -32,6 +32,7 @@ else
     ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
     ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
+    ${csudo}rm -f ${bin_link_dir}/taoskeeper   || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :
     ${csudo}rm -f ${inc_link_dir}/taos.h     || :
     ${csudo}rm -f ${inc_link_dir}/taosdef.h     || :

packaging/deb/makedeb.sh

@@ -44,8 +44,31 @@ mkdir -p ${pkg_dir}${install_home_path}/include
 #mkdir -p ${pkg_dir}${install_home_path}/init.d
 mkdir -p ${pkg_dir}${install_home_path}/script
 
+# download taoskeeper and build
+if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
+  arch=amd64
+elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
+  arch=386
+elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
+  arch=arm
+elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
+  arch=arm64
+else
+  arch=$cpuType
+fi
+
+echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
+echo "$top_dir=${top_dir}"
+taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
+echo "taoskeeper_binary: ${taoskeeper_binary}"
+
+# copy config files
+cp $(dirname ${taoskeeper_binary})/config/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg
+cp $(dirname ${taoskeeper_binary})/taoskeeper.service ${pkg_dir}${install_home_path}/cfg
+
 cp ${compile_dir}/../packaging/cfg/taos.cfg         ${pkg_dir}${install_home_path}/cfg
 cp ${compile_dir}/../packaging/cfg/taosd.service    ${pkg_dir}${install_home_path}/cfg
+
 if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.toml		${pkg_dir}${install_home_path}/cfg || :
 fi
@@ -53,6 +76,7 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then
     cp ${compile_dir}/test/cfg/taosadapter.service	${pkg_dir}${install_home_path}/cfg || :
 fi
 
+cp ${taoskeeper_binary}                      ${pkg_dir}${install_home_path}/bin
 #cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_path}/init.d
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
@@ -143,6 +167,7 @@ else
   exit 1
 fi
 
+rm -rf ${pkg_dir}/build-taoskeeper
 # make deb package
 dpkg -b ${pkg_dir} $debname
 echo "make deb package success!"
@@ -150,4 +175,5 @@ echo "make deb package success!"
 cp ${pkg_dir}/*.deb ${output_dir}
 
 # clean temp dir
+
 rm -rf ${pkg_dir}

packaging/rpm/makerpm.sh

@@ -35,14 +35,16 @@ function cp_rpm_package() {
     local cur_dir
     cd $1
     cur_dir=$(pwd)
-
+    echo "cp_rpm_package cd: ${cur_dir}"
     for dirlist in "$(ls ${cur_dir})"; do
         if test -d ${dirlist}; then
             cd ${dirlist}
+            echo 'cp_rpm_package ${cur_dir}/${dirlist}'
             cp_rpm_package ${cur_dir}/${dirlist}
             cd ..
         fi
         if test -e ${dirlist}; then
+            echo "${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm"
             cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm
         fi
     done
@@ -54,6 +56,25 @@ fi
 ${csudo}mkdir -p ${pkg_dir}
 cd ${pkg_dir}
 
+# download taoskeeper and build
+if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then
+  arch=amd64
+elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then
+  arch=386
+elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then
+  arch=arm
+elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then
+  arch=arm64
+else
+  arch=$cpuType
+fi
+
+cd ${top_dir}
+echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper"
+taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper`
+echo "taoskeeper_binary: ${taoskeeper_binary}"
+cd ${package_dir}
+
 ${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS
 
 ${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file}
@@ -85,3 +106,4 @@ mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname}
 
 cd ..
 ${csudo}rm -rf ${pkg_dir}
+rm -rf ${top_dir}/build-taoskeeper
\ No newline at end of file

packaging/rpm/tdengine.spec

@@ -3,6 +3,7 @@
 %define cfg_install_dir  /etc/taos
 %define __strip /bin/true
 %global __python /usr/bin/python3
+%global _build_id_links none
 
 Name:		tdengine
 Version:	%{_version}
@@ -62,6 +63,15 @@ fi
 if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then
     cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg
 fi
+
+if [ -f %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml ]; then
+    cp %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml %{buildroot}%{homepath}/cfg ||:
+fi
+
+if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then
+    cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||:
+fi
+
 #cp %{_compiledir}/../packaging/rpm/taosd            %{buildroot}%{homepath}/init.d
 cp %{_compiledir}/../packaging/tools/post.sh        %{buildroot}%{homepath}/script
 cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/script
@@ -73,8 +83,12 @@ cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/udfd                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosBenchmark           %{buildroot}%{homepath}/bin
 
+if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then
+    cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin
+fi
+
 if [ -f %{_compiledir}/build/bin/taosadapter ]; then
-    cp %{_compiledir}/build/bin/taosadapter                    %{buildroot}%{homepath}/bin ||:
+    cp %{_compiledir}/build/bin/taosadapter                    %{buildroot}%{homepath}/bin
 fi
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile}            %{buildroot}%{homepath}/driver ||:
@@ -119,7 +133,9 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then
         cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig
     fi
 fi
-
+ls -al %{buildroot}%{homepath}/bin
+tree -L 5
+echo "==============================copying files done"
 #Scripts executed before installation
 %pre
 if [ -f /var/lib/taos/dnode/dnodeCfg.json ]; then
@@ -196,6 +212,7 @@ if [ $1 -eq 0 ];then
     ${csudo}rm -f ${bin_link_dir}/taosd      || :
     ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
+    ${csudo}rm -f ${bin_link_dir}/taoskeeper       || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :
     ${csudo}rm -f ${inc_link_dir}/taos.h     || :
     ${csudo}rm -f ${inc_link_dir}/taosdef.h     || :

packaging/tools/makeclient.sh

@@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then
       mkdir -p ${install_dir}/connector
       if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
           if [ "$osType" != "Darwin" ]; then
-              [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
+              jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
+              [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
           fi
           git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
           rm -rf ${install_dir}/connector/go/.git ||:

packaging/tools/makepkg.sh

@@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
     connector_dir="${code_dir}/connector"
     mkdir -p ${install_dir}/connector
     if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
-        [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
+        tmp_pwd=`pwd`
+    	  cd ${install_dir}/connector
+    	  if [ ! -d taos-connector-jdbc ];then
+          	git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||:
+    	  fi
+    	  cd taos-connector-jdbc
+    	  mvn clean package -Dmaven.test.skip=true
+    	  echo  ${build_dir}/lib/
+    	  cp target/*.jar  ${build_dir}/lib/
+    	  cd ${install_dir}/connector
+    	  rm -rf taos-connector-jdbc
+    	  cd ${tmp_pwd}
+   	    jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l)
+        [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || :
         git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go
         rm -rf ${install_dir}/connector/go/.git ||:
 

packaging/tools/post.sh

@@ -436,7 +436,7 @@ function local_fqdn_check() {
 
 function install_taosadapter_config() {
     if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then
-        [ ! -d %{cfg_install_dir} ] &&
+        [ ! -d ${cfg_install_dir} ] &&
             ${csudo}${csudo}mkdir -p ${cfg_install_dir}
         [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir}
         [ -f ${cfg_install_dir}/taosadapter.toml ] &&
@@ -451,19 +451,26 @@ function install_taosadapter_config() {
 }
 
 function install_taoskeeper_config() {
-    if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then
-        [ ! -d %{cfg_install_dir} ] &&
-            ${csudo}${csudo}mkdir -p ${cfg_install_dir}
-        [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir}
-        [ -f ${cfg_install_dir}/keeper.toml ] &&
-            ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml
+    # if new environment without taoskeeper
+    if [[ ! -f "${cfg_install_dir}/keeper.toml" ]] && [[ ! -f "${cfg_install_dir}/taoskeeper.toml" ]]; then
+        [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir}
+        [ -f ${cfg_dir}/taoskeeper.toml ] && ${csudo}cp ${cfg_dir}/taoskeeper.toml ${cfg_install_dir}
+        [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
+            ${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml
+    fi
+    # if old machine with taoskeeper.toml file
+    if [ -f ${cfg_install_dir}/taoskeeper.toml ]; then
+        ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
     fi
 
-    [ -f ${cfg_dir}/keeper.toml ] &&
-        ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new
+    if [ -f ${cfg_install_dir}/keeper.toml ]; then
+        echo "The file keeper.toml will be renamed to taoskeeper.toml"
+        ${csudo}mv ${cfg_install_dir}/keeper.toml ${cfg_install_dir}/taoskeeper.toml
+        ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new
+    fi
 
-    [ -f ${cfg_install_dir}/keeper.toml ] &&
-        ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir}
+    [ -f ${cfg_install_dir}/taoskeeper.toml ] &&
+        ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir}
 }
 
 function install_config() {
@@ -655,6 +662,15 @@ function install_taosadapter_service() {
     fi
 }
 
+function install_taoskeeper_service() {
+    if ((${service_mod}==0)); then
+        [ -f ${script_dir}/../cfg/taoskeeper.service ] &&\
+            ${csudo}cp ${script_dir}/../cfg/taoskeeper.service \
+            ${service_config_dir}/ || :
+        ${csudo}systemctl daemon-reload
+    fi
+}
+
 function install_service() {
   log_print "start install service"
   if [ "$osType" != "Darwin" ]; then
@@ -732,6 +748,7 @@ function install_TDengine() {
     install_taosadapter_config
     install_taoskeeper_config
     install_taosadapter_service
+    install_taoskeeper_service
     install_service
     install_app
 

packaging/tools/preun.sh

@@ -17,7 +17,7 @@ cfg_link_dir="/usr/local/taos/cfg"
 
 service_config_dir="/etc/systemd/system"
 taos_service_name="taosd"
-
+taoskeeper_service_name="taoskeeper"
 csudo=""
 if command -v sudo > /dev/null; then
     csudo="sudo "
@@ -57,6 +57,13 @@ function kill_taosd() {
   fi
 }
 
+function kill_taoskeeper() {
+  pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
+  if [ -n "$pid" ]; then
+    ${csudo}kill -9 $pid   || :
+  fi
+}
+
 function clean_service_on_systemd() {
     taosadapter_service_config="${service_config_dir}/taosadapter.service"
     if systemctl is-active --quiet taosadapter; then
@@ -76,6 +83,12 @@ function clean_service_on_systemd() {
 
     [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config}
 
+    taoskeeper_service_config="${service_config_dir}/${taoskeeper_service_name}.service"
+    if systemctl is-active --quiet ${taoskeeper_service_name}; then
+        echo "TDengine taoskeeper is running, stopping it..."
+        ${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null
+    fi
+    [ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config}
 }
 
 function clean_service_on_sysvinit() {
@@ -111,6 +124,7 @@ function clean_service() {
         # must manual stop taosd
         kill_taosadapter
         kill_taosd
+        kill_taoskeeper
     fi
 }
 
@@ -124,6 +138,7 @@ ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
 ${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
 ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
 ${csudo}rm -f ${bin_link_dir}/set_core   || :
+${csudo}rm -f ${bin_link_dir}/taoskeeper  || :
 ${csudo}rm -f ${cfg_link_dir}/*.new      || :
 ${csudo}rm -f ${inc_link_dir}/taos.h     || :
 ${csudo}rm -f ${inc_link_dir}/taosdef.h  || :

source/client/inc/clientSml.h

@@ -70,7 +70,7 @@ extern "C" {
 #define VALUE_LEN 6
 
 #define OTD_JSON_FIELDS_NUM     4
-#define MAX_RETRY_TIMES 100
+#define MAX_RETRY_TIMES 10
 typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
 
 typedef enum {
@@ -107,6 +107,7 @@ typedef struct {
   int32_t colsLen;
   int32_t timestampLen;
 
+  bool    measureEscaped;
   SArray *colArray;
 } SSmlLineInfo;
 
@@ -206,6 +207,19 @@ typedef struct {
 
 #define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
 
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
+
 extern int64_t smlFactorNS[3];
 extern int64_t smlFactorS[3];
 
@@ -237,6 +251,7 @@ uint8_t           smlGetTimestampLen(int64_t num);
 void              clearColValArray(SArray* pCols);
 void              smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag);
 
+void freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);

source/client/src/clientMain.c

@@ -108,7 +108,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
   if (pass == NULL) {
     pass = TSDB_DEFAULT_PASS;
   }
-  
+
   STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
   if (pObj) {
     int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
@@ -359,11 +359,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
       case TSDB_DATA_TYPE_NCHAR: {
         int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
         if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
-          if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){
+          if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) {
             tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
           }
         } else {
-          if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){
+          if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) {
             tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes);
           }
         }
@@ -705,16 +705,16 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
 
   int code = TSDB_CODE_SUCCESS;
   taosThreadMutexLock(&pTscObj->mutex);
-  if(database == NULL || len <= 0){
-    if(required != NULL) *required = strlen(pTscObj->db) + 1;
+  if (database == NULL || len <= 0) {
+    if (required != NULL) *required = strlen(pTscObj->db) + 1;
     terrno = TSDB_CODE_INVALID_PARA;
     code = -1;
-  }else if(len < strlen(pTscObj->db) + 1){
+  } else if (len < strlen(pTscObj->db) + 1) {
     tstrncpy(database, pTscObj->db, len);
-    if(required) *required = strlen(pTscObj->db) + 1;
+    if (required) *required = strlen(pTscObj->db) + 1;
     terrno = TSDB_CODE_INVALID_PARA;
     code = -1;
-  }else{
+  } else {
     strcpy(database, pTscObj->db);
     code = 0;
   }
@@ -741,6 +741,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
   taosArrayDestroy(pCatalogReq->pUser);
   taosArrayDestroy(pCatalogReq->pTableIndex);
   taosArrayDestroy(pCatalogReq->pTableCfg);
+  taosArrayDestroy(pCatalogReq->pTableTag);
   taosMemoryFree(pCatalogReq);
 }
 
@@ -975,8 +976,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 
   if (TSDB_CODE_SUCCESS == code) {
     pRequest->stmtType = pRequest->pQuery->pRoot->type;
-    phaseAsyncQuery(pWrapper);
-  } else {
+    code = phaseAsyncQuery(pWrapper);
+  }
+
+  if (TSDB_CODE_SUCCESS != code) {
     tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code),
              pRequest->requestId);
     destorySqlCallbackWrapper(pWrapper);
@@ -1042,11 +1045,11 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
 }
 
 void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
-  if(ASSERT(res != NULL && fp != NULL)){
+  if (ASSERT(res != NULL && fp != NULL)) {
     tscError("taos_fetch_rows_a invalid paras");
     return;
   }
-  if(ASSERT(TD_RES_QUERY(res))){
+  if (ASSERT(TD_RES_QUERY(res))) {
     tscError("taos_fetch_rows_a res is NULL");
     return;
   }
@@ -1092,11 +1095,11 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
 }
 
 void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
-  if(ASSERT(res != NULL && fp != NULL)){
+  if (ASSERT(res != NULL && fp != NULL)) {
     tscError("taos_fetch_rows_a invalid paras");
     return;
   }
-  if(ASSERT(TD_RES_QUERY(res))){
+  if (ASSERT(TD_RES_QUERY(res))) {
     tscError("taos_fetch_rows_a res is NULL");
     return;
   }
@@ -1111,11 +1114,11 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
 }
 
 const void *taos_get_raw_block(TAOS_RES *res) {
-  if(ASSERT(res != NULL)){
+  if (ASSERT(res != NULL)) {
     tscError("taos_fetch_rows_a invalid paras");
     return NULL;
   }
-  if(ASSERT(TD_RES_QUERY(res))){
+  if (ASSERT(TD_RES_QUERY(res))) {
     tscError("taos_fetch_rows_a res is NULL");
     return NULL;
   }
@@ -1273,7 +1276,6 @@ _return:
   return code;
 }
 
-
 int taos_load_table_info(TAOS *taos, const char *tableNameList) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;

source/client/src/clientSml.c

@@ -535,7 +535,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
   if (index) {
     if (colField[*index].type != kv->type) {
       uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      return TSDB_CODE_SML_INVALID_DATA;
     }
 
     if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR &&
@@ -765,8 +765,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
 
     size_t superTableLen = 0;
     void  *superTable = taosHashGetKey(tmp, &superTableLen);
+    char* measure = taosMemoryMalloc(superTableLen);
+    memcpy(measure, superTable, superTableLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, superTableLen);
     memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
-    memcpy(pName.tname, superTable, superTableLen);
+    memcpy(pName.tname, measure, superTableLen);
+    taosMemoryFree(measure);
 
     code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
 
@@ -1049,7 +1053,7 @@ void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
   //  }
   //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
-  taosArrayDestroy(tag->tags);
+  taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
 }
 
@@ -1063,6 +1067,12 @@ void clearColValArray(SArray *pCols) {
   }
 }
 
+void freeSSmlKv(void* data){
+  SSmlKv *kv = (SSmlKv*)data;
+  if(kv->keyEscaped) taosMemoryFree((void*)(kv->key));
+  if(kv->valueEscaped) taosMemoryFree((void*)(kv->value));
+}
+
 void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
@@ -1098,11 +1108,11 @@ void smlDestroyInfo(SSmlHandle *info) {
   }
   taosArrayDestroy(info->valueJsonArray);
 
-  taosArrayDestroy(info->preLineTagKV);
+  taosArrayDestroyEx(info->preLineTagKV, freeSSmlKv);
 
   if (!info->dataFormat) {
     for (int i = 0; i < info->lineNum; i++) {
-      taosArrayDestroy(info->lines[i].colArray);
+      taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
@@ -1165,8 +1175,10 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
   }
   for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
+    terrno = 0;
     taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
     if (terrno == TSDB_CODE_DUP_KEY) {
+      taosHashCleanup(kvHash);
       return terrno;
     }
   }
@@ -1240,12 +1252,13 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
+      taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
+      terrno = 0;
       smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
       if (terrno == TSDB_CODE_DUP_KEY) {
         return terrno;
       }
       smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
-      taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
     }
   }
   uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
@@ -1304,9 +1317,15 @@ static int32_t smlInsertData(SSmlHandle *info) {
     uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
            tableData->uid, info->dataFormat);
 
+    int   measureLen = tableData->sTableNameLen;
+    char* measure = (char*)taosMemoryMalloc(tableData->sTableNameLen);
+    memcpy(measure, tableData->sTableName, tableData->sTableNameLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+
     code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
-                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
+                       (*pMeta)->tableMeta, tableData->childTableName, measure, measureLen,
                        info->ttl, info->msgBuf.buf, info->msgBuf.len);
+    taosMemoryFree(measure);
     if (code != TSDB_CODE_SUCCESS) {
       uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
       return code;
@@ -1420,14 +1439,14 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
 
     char cTmp = 0;  // for print tmp if is raw
     if (info->isRawLine) {
-      cTmp = tmp[len - 1];
-      tmp[len - 1] = '\0';
+      cTmp = tmp[len];
+      tmp[len] = '\0';
     }
 
     uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
            info->isRawLine, numLines, info->protocol, len, tmp);
     if (info->isRawLine) {
-      tmp[len - 1] = cTmp;
+      tmp[len] = cTmp;
     }
 
     if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
@@ -1449,6 +1468,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
       code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
     }
     if (code != TSDB_CODE_SUCCESS) {
+      tmp[len] = '\0';
       uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
       return code;
     }
@@ -1494,8 +1514,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
 
   do {
     code = smlModifyDBSchemas(info);
-    if (code == 0) break;
-    taosMsleep(500);
+    if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break;
+    taosMsleep(100);
     uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
   } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
 

source/client/src/clientSmlLine.c

@@ -21,32 +21,33 @@
 #include "clientSml.h"
 
 // comma ,
-// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH)
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
 // space
-// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH)
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
 // equal =
-// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH)
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
 // quote "
-// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH)
-#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
+//#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
 // SLASH
-// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH)
 
-#define IS_SLASH_LETTER(sql)                                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \
-                           *(sql) == SLASH))  //  (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) ||
-                                              //  IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql))
+#define IS_SLASH_LETTER_IN_FIELD_VALUE(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+#define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql)                                                         \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
 
-#define PROCESS_SLASH(key, keyLen)           \
+#define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {                        \
+    if (IS_SLASH_LETTER_IN_FIELD_VALUE(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
+
+#define PROCESS_SLASH_IN_TAG_FIELD_KEY(key, keyLen)           \
   for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER(key + i)) {          \
+    if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(key + i)) {          \
       MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      i--;                                   \
       keyLen--;                              \
     }                                        \
   }
@@ -151,7 +152,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
       SSmlSTableMeta *sMeta = NULL;
       if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
+        char* measure = currElement->measure;
+        int   measureLen = currElement->measureLen;
+        if(currElement->measureEscaped){
+          measure = (char*)taosMemoryMalloc(currElement->measureLen);
+          memcpy(measure, currElement->measure, currElement->measureLen);
+          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+        }
+        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+        if(currElement->measureEscaped){
+          taosMemoryFree(measure);
+        }
         if (pTableMeta == NULL) {
           info->dataFormat = false;
           info->reRun = true;
@@ -171,17 +182,18 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       info->maxTagKVs = (*tmp)->tags;
     }
   }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
     }
 
-    bool hasSlash = false;
     // parse key
     const char *key = *sql;
     size_t      keyLen = 0;
+    bool        keyEscaped = false;
+    size_t      keyLenEscaped = 0;
     while (*sql < sqlEnd) {
       if (unlikely(IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@@ -192,16 +204,14 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         (*sql)++;
         break;
       }
-      if (!hasSlash) {
-        hasSlash = (*(*sql) == SLASH);
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        keyLenEscaped++;
+        keyEscaped = true;
       }
       (*sql)++;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(key, keyLen)
-    }
 
-    if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
+    if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
@@ -209,7 +219,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     // parse value
     const char *value = *sql;
     size_t      valueLen = 0;
-    hasSlash = false;
+    bool        valueEscaped = false;
+    size_t      valueLenEscaped = 0;
     while (*sql < sqlEnd) {
       // parse value
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
@@ -219,8 +230,9 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         return TSDB_CODE_SML_INVALID_DATA;
       }
 
-      if (!hasSlash) {
-        hasSlash = (*(*sql) == SLASH);
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        valueLenEscaped++;
+        valueEscaped = true;
       }
 
       (*sql)++;
@@ -232,15 +244,24 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       return TSDB_CODE_SML_INVALID_DATA;
     }
 
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(value, valueLen)
-    }
-
-    if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
+    if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
       return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
+    if (keyEscaped){
+      char *tmp = (char*)taosMemoryMalloc(keyLen);
+      memcpy(tmp, key, keyLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
+      key = tmp;
+    }
+    if (valueEscaped){
+      char *tmp = (char*)taosMemoryMalloc(valueLen);
+      memcpy(tmp, value, valueLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
+      value = tmp;
+    }
+    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen, .keyEscaped = keyEscaped, .valueEscaped = valueEscaped};
+    taosArrayPush(preLineKV, &kv);
     if (info->dataFormat) {
       if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
         info->dataFormat = false;
@@ -266,7 +287,6 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         info->needModifySchema = true;
       }
     }
-    taosArrayPush(preLineKV, &kv);
 
     cnt++;
     if (IS_SPACE(*sql)) {
@@ -285,6 +305,11 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   tinfo->tags = taosArrayDup(preLineKV, NULL);
+  for(size_t i = 0; i < taosArrayGetSize(preLineKV); i++){
+    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
+    if(kv->keyEscaped)kv->key = NULL;
+    if(kv->valueEscaped)kv->value = NULL;
+  }
 
   smlSetCTableName(tinfo);
   tinfo->uid = info->uid++;
@@ -321,7 +346,17 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       SSmlSTableMeta **tmp =
           (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
       if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen);
+        char* measure = currElement->measure;
+        int   measureLen = currElement->measureLen;
+        if(currElement->measureEscaped){
+          measure = (char*)taosMemoryMalloc(currElement->measureLen);
+          memcpy(measure, currElement->measure, currElement->measureLen);
+          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+        }
+        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+        if(currElement->measureEscaped){
+          taosMemoryFree(measure);
+        }
         if (pTableMeta == NULL) {
           info->dataFormat = false;
           info->reRun = true;
@@ -352,10 +387,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       break;
     }
 
-    bool hasSlash = false;
     // parse key
     const char *key = *sql;
     size_t      keyLen = 0;
+    bool        keyEscaped = false;
+    size_t      keyLenEscaped = 0;
     while (*sql < sqlEnd) {
       if (unlikely(IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
@@ -366,16 +402,14 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         (*sql)++;
         break;
       }
-      if (!hasSlash) {
-        hasSlash = (*(*sql) == SLASH);
+      if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        keyLenEscaped++;
+        keyEscaped = true;
       }
       (*sql)++;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(key, keyLen)
-    }
 
-    if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
+    if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
       return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
     }
@@ -383,11 +417,13 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     // parse value
     const char *value = *sql;
     size_t      valueLen = 0;
-    hasSlash = false;
-    bool isInQuote = false;
+    bool        valueEscaped = false;
+    size_t      valueLenEscaped = 0;
+    bool        isInQuote = false;
+    const char *escapeChar = NULL;
     while (*sql < sqlEnd) {
       // parse value
-      if (unlikely(IS_QUOTE(*sql))) {
+      if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
         isInQuote = !isInQuote;
         (*sql)++;
         continue;
@@ -395,13 +431,12 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       if (!isInQuote) {
         if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
           break;
-        } else if (unlikely(IS_EQUAL(*sql))) {
-          smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-          return TSDB_CODE_SML_INVALID_DATA;
         }
       }
-      if (!hasSlash) {
-        hasSlash = (*(*sql) == SLASH);
+      if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
+        escapeChar = *sql;
+        valueEscaped = true;
+        valueLenEscaped++;
       }
 
       (*sql)++;
@@ -416,9 +451,6 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
       return TSDB_CODE_SML_INVALID_DATA;
     }
-    if (unlikely(hasSlash)) {
-      PROCESS_SLASH(value, valueLen)
-    }
 
     SSmlKv  kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen};
     int32_t ret = smlParseValue(&kv, &info->msgBuf);
@@ -427,11 +459,28 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       return ret;
     }
 
+    if (keyEscaped){
+      char *tmp = (char*)taosMemoryMalloc(kv.keyLen);
+      memcpy(tmp, key, kv.keyLen);
+      PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
+      kv.key = tmp;
+      kv.keyEscaped = keyEscaped;
+    }
+
+    if (valueEscaped){
+      char *tmp = (char*)taosMemoryMalloc(kv.length);
+      memcpy(tmp, kv.value, kv.length);
+      PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
+      kv.value = tmp;
+      kv.valueEscaped = valueEscaped;
+    }
+
     if (info->dataFormat) {
       // cnt begin 0, add ts so + 2
       if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       // bind data
@@ -440,22 +489,26 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         uDebug("smlBuildCol error, retry");
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       if (cnt >= taosArrayGetSize(info->masColKVs)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->masColKVs, cnt);
       if (kv.type != maxKV->type) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
       if (unlikely(!IS_SAME_KEY)) {
         info->dataFormat = false;
         info->reRun = true;
+        freeSSmlKv(&kv);
         return TSDB_CODE_SUCCESS;
       }
 
@@ -463,6 +516,7 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         maxKV->length = kv.length;
         info->needModifySchema = true;
       }
+      freeSSmlKv(&kv);
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -487,10 +541,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   elements->measure = sql;
 
   // parse measure
+  size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
-    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) {
-      MOVE_FORWARD_ONE(sql, sqlEnd - sql);
-      sqlEnd--;
+    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
+      elements->measureEscaped = true;
+      measureLenEscaped++;
+      sql++;
       continue;
     }
     if (unlikely(IS_COMMA(sql))) {
@@ -503,7 +559,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     sql++;
   }
   elements->measureLen = sql - elements->measure;
-  if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) {
+  if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen - measureLenEscaped))) {
     smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL);
     return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
   }
@@ -581,7 +637,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
                .keyLen = TS_LEN,
                .type = TSDB_DATA_TYPE_TIMESTAMP,
                .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
+               .keyEscaped = false,
+               .valueEscaped = false};
   if (info->dataFormat) {
     uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
     ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);

source/client/src/clientTmq.c

@@ -373,9 +373,6 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
   SArray* container = &list->container;
   if (src == NULL || src[0] == 0) return -1;
   char* topic = taosStrdup(src);
-  if (topic[0] != '`') {
-    strtolower(topic, src);
-  }
   if (taosArrayPush(container, &topic) == NULL) return -1;
   return 0;
 }
@@ -1243,9 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pParam);
 
   if (code != 0) {
-    tscWarn("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
-            vgId, epoch, tstrerror(code), requestId);
-
     if (pMsg->pData) taosMemoryFree(pMsg->pData);
     if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
 
@@ -1267,6 +1261,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
       taosWriteQitem(tmq->mqueue, pRspWrapper);
     } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) {  // poll data while insert
       taosMsleep(500);
+    } else{
+      tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, epoch %d, since %s, reqId:0x%" PRIx64, tmq->consumerId,
+               vgId, epoch, tstrerror(code), requestId);
     }
 
     goto CREATE_MSG_FAIL;
@@ -1667,11 +1664,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p
     return handleErrorBeforePoll(pVg, pTmq);
   }
 
-  sendInfo->msgInfo = (SDataBuf){
-      .pData = msg,
-      .len = msgSize,
-      .handle = NULL,
-  };
+  sendInfo->msgInfo = (SDataBuf){ .pData = msg, .len = msgSize, .handle = NULL };
 
   sendInfo->requestId = req.reqId;
   sendInfo->requestObjRefId = 0;

source/client/test/smlTest.cpp

@@ -50,8 +50,9 @@ TEST(testCase, smlParseInfluxString_Test) {
   int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(elements.measure, sql);
-  ASSERT_EQ(elements.measureLen, strlen(",st"));
-  ASSERT_EQ(elements.measureTagsLen, strlen(",st,t1=3,t2=4,t3=t3"));
+  ASSERT_EQ(elements.measureLen, strlen("\\,st"));
+  ASSERT_EQ(elements.measureEscaped, true);
+  ASSERT_EQ(elements.measureTagsLen, strlen("\\,st,t1=3,t2=4,t3=t3"));
 
   ASSERT_EQ(elements.tags, sql + elements.measureLen + 1);
   ASSERT_EQ(elements.tagsLen, strlen("t1=3,t2=4,t3=t3"));
@@ -204,7 +205,26 @@ TEST(testCase, smlParseCols_Error_Test) {
                         "st,t=1 c=-3.402823466e+39u64 1626006833639000000",
                         "st,t=1 c=-339u64 1626006833639000000",
                         "st,t=1 c=18446744073709551616u64 1626006833639000000",
-                        "st,t=1 c=1=2 1626006833639000000"};
+                        "st,t=1 c=1=2 1626006833639000000,",
+                        // escape error test
+                        // measure comma,space
+                        "s,t,t=1 c=1 1626006833639000000,",
+                        "s t,t=1 c=1 1626006833639000000,",
+                        //tag key comma,equal,space
+                        "st,t,t=1 c=2 1626006833639000000,",
+                        "st,t=t=1 c=2 1626006833639000000,",
+                        "st,t t=1 c=2 1626006833639000000,",
+                        //tag value comma,equal,space
+                        "st,tt=a,a c=2 1626006833639000000,",
+                        "st,t=t=a a c=2 1626006833639000000,",
+                        "st,t t=a=a c=2 1626006833639000000,",
+                        //field key comma,equal,space
+                        "st,tt=aa c,1=2 1626006833639000000,",
+                        "st,tt=aa c=1=2 1626006833639000000,",
+                        "st,tt=aa c 1=2 1626006833639000000,",
+                        //field value    double quote,slash
+                        "st,tt=aa c=\"a\"a\" 1626006833639000000,",
+  };
 
   SSmlHandle *info = smlBuildSmlInfo(NULL);
   info->protocol = TSDB_SML_LINE_PROTOCOL;
@@ -256,16 +276,18 @@ TEST(testCase, smlParseCols_Test) {
   ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
   ASSERT_EQ(kv->keyLen, 5);
   ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
-  ASSERT_EQ(kv->length, 17);
-  ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0);
+  ASSERT_EQ(kv->length, 18);
+  ASSERT_EQ(kv->keyEscaped, true);
+  ASSERT_EQ(kv->valueEscaped, false);
+  ASSERT_EQ(strncasecmp(kv->value, "pass\\,it ", 9), 0);
 
   // nchar
   kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
   ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
   ASSERT_EQ(kv->keyLen, 4);
   ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
-  ASSERT_EQ(kv->length, 8);
-  ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0);
+  ASSERT_EQ(kv->length, 9);
+  ASSERT_EQ(strncasecmp(kv->value, "ii\\=sd", 5), 0);
 
   // bool
   kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);

source/common/src/tglobal.c

@@ -198,7 +198,7 @@ int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointTickInterval = 1;
 int32_t tsTtlUnit = 86400;
-int32_t tsTtlPushInterval = 86400;
+int32_t tsTtlPushInterval = 3600;
 int32_t tsGrantHBInterval = 60;
 int32_t tsUptimeInterval = 300;    // seconds
 char    tsUdfdResFuncs[512] = "";  // udfd resident funcs that teardown when udfd exits

source/common/src/tmsg.c

@@ -1409,6 +1409,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
   return 0;
 }
 
+void tFreeSAlterUserReq(SAlterUserReq *pReq) { taosMemoryFreeClear(pReq->tagCond); }
+
 int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -1635,6 +1637,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
       int32_t ref = 0;
       if (tDecodeI32(pDecoder, &ref) < 0) return -1;
       taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref));
+      taosMemoryFree(key);
     }
   }
 
@@ -1831,7 +1834,6 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
     if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1;
   }
 
-
   if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
 
   tEndEncode(&encoder);
@@ -1876,7 +1878,6 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
     if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1;
   }
 
-
   if (!tDecodeIsEnd(&decoder)) {
     if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
   } else {
@@ -2053,12 +2054,12 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
   if (pRsp->pFuncExtraInfos == NULL) return -1;
   if (tDecodeIsEnd(&decoder)) {
     for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) {
-      SFuncExtraInfo  extraInfo  = { 0 };
+      SFuncExtraInfo extraInfo = {0};
       taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo);
     }
   } else {
     for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) {
-      SFuncExtraInfo extraInfo = { 0 };
+      SFuncExtraInfo extraInfo = {0};
       if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1;
       if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1;
       taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo);

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -119,6 +119,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
         pVnode->pFetchQ->threadId);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
 
+  tqNotifyClose(pVnode->pImpl->pTq);
   dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
   while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
 
@@ -141,7 +142,6 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   dInfo("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (commitAndRemoveWal) {
-    char path[TSDB_FILENAME_LEN] = {0};
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP);
     dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path);
     tfsRmdir(pMgmt->pTfs, path);

source/dnode/mgmt/node_mgmt/src/dmNodes.c

@@ -132,11 +132,15 @@ int32_t dmRunDnode(SDnode *pDnode) {
   int32_t count = 0;
   if (dmOpenNodes(pDnode) != 0) {
     dError("failed to open nodes since %s", terrstr());
+    dmCloseNodes(pDnode);
     return -1;
   }
 
   if (dmStartNodes(pDnode) != 0) {
     dError("failed to start nodes since %s", terrstr());
+    dmSetStatus(pDnode, DND_STAT_STOPPED);
+    dmStopNodes(pDnode);
+    dmCloseNodes(pDnode);
     return -1;
   }
 

source/dnode/mnode/impl/src/mndDb.c

@@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
   if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
+  if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
+  if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
   if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
   if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
 
@@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
   if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
   if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
-  if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
-  if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
   if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
 }
 
@@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
     int32_t dbLen = strlen(dbObj.name) + 1;
     mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
     dbObj.cfg.hashPrefix += dbLen;
+  } else if (dbObj.cfg.hashPrefix < 0) {
+    int32_t dbLen = strlen(dbObj.name) + 1;
+    mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
+    dbObj.cfg.hashPrefix -= dbLen;
   }
 
   SVgObj *pVgroups = NULL;
@@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
     int16_t hashPrefix = pDb->cfg.hashPrefix;
     if (hashPrefix > 0) {
       hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
+    } else if (hashPrefix < 0) {
+      hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
     }
     colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);
 

source/dnode/mnode/impl/src/mndMain.c

@@ -124,11 +124,7 @@ static void mndCalMqRebalance(SMnode *pMnode) {
   int32_t contLen = 0;
   void   *pReq = mndBuildTimerMsg(&contLen);
   if (pReq != NULL) {
-    SRpcMsg rpcMsg = {
-        .msgType = TDMT_MND_TMQ_TIMER,
-        .pCont = pReq,
-        .contLen = contLen,
-    };
+    SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen };
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 }

source/dnode/mnode/impl/src/mndUser.c

@@ -390,6 +390,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &ref, _OVER);
 
       taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref));
+      taosMemoryFree(key);
     }
   }
 
@@ -956,6 +957,7 @@ _OVER:
     mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
   }
 
+  tFreeSAlterUserReq(&alterReq);
   mndReleaseUser(pMnode, pOperUser);
   mndReleaseUser(pMnode, pUser);
   mndUserFreeObj(&newUser);

source/dnode/vnode/inc/vnode.h

@@ -162,7 +162,6 @@ int32_t     metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType);
 #endif
 
 // tsdb
-// typedef struct STsdb STsdb;
 typedef struct STsdbReader STsdbReader;
 
 #define TSDB_DEFAULT_STT_FILE  8

source/dnode/vnode/src/inc/tq.h

@@ -180,16 +180,9 @@ int32_t tqStreamTasksScanWal(STQ* pTq);
 
 // tq util
 char*   createStreamTaskIdStr(int64_t streamId, int32_t taskId);
-void    createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId);
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver);
-int32_t launchTaskForWalBlock(SStreamTask* pTask, SFetchRet* pRet, STqOffset* pOffset);
 int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
 
-void    doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver);
-void    saveOffsetForAllTasks(STQ* pTq, int64_t ver);
-void    initOffsetForAllRestoreTasks(STQ* pTq);
-int32_t transferToWalReadTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/tsdb.h

@@ -16,6 +16,7 @@
 #ifndef _TD_VNODE_TSDB_H_
 #define _TD_VNODE_TSDB_H_
 
+#include "tsimplehash.h"
 #include "vnodeInt.h"
 
 #ifdef __cplusplus
@@ -125,11 +126,13 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter);
 // SRowMerger
 int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema);
 int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
-
-// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema);
 void tsdbRowMergerClear(SRowMerger *pMerger);
-// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow);
 int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow);
+
+int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema);
+void tsdbRowMergerClear_rv(SRowMerger* pMerger);
+void tsdbRowMergerCleanup_rv(SRowMerger* pMerger);
+
 // TABLEID
 int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
 // TSDBKEY
@@ -224,7 +227,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward,
 void   *tsdbTbDataIterDestroy(STbDataIter *pIter);
 void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
 bool    tsdbTbDataIterNext(STbDataIter *pIter);
-void    tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum);
+void    tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum);
 
 // STbData
 int32_t tsdbGetNRowsInTbData(STbData *pTbData);
@@ -322,8 +325,9 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData,
 #define TSDB_STT_FILE_DATA_ITER  2
 #define TSDB_TOMB_FILE_DATA_ITER 3
 
-#define TSDB_FILTER_FLAG_BY_VERSION 0x1
-#define TSDB_FILTER_FLAG_BY_TABLEID 0x2
+#define TSDB_FILTER_FLAG_BY_VERSION           0x1
+#define TSDB_FILTER_FLAG_BY_TABLEID           0x2
+#define TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE 0x4
 
 #define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn)))
 /* open */
@@ -705,7 +709,6 @@ typedef struct SSttBlockLoadInfo {
 typedef struct SMergeTree {
   int8_t             backward;
   SRBTree            rbt;
-  SArray            *pIterList;
   SLDataIter        *pIter;
   bool               destroyLoadInfo;
   SSttBlockLoadInfo *pLoadInfo;
@@ -751,13 +754,29 @@ struct SDiskDataBuilder {
   SBlkInfo     bi;
 };
 
+typedef struct SLDataIter {
+  SRBTreeNode        node;
+  SSttBlk           *pSttBlk;
+  SDataFReader      *pReader;
+  int32_t            iStt;
+  int8_t             backward;
+  int32_t            iSttBlk;
+  int32_t            iRow;
+  SRowInfo           rInfo;
+  uint64_t           uid;
+  STimeWindow        timeWindow;
+  SVersionRange      verRange;
+  SSttBlockLoadInfo *pBlockLoadInfo;
+  bool               ignoreEarlierTs;
+} SLDataIter;
+
+#define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row))
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
                        STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
-                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange);
+                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter);
 void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 bool    tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
-TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 
 SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt);
@@ -782,6 +801,7 @@ typedef struct SCacheRowsReader {
   STableKeyInfo     *pTableList;  // table id list
   int32_t            numOfTables;
   SSttBlockLoadInfo *pLoadInfo;
+  SLDataIter        *pDataIter;
   STsdbReadSnap     *pReadSnap;
   SDataFReader      *pDataFReader;
   SDataFReader      *pDataFReaderLast;

source/dnode/vnode/src/inc/vnodeInt.h

@@ -190,6 +190,7 @@ int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
 int  tqInit();
 void tqCleanUp();
 STQ* tqOpen(const char* path, SVnode* pVnode);
+void tqNotifyClose(STQ*);
 void tqClose(STQ*);
 int  tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int  tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp,

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -187,23 +187,24 @@ _err:
 
 int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t    code = 0;
+  int32_t    line = 0;
   SMeta*     pMeta = pWriter->pMeta;
   SMetaEntry metaEntry = {0};
   SDecoder*  pDecoder = &(SDecoder){0};
 
   tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
   code = metaDecodeEntry(pDecoder, &metaEntry);
-  if (code) goto _err;
+  VND_CHECK_CODE(code, line, _err);
 
   code = metaHandleEntry(pMeta, &metaEntry);
-  if (code) goto _err;
+  VND_CHECK_CODE(code, line, _err);
 
   tDecoderClear(pDecoder);
   return code;
 
 _err:
   tDecoderClear(pDecoder);
-  metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  metaError("vgId:%d, vnode snapshot meta write failed since %s at line:%d", TD_VID(pMeta->pVnode), terrstr(), line);
   return code;
 }
 

source/dnode/vnode/src/meta/metaTable.c

@@ -936,8 +936,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     int   tLen = 0;
 
     if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
-      version = ((SUidIdxVal *)tData)[0].version;
-      STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version};
+      STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
       if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
         SDecoder   tdc = {0};
         SMetaEntry stbEntry = {0};
@@ -1029,7 +1028,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
   metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode),
             pME->version, pME->uid, ctimeKey.ctime);
 
-  return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -1044,7 +1043,7 @@ int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   if (metaBuildNColIdxKey(&ncolKey, pME) < 0) {
     return 0;
   }
-  return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -1878,24 +1877,24 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
 }
 
 static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
+  return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
   if (ttlKey.dtime == 0) return 0;
-  return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
 
-  return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
+  return tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
                      ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn);
 }
 
@@ -2065,49 +2064,66 @@ _exit:
 }
 
 int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+  int32_t code = 0;
+  int32_t line = 0;
   metaWLock(pMeta);
 
   // save to table.db
-  if (metaSaveToTbDb(pMeta, pME) < 0) goto _err;
+  code = metaSaveToTbDb(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   // update uid.idx
-  if (metaUpdateUidIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateUidIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   // update name.idx
-  if (metaUpdateNameIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateNameIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_CHILD_TABLE) {
     // update ctb.idx
-    if (metaUpdateCtbIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateCtbIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
 
     // update tag.idx
-    if (metaUpdateTagIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateTagIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   } else {
     // update schema.db
-    if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err;
+    code = metaSaveToSkmDb(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
 
     if (pME->type == TSDB_SUPER_TABLE) {
-      if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
+      code = metaUpdateSuidIdx(pMeta, pME);
+      VND_CHECK_CODE(code, line, _err);
     }
   }
 
-  if (metaUpdateCtimeIdx(pMeta, pME) < 0) goto _err;
+  code = metaUpdateCtimeIdx(pMeta, pME);
+  VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_NORMAL_TABLE) {
-    if (metaUpdateNcolIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateNcolIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   }
 
   if (pME->type != TSDB_SUPER_TABLE) {
-    if (metaUpdateTtlIdx(pMeta, pME) < 0) goto _err;
+    code = metaUpdateTtlIdx(pMeta, pME);
+    VND_CHECK_CODE(code, line, _err);
   }
 
   metaULock(pMeta);
+  metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode),
+            pME->version, pME->uid, pME->name);
   return 0;
 
 _err:
   metaULock(pMeta);
+  metaError("vgId:%d, failed to handle meta entry since %s at line:%d, ver:%" PRId64 ", uid:%" PRId64 ", name:%s",
+            TD_VID(pMeta->pVnode), terrstr(), line, pME->version, pME->uid, pME->name);
   return -1;
 }
+
 // refactor later
 void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
 void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

source/dnode/vnode/src/tq/tq.c

@@ -154,6 +154,31 @@ void tqClose(STQ* pTq) {
   taosMemoryFree(pTq);
 }
 
+void tqNotifyClose(STQ* pTq) {
+  if (pTq != NULL) {
+    taosWLockLatch(&pTq->pStreamMeta->lock);
+
+    void* pIter = NULL;
+    while (1) {
+      pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
+      if (pIter == NULL) {
+        break;
+      }
+
+      SStreamTask* pTask = *(SStreamTask**)pIter;
+      tqDebug("vgId:%d s-task:%s set dropping flag", pTq->pStreamMeta->vgId, pTask->id.idStr);
+      pTask->status.taskStatus = TASK_STATUS__STOP;
+
+      int64_t st = taosGetTimestampMs();
+      qKillTask(pTask->exec.pExecutor, TSDB_CODE_SUCCESS);
+      int64_t el = taosGetTimestampMs() - st;
+      tqDebug("vgId:%d s-task:%s is closed in %" PRId64 "ms", pTq->pStreamMeta->vgId, pTask->id.idStr, el);
+    }
+
+    taosWUnLockLatch(&pTq->pStreamMeta->lock);
+  }
+}
+
 static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch,
                              int64_t consumerId, int32_t type) {
   int32_t len = 0;
@@ -573,6 +598,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->pMsgCb = &pTq->pVnode->msgCb;
   pTask->pMeta = pTq->pStreamMeta;
   pTask->chkInfo.version = ver;
+  pTask->chkInfo.currentVer = ver;
 
   // expand executor
   if (pTask->fillHistory) {
@@ -966,14 +992,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   int32_t* pRef = taosMemoryMalloc(sizeof(int32_t));
   *pRef = 1;
 
+  taosWLockLatch(&pTq->pStreamMeta->lock);
+
   void* pIter = NULL;
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) break;
+    if (pIter == NULL) {
+      break;
+    }
+
     SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      continue;
+    }
 
-    qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver);
+    qDebug("s-task:%s delete req enqueue, ver: %" PRId64, pTask->id.idStr, ver);
 
     if (!failed) {
       SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0);
@@ -983,15 +1016,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
       atomic_add_fetch_32(pRefBlock->dataRef, 1);
 
       if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) {
-        qError("stream task input del failed, task id %d", pTask->id.taskId);
-
         atomic_sub_fetch_32(pRef, 1);
         taosFreeQitem(pRefBlock);
         continue;
       }
 
       if (streamSchedExec(pTask) < 0) {
-        qError("stream task launch failed, task id %d", pTask->id.taskId);
+        qError("s-task:%s stream task launch failed", pTask->id.idStr);
         continue;
       }
 
@@ -1000,8 +1031,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
     }
   }
 
+  taosWUnLockLatch(&pTq->pStreamMeta->lock);
+
   int32_t ref = atomic_sub_fetch_32(pRef, 1);
-  /*A(ref >= 0);*/
   if (ref == 0) {
     blockDataDestroy(pDelBlock);
     taosMemoryFree(pRef);
@@ -1032,23 +1064,9 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
   }
   blockDataDestroy(pDelBlock);
 #endif
-
   return 0;
 }
 
-static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit,
-                                         const char* key, int64_t ver) {
-  doSaveTaskOffset(pOffsetStore, key, ver);
-  int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver);
-
-  // remove the offset, if all functions are completed successfully.
-  if (code == TSDB_CODE_SUCCESS) {
-    tqOffsetDelete(pOffsetStore, key);
-  }
-
-  return code;
-}
-
 int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
 #if 0
   void* pIter = NULL;
@@ -1309,9 +1327,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
     return -1;
   }
 
-  tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
-  initOffsetForAllRestoreTasks(pTq);
-
+  tqDebug("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks);
   pRunReq->head.vgId = vgId;
   pRunReq->streamId = 0;
   pRunReq->taskId = WAL_READ_TASKS_ID;

source/dnode/vnode/src/tq/tqRead.c

@@ -1023,6 +1023,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
   }
 
   // update the table list handle for each stream scanner/wal reader
+  taosWLockLatch(&pTq->pStreamMeta->lock);
   while (1) {
     pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
@@ -1039,5 +1040,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
     }
   }
 
+  taosWUnLockLatch(&pTq->pStreamMeta->lock);
+
   return 0;
 }

source/dnode/vnode/src/tq/tqRestore.c

@@ -15,13 +15,12 @@
 
 #include "tq.h"
 
-static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle);
-static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList);
+static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle);
 
 // this function should be executed by stream threads.
 // there is a case that the WAL increases more fast than the restore procedure, and this restore procedure
 // will not stop eventually.
-int tqStreamTasksScanWal(STQ* pTq) {
+int32_t tqStreamTasksScanWal(STQ* pTq) {
   int32_t vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
   int64_t st = taosGetTimestampMs();
@@ -32,7 +31,7 @@ int tqStreamTasksScanWal(STQ* pTq) {
 
     // check all restore tasks
     bool shouldIdle = true;
-    streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &shouldIdle);
+    createStreamRunReq(pTq->pStreamMeta, &shouldIdle);
 
     int32_t times = 0;
 
@@ -55,87 +54,86 @@ int tqStreamTasksScanWal(STQ* pTq) {
 
   int64_t el = (taosGetTimestampMs() - st);
   tqDebug("vgId:%d scan wal for stream tasks completed, elapsed time:%"PRId64" ms", vgId, el);
-
-  // restore wal scan flag
-//  atomic_store_8(&pTq->pStreamMeta->walScan, 0);
   return 0;
 }
 
-//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
-//  int32_t numOfTask = taosArrayGetSize(pTaskList);
-//  if (numOfTask <= 0)  {
-//    return TSDB_CODE_SUCCESS;
-//  }
-//
-//  // todo: add lock
-//  for (int32_t i = 0; i < numOfTask; ++i) {
-//    SStreamTask* pTask = taosArrayGetP(pTaskList, i);
-//    tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
-//            pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
-//    taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
-//
-//    // NOTE: do not change the following order
-//    atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
-//    taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
-//  }
-//
-//  return TSDB_CODE_SUCCESS;
-//}
-
-int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
+static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) {
+  SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t));
   void*   pIter = NULL;
-  int32_t vgId = pStreamMeta->vgId;
-
-  *pScanIdle = true;
-
-  bool allWalChecked = true;
-  tqDebug("vgId:%d start to check wal to extract new submit block", vgId);
 
-  while (1) {
+  taosWLockLatch(&pStreamMeta->lock);
+  while(1) {
     pIter = taosHashIterate(pStreamMeta->pTasks, pIter);
     if (pIter == NULL) {
       break;
     }
 
     SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+    taosArrayPush(pTaskIdList, &pTask->id.taskId);
+  }
+
+  taosWUnLockLatch(&pStreamMeta->lock);
+  return pTaskIdList;
+}
+
+int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
+  *pScanIdle = true;
+  bool    noNewDataInWal = true;
+  int32_t vgId = pStreamMeta->vgId;
+
+  int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks);
+  if (numOfTasks == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks);
+  SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks);
+
+  // update the new task number
+  numOfTasks = taosArrayGetSize(pTaskIdList);
+  for (int32_t i = 0; i < numOfTasks; ++i) {
+    int32_t*     pTaskId = taosArrayGet(pTaskIdList, i);
+    SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId);
+    if (pTask == NULL) {
       continue;
     }
 
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE ||
-        pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
-              pTask->status.taskStatus);
+    int32_t status = pTask->status.taskStatus;
+    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
+      tqDebug("s-task:%s not source task, no need to start", pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    // check if offset value exists
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
+    if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE ||
+        status == TASK_STATUS__WAIT_DOWNSTREAM) {
+      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status);
+      streamMetaReleaseTask(pStreamMeta, pTask);
+      continue;
+    }
 
     if (tInputQueueIsFull(pTask)) {
       tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
     *pScanIdle = false;
 
-    // check if offset value exists
-    STqOffset* pOffset = tqOffsetRead(pOffsetStore, key);
-    ASSERT(pOffset != NULL);
-
     // seek the stored version and extract data from WAL
-    int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version);
+    int32_t code = walReadSeekVer(pTask->exec.pWalReader, pTask->chkInfo.currentVer);
     if (code != TSDB_CODE_SUCCESS) {  // no data in wal, quit
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
     // append the data for the stream
-    tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr);
+    tqDebug("vgId:%d s-task:%s wal reader seek to ver:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.currentVer);
 
     SPackedData packData = {0};
     code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData);
     if (code != TSDB_CODE_SUCCESS) {  // failed, continue
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
@@ -143,28 +141,32 @@ int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetSto
     if (p == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr);
+      streamMetaReleaseTask(pStreamMeta, pTask);
       continue;
     }
 
-    allWalChecked = false;
+    noNewDataInWal = false;
 
-    tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr);
     code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver);
     if (code == TSDB_CODE_SUCCESS) {
-      pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader);
+      pTask->chkInfo.currentVer = walReaderGetCurrentVer(pTask->exec.pWalReader);
       tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr,
-              pOffset->val.version);
+              pTask->chkInfo.currentVer);
     } else {
-      // do nothing
+      tqError("s-task:%s append input queue failed, ver:%"PRId64, pTask->id.idStr, pTask->chkInfo.currentVer);
     }
 
     streamDataSubmitDestroy(p);
     taosFreeQitem(p);
+    streamMetaReleaseTask(pStreamMeta, pTask);
   }
 
-  if (allWalChecked) {
+  // all wal are checked, and no new data available in wal.
+  if (noNewDataInWal) {
     *pScanIdle = true;
   }
+
+  taosArrayDestroy(pTaskIdList);
   return 0;
 }
 

source/dnode/vnode/src/tq/tqUtil.c

@@ -25,21 +25,6 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) {
   return taosStrdup(buf);
 }
 
-// stream_task:stream_id:task_id
-void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) {
-  int32_t n = 12;
-  char* p = dst;
-
-  memcpy(p, "stream_task:", n);
-  p += n;
-
-  int32_t inc = tintToHex(streamId, p);
-  p += inc;
-
-  *(p++) = ':';
-  tintToHex(taskId, p);
-}
-
 int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) {
   int32_t code = tAppendDataToInputQueue(pTask, pQueueItem);
   if (code < 0) {
@@ -55,75 +40,6 @@ int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueI
   return TSDB_CODE_SUCCESS;
 }
 
-void initOffsetForAllRestoreTasks(STQ* pTq) {
-  void* pIter = NULL;
-
-  while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) {
-      break;
-    }
-
-    SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
-      continue;
-    }
-
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus);
-      continue;
-    }
-
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version);
-    }
-  }
-}
-
-void saveOffsetForAllTasks(STQ* pTq, int64_t ver) {
-  void* pIter = NULL;
-
-  while(1) {
-    pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter);
-    if (pIter == NULL) {
-      break;
-    }
-
-    SStreamTask* pTask = *(SStreamTask**)pIter;
-    if (pTask->taskLevel != TASK_LEVEL__SOURCE) {
-      continue;
-    }
-
-    if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
-      tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr,
-              pTask->status.taskStatus);
-      continue;
-    }
-
-    char key[128] = {0};
-    createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId);
-
-    STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key);
-    if (pOffset == NULL) {
-      doSaveTaskOffset(pTq->pOffsetStore, key, ver);
-    }
-  }
-}
-
-void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) {
-  STqOffset offset = {0};
-  tqOffsetResetToLog(&offset.val, ver);
-
-  tstrncpy(offset.subKey, pKey, tListLen(offset.subKey));
-
-  // keep the offset info in the offset store
-  tqOffsetWrite(pOffsetStore, &offset);
-}
-
 static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) {
   pRsp->reqOffset = pReq->reqOffset;
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -598,6 +598,7 @@ typedef struct {
   SMergeTree         mergeTree;
   SMergeTree        *pMergeTree;
   SSttBlockLoadInfo *pLoadInfo;
+  SLDataIter*        pDataIter;
   int64_t            lastTs;
 } SFSLastNextRowIter;
 
@@ -645,7 +646,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
       }
       tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
-                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
+                     &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter);
       state->pMergeTree = &state->mergeTree;
       state->state = SFSLASTNEXTROW_BLOCKROW;
     }
@@ -667,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         state->state = SFSLASTNEXTROW_FILESET;
         goto _next_fileset;
       }
-      state->row = tMergeTreeGetRow(&state->mergeTree);
+      state->row = *tMergeTreeGetRow(&state->mergeTree);
       *ppRow = &state->row;
 
       if (TSDBROW_TS(&state->row) <= state->lastTs) {
@@ -1211,7 +1212,7 @@ typedef struct {
 } CacheNextRowIter;
 
 static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
-                               SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
+                               SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader,
                                SDataFReader **pDataFReaderLast, int64_t lastTs) {
   int code = 0;
 
@@ -1274,6 +1275,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
   pIter->fsLastState.pLoadInfo = pLoadInfo;
   pIter->fsLastState.pDataFReader = pDataFReaderLast;
   pIter->fsLastState.lastTs = lastTs;
+  pIter->fsLastState.pDataIter = pLDataIter;
 
   pIter->fsState.state = SFSNEXTROW_FS;
   pIter->fsState.pTsdb = pTsdb;
@@ -1465,7 +1467,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
                   &pr->pDataFReaderLast, pr->lastTs);
 
   do {
@@ -1622,7 +1624,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach
   TSKEY lastRowTs = TSKEY_MAX;
 
   CacheNextRowIter iter = {0};
-  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader,
+  nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader,
                   &pr->pDataFReaderLast, pr->lastTs);
 
   do {

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -187,13 +187,21 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
     }
   }
 
-  int32_t numOfStt = ((SVnode*)pVnode)->config.sttTrigger;
+  SVnodeCfg* pCfg = &((SVnode*)pVnode)->config;
+
+  int32_t numOfStt = pCfg->sttTrigger;
   p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt);
   if (p->pLoadInfo == NULL) {
     tsdbCacherowsReaderClose(p);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter));
+  if (p->pDataIter == NULL) {
+    tsdbCacherowsReaderClose(p);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
   p->idstr = taosStrdup(idstr);
   taosThreadMutexInit(&p->readerMutex, NULL);
 
@@ -215,6 +223,7 @@ void* tsdbCacherowsReaderClose(void* pReader) {
     taosMemoryFree(p->pSchema);
   }
 
+  taosMemoryFreeClear(p->pDataIter);
   taosMemoryFree(p->pCurrSchema);
 
   destroyLastBlockLoadInfo(p->pLoadInfo);

source/dnode/vnode/src/tsdb/tsdbDataIter.c

@@ -14,6 +14,7 @@
  */
 
 #include "tsdb.h"
+#include "vnodeInt.h"
 
 // STsdbDataIter2
 /* open */
@@ -202,13 +203,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
   for (;;) {
     while (pIter->dIter.iRow < pIter->dIter.bData.nRow) {
       if (pFilterInfo) {
-        if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
-          if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) {
-            pIter->dIter.iRow = pIter->dIter.bData.nRow;
-            continue;
-          }
-        }
-
         if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
           if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver ||
               pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) {
@@ -232,13 +226,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
 
         // filter
         if (pFilterInfo) {
-          if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
-            if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) {
-              pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem;
-              continue;
-            }
-          }
-
           if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
             if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) {
               pIter->dIter.iDataBlk++;
@@ -262,13 +249,23 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
         if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) {
           SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx);
 
-          if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) {
-            int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
-            if (c == 0) {
-              pIter->dIter.iBlockIdx++;
-              continue;
-            } else if (c < 0) {
-              ASSERT(0);
+          if (pFilterInfo) {
+            if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
+              int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid);
+              if (c == 0) {
+                pIter->dIter.iBlockIdx++;
+                continue;
+              } else if (c < 0) {
+                ASSERT(0);
+              }
+            }
+
+            if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+              SMetaInfo info;
+              if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pBlockIdx->uid, &info, NULL)) {
+                pIter->dIter.iBlockIdx++;
+                continue;
+              }
             }
           }
 
@@ -304,14 +301,24 @@ static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* p
   for (;;) {
     while (pIter->sIter.iRow < pIter->sIter.bData.nRow) {
       if (pFilterInfo) {
+        int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
         if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) {
-          int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow];
           if (pFilterInfo->tbid.uid == uid) {
             pIter->sIter.iRow++;
             continue;
           }
         }
 
+        if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+          if (pIter->rowInfo.uid != uid) {
+            SMetaInfo info;
+            if (metaGetInfo(pIter->sIter.pReader->pTsdb->pVnode->pMeta, uid, &info, NULL)) {
+              pIter->sIter.iRow++;
+              continue;
+            }
+          }
+        }
+
         if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) {
           if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] ||
               pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) {
@@ -395,6 +402,16 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo*
       if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) {
         SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx);
 
+        if (pFilterInfo) {
+          if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) {
+            SMetaInfo info;
+            if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pDelIdx->uid, &info, NULL)) {
+              pIter->tIter.iDelIdx++;
+              continue;
+            }
+          }
+        }
+
         code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData);
         TSDB_CHECK_CODE(code, lino, _exit);
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <util/tsimplehash.h>
 #include "tsdb.h"
 
 #define MEM_MIN_HASH 1024
@@ -298,12 +299,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) {
   return rowsNum;
 }
 
-void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj*        pTableMap, int64_t *rowsNum) {
+void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) {
   taosRLockLatch(&pMemTable->latch);
   for (int32_t i = 0; i < pMemTable->nBucket; ++i) {
     STbData *pTbData = pMemTable->aBucket[i];
     while (pTbData) {
-      void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
+      void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid));
       if (p == NULL) {
         pTbData = pTbData->next;
         continue;

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -16,22 +16,6 @@
 #include "tsdb.h"
 
 // SLDataIter =================================================
-struct SLDataIter {
-  SRBTreeNode        node;
-  SSttBlk           *pSttBlk;
-  SDataFReader      *pReader;
-  int32_t            iStt;
-  int8_t             backward;
-  int32_t            iSttBlk;
-  int32_t            iRow;
-  SRowInfo           rInfo;
-  uint64_t           uid;
-  STimeWindow        timeWindow;
-  SVersionRange      verRange;
-  SSttBlockLoadInfo *pBlockLoadInfo;
-  bool               ignoreEarlierTs;
-};
-
 SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols,
                                             int32_t numOfSttTrigger) {
   SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo));
@@ -88,6 +72,10 @@ void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double
 }
 
 void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
+  if (pLoadInfo == NULL) {
+    return NULL;
+  }
+
   for (int32_t i = 0; i < pLoadInfo->numOfStt; ++i) {
     pLoadInfo[i].currentLoadBlockIndex = 1;
     pLoadInfo[i].blockIndex[0] = -1;
@@ -264,25 +252,21 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint
   }
 }
 
-int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
+int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
                        uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo,
                        const char *idStr, bool strictTimeRange) {
   int32_t code = TSDB_CODE_SUCCESS;
 
-  *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
-  if (*pIter == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
-  }
-
-  (*pIter)->uid = uid;
-  (*pIter)->pReader = pReader;
-  (*pIter)->iStt = iStt;
-  (*pIter)->backward = backward;
-  (*pIter)->verRange = *pRange;
-  (*pIter)->timeWindow = *pTimeWindow;
+  pIter->uid = uid;
+  pIter->pReader = pReader;
+  pIter->iStt = iStt;
+  pIter->backward = backward;
+  pIter->verRange.minVer = pRange->minVer;
+  pIter->verRange.maxVer = pRange->maxVer;
+  pIter->timeWindow.skey = pTimeWindow->skey;
+  pIter->timeWindow.ekey = pTimeWindow->ekey;
 
-  (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
+  pIter->pBlockLoadInfo = pBlockLoadInfo;
 
   if (!pBlockLoadInfo->sttBlockLoaded) {
     int64_t st = taosGetTimestampUs();
@@ -290,7 +274,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
-      goto _exit;
+      return code;
     }
 
     // only apply to the child tables, ordinary tables will not incur this filter procedure.
@@ -306,7 +290,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
           // no qualified stt block existed
           taosArrayClear(pBlockLoadInfo->aSttBlk);
 
-          (*pIter)->iSttBlk = -1;
+          pIter->iSttBlk = -1;
           double el = (taosGetTimestampUs() - st) / 1000.0;
           tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
           return code;
@@ -339,31 +323,27 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
-  (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
-  if ((*pIter)->iSttBlk != -1) {
-    (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk);
-    (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1;
-
-    if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) ||
-                        (!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) {
-      (*pIter)->pSttBlk = NULL;
+  pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
+  if (pIter->iSttBlk != -1) {
+    pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
+    pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1;
+
+    if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||
+                        (!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {
+      pIter->pSttBlk = NULL;
     }
 
-    if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) ||
-                     (!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) {
-      (*pIter)->pSttBlk = NULL;
-      (*pIter)->ignoreEarlierTs = true;
+    if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||
+                     (!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {
+      pIter->pSttBlk = NULL;
+      pIter->ignoreEarlierTs = true;
     }
   }
 
   return code;
-
-_exit:
-  taosMemoryFree(*pIter);
-  return code;
 }
 
-void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); }
+void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */}
 
 void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) {
   int32_t step = pIter->backward ? -1 : 1;
@@ -590,43 +570,38 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR
 
 int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
                        STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
-                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange) {
+                       bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
   pMTree->backward = backward;
   pMTree->pIter = NULL;
-  pMTree->pIterList = taosArrayInit(4, POINTER_BYTES);
-  if (pMTree->pIterList == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
   pMTree->idStr = idStr;
+
   if (!pMTree->backward) {  // asc
     tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
   } else {  // desc
     tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn);
   }
-  int32_t code = TSDB_CODE_SUCCESS;
 
   pMTree->pLoadInfo = pBlockLoadInfo;
   pMTree->destroyLoadInfo = destroyLoadInfo;
   pMTree->ignoreEarlierTs = false;
 
   for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) {  // open all last file
-    struct SLDataIter *pIter = NULL;
-    code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
+    memset(&pLDataIter[i], 0, sizeof(SLDataIter));
+    code = tLDataIterOpen(&pLDataIter[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange,
                           &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;
     }
 
-    bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
+    bool hasVal = tLDataIterNextRow(&pLDataIter[i], pMTree->idStr);
     if (hasVal) {
-      taosArrayPush(pMTree->pIterList, &pIter);
-      tMergeTreeAddIter(pMTree, pIter);
+      tMergeTreeAddIter(pMTree, &pLDataIter[i]);
     } else {
       if (!pMTree->ignoreEarlierTs) {
-        pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;
+        pMTree->ignoreEarlierTs = pLDataIter[i].ignoreEarlierTs;
       }
-      tLDataIterClose(pIter);
     }
   }
 
@@ -674,18 +649,8 @@ bool tMergeTreeNext(SMergeTree *pMTree) {
   return pMTree->pIter != NULL;
 }
 
-TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; }
-
 void tMergeTreeClose(SMergeTree *pMTree) {
-  size_t size = taosArrayGetSize(pMTree->pIterList);
-  for (int32_t i = 0; i < size; ++i) {
-    SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i);
-    tLDataIterClose(pIter);
-  }
-
-  pMTree->pIterList = taosArrayDestroy(pMTree->pIterList);
   pMTree->pIter = NULL;
-
   if (pMTree->destroyLoadInfo) {
     pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo);
     pMTree->destroyLoadInfo = false;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -70,10 +70,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
 
   if (pReader->pIter) {
     // iter to next with filter info (sver, ever)
-    code = tsdbDataIterNext2(pReader->pIter,
-                             &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
-                                                .sver = pReader->sver,
-                                                .ever = pReader->ever});
+    code = tsdbDataIterNext2(
+        pReader->pIter,
+        &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
+                           .sver = pReader->sver,
+                           .ever = pReader->ever});
     TSDB_CHECK_CODE(code, lino, _exit);
 
     if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) {
@@ -94,10 +95,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) {
 
     if (pReader->pIter) {
       // iter to valid row
-      code = tsdbDataIterNext2(pReader->pIter,
-                               &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
-                                                  .sver = pReader->sver,
-                                                  .ever = pReader->ever});
+      code = tsdbDataIterNext2(
+          pReader->pIter,
+          &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
+                             .sver = pReader->sver,
+                             .ever = pReader->ever});
       TSDB_CHECK_CODE(code, lino, _exit);
 
       if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) {
@@ -139,7 +141,8 @@ static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInf
   int32_t lino = 0;
 
   if (pReader->pIter) {
-    code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION,  // flag
+    code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION |
+                                                                        TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,  // flag
                                                                 .sver = pReader->sver,
                                                                 .ever = pReader->ever});
     TSDB_CHECK_CODE(code, lino, _exit);
@@ -346,8 +349,9 @@ static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppD
   int32_t lino = 0;
 
   code = tsdbDataIterNext2(
-      pReader->pTIter,
-      &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, .sver = pReader->sver, .ever = pReader->ever});
+      pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE,
+                                          .sver = pReader->sver,
+                                          .ever = pReader->ever});
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if (ppDelInfo) {

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -712,124 +712,163 @@ _exit:
 int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
   int32_t   code = 0;
   TSDBKEY   key = TSDBROW_KEY(pRow);
-  SColVal  *pColVal = &(SColVal){0};
+  SColVal * pColVal = &(SColVal){0};
   STColumn *pTColumn;
   int32_t   iCol, jCol = 1;
 
   if (NULL == pTSchema) {
     pTSchema = pMerger->pTSchema;
   }
-  ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
 
-  for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
-    pTColumn = &pMerger->pTSchema->columns[iCol];
-    if (pTSchema->columns[jCol].colId < pTColumn->colId) {
-      ++jCol;
-      --iCol;
-      continue;
-    } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
-      continue;
+  if (taosArrayGetSize(pMerger->pArray) == 0) {
+    // ts
+    jCol = 0;
+    pTColumn = &pTSchema->columns[jCol++];
+
+    ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
+
+    *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
+    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      return code;
+      //      goto _exit;
     }
 
-    tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
+    // other
+    for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pMerger->pTSchema->numOfCols; ++iCol) {
+      pTColumn = &pMerger->pTSchema->columns[iCol];
+      if (pTSchema->columns[jCol].colId < pTColumn->colId) {
+        ++jCol;
+        --iCol;
+        continue;
+      } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
+        taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
+        continue;
+      }
 
-    if (key.version > pMerger->version) {
-      if (!COL_VAL_IS_NONE(pColVal)) {
-        if (IS_VAR_DATA_TYPE(pColVal->type)) {
-          SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
-          if (!COL_VAL_IS_NULL(pColVal)) {
-            code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
-            if (code) return code;
+      tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
+      if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+        uint8_t *pVal = pColVal->value.pData;
 
-            pTColVal->value.nData = pColVal->value.nData;
-            if (pTColVal->value.nData) {
-              memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData);
+        pColVal->value.pData = NULL;
+        code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
+        if (code) {
+          return TSDB_CODE_OUT_OF_MEMORY;
+        }
+
+        if (pColVal->value.nData) {
+          memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+        }
+      }
+
+      if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        return code;
+      }
+    }
+
+    for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) {
+      pTColumn = &pMerger->pTSchema->columns[iCol];
+      taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type));
+    }
+
+    pMerger->version = key.version;
+    return 0;
+  } else {
+    ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
+
+    for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
+      pTColumn = &pMerger->pTSchema->columns[iCol];
+      if (pTSchema->columns[jCol].colId < pTColumn->colId) {
+        ++jCol;
+        --iCol;
+        continue;
+      } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
+        continue;
+      }
+
+      tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
+
+      if (key.version > pMerger->version) {
+        if (!COL_VAL_IS_NONE(pColVal)) {
+          if (IS_VAR_DATA_TYPE(pColVal->type)) {
+            SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+            if (!COL_VAL_IS_NULL(pColVal)) {
+              code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
+              if (code) return code;
+
+              pTColVal->value.nData = pColVal->value.nData;
+              if (pTColVal->value.nData) {
+                memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData);
+              }
+              pTColVal->flag = 0;
+            } else {
+              tFree(pTColVal->value.pData);
+              taosArraySet(pMerger->pArray, iCol, pColVal);
             }
-            pTColVal->flag = 0;
           } else {
-            tFree(pTColVal->value.pData);
             taosArraySet(pMerger->pArray, iCol, pColVal);
           }
-        } else {
-          taosArraySet(pMerger->pArray, iCol, pColVal);
         }
-      }
-    } else if (key.version < pMerger->version) {
-      SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
-      if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
-          code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
-          if (code) return code;
+      } else if (key.version < pMerger->version) {
+        SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
+        if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
+          if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
+            code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
+            if (code) return code;
 
-          tColVal->value.nData = pColVal->value.nData;
-          if (pColVal->value.nData) {
-            memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+            tColVal->value.nData = pColVal->value.nData;
+            if (pColVal->value.nData) {
+              memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData);
+            }
+            tColVal->flag = 0;
+          } else {
+            taosArraySet(pMerger->pArray, iCol, pColVal);
           }
-          tColVal->flag = 0;
-        } else {
-          taosArraySet(pMerger->pArray, iCol, pColVal);
         }
+      } else {
+        ASSERT(0 && "dup versions not allowed");
       }
-    } else {
-      ASSERT(0 && "dup versions not allowed");
     }
-  }
 
-  pMerger->version = key.version;
-  return code;
+    pMerger->version = key.version;
+    return code;
+  }
 }
-/*
-int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
-  int32_t   code = 0;
-  TSDBKEY   key = TSDBROW_KEY(pRow);
-  SColVal  *pColVal = &(SColVal){0};
-  STColumn *pTColumn;
 
-  pMerger->pTSchema = pTSchema;
-  pMerger->version = key.version;
-
-  pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
+int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) {
+  pMerger->pTSchema = pSchema;
+  pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
   if (pMerger->pArray == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
+    return TSDB_CODE_OUT_OF_MEMORY;
+  } else {
+    return TSDB_CODE_SUCCESS;
   }
+}
 
-  // ts
-  pTColumn = &pTSchema->columns[0];
-
-  ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
-
-  *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
-  if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
+void tsdbRowMergerClear_rv(SRowMerger* pMerger) {
+  for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
+    }
   }
 
-  // other
-  for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
-    tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal);
-    if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
-      uint8_t *pVal = pColVal->value.pData;
-
-      pColVal->value.pData = NULL;
-      code = tRealloc(&pColVal->value.pData, pColVal->value.nData);
-      if (code) goto _exit;
-
-      if (pColVal->value.nData) {
-        memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
-      }
-    }
+  taosArrayClear(pMerger->pArray);
+}
 
-    if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _exit;
+void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) {
+  int32_t numOfCols = taosArrayGetSize(pMerger->pArray);
+  for (int32_t iCol = 1; iCol < numOfCols; iCol++) {
+    SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+    if (IS_VAR_DATA_TYPE(pTColVal->type)) {
+      tFree(pTColVal->value.pData);
     }
   }
 
-_exit:
-  return code;
+  taosArrayDestroy(pMerger->pArray);
 }
-*/
+
 void tsdbRowMergerClear(SRowMerger *pMerger) {
   for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
     SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
@@ -1218,9 +1257,10 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
     int32_t   iColumn = 1;
     STColumn *pTColumn = &pTSchema->columns[iColumn];
     for (int32_t iCid = 0; iCid < nCid; iCid++) {
-      if (ASSERTS(pTColumn != NULL, "invalid input param")) {
-        code = TSDB_CODE_INVALID_PARA;
-        goto _exit;
+
+      // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer
+      if (pTColumn == NULL) {
+        continue;
       }
 
       while (pTColumn->colId < aCid[iCid]) {
@@ -1229,9 +1269,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
         pTColumn = &pTSchema->columns[iColumn];
       }
 
-      if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) {
-        code = TSDB_CODE_INVALID_PARA;
-        goto _exit;
+      if (pTColumn->colId != aCid[iCid]) {
+        continue;
       }
 
       tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type,

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -1348,6 +1348,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
           code = terrno;
           goto _exit;
         }
+        terrno = 0;
         pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid;  // update uid if table exist for using below
       }
     }

source/libs/command/src/command.c

@@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
 
   char* retentions = buildRetension(pCfg->pRetensions);
   int32_t dbFNameLen = strlen(dbFName);
-  int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
+  int32_t hashPrefix = 0;
+  if (pCfg->hashPrefix > 0) {
+    hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
+  } else if (pCfg->hashPrefix < 0) {
+    hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
+  }
 
   len += sprintf(
       buf2 + VARSTR_HEADER_SIZE,

source/libs/executor/src/executor.c

@@ -1108,6 +1108,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       pScanBaseInfo->dataReader = NULL;
 
       // let's seek to the next version in wal file
+      int64_t firstVer = walGetFirstVer(pInfo->tqReader->pWalReader->pWal);
+      if (pOffset->version + 1 < firstVer){
+        pOffset->version = firstVer - 1;
+      }
+
       if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) {
         qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id);
         return -1;

tests/script/win-test-file

@@ -3,6 +3,7 @@
 ./test.sh -f tsim/user/privilege_db.sim
 ./test.sh -f tsim/user/privilege_sysinfo.sim
 ./test.sh -f tsim/user/privilege_topic.sim
+./test.sh -f tsim/user/privilege_table.sim
 ./test.sh -f tsim/db/alter_option.sim
 rem ./test.sh -f tsim/db/alter_replica_13.sim
 ./test.sh -f tsim/db/alter_replica_31.sim