Merge branch '3.0' into fix/TD-15172

contrib/CMakeLists.txt

@@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 # ================================================================================================
 # googletest
 if(${BUILD_TEST})
-    add_subdirectory(googletest)
+    add_subdirectory(googletest EXCLUDE_FROM_ALL)
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
@@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
 option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
 option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
-add_subdirectory(cJson)
+add_subdirectory(cJson EXCLUDE_FROM_ALL)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
@@ -152,7 +152,7 @@ target_include_directories(
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
 # lz4
-add_subdirectory(lz4/build/cmake)
+add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
@@ -160,7 +160,7 @@ target_include_directories(
 
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
-add_subdirectory(zlib)
+add_subdirectory(zlib EXCLUDE_FROM_ALL)
 target_include_directories(
     zlibstatic
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 # leveldb
 if(${BUILD_WITH_LEVELDB})
     option(LEVELDB_BUILD_TESTS "" OFF)
-    add_subdirectory(leveldb)
+    add_subdirectory(leveldb EXCLUDE_FROM_ALL)
     target_include_directories(
         leveldb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
@@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB})
     option(WITH_TOOLS "" OFF)
     option(WITH_LIBURING "" OFF)
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    add_subdirectory(rocksdb)
+    add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
     target_include_directories(
         rocksdb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
@@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB})
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
-    add_subdirectory(lucene)
+    add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
       lucene++ 
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
@@ -213,13 +213,13 @@ endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
 if(${BUILD_WITH_NURAFT})
-    add_subdirectory(nuraft)
+    add_subdirectory(nuraft EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_NURAFT})
 
 # pthread
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE release)
-    add_definitions(-DPTW32_STATIC_LIB) 
+    add_definitions(-DPTW32_STATIC_LIB)
     add_subdirectory(pthread)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
     add_library(pthread STATIC IMPORTED GLOBAL)
@@ -228,12 +228,12 @@ endif()
 
 # iconv
 if(${BUILD_WITH_ICONV})
-    add_subdirectory(iconv)
+    add_subdirectory(iconv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_ICONV})
 
 # wingetopt
 if(${BUILD_WINGETOPT})
-    add_subdirectory(wingetopt)
+    add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
 endif(${BUILD_WINGETOPT})
 
 # msvcregex
@@ -293,7 +293,7 @@ if(${BUILD_WITH_UV})
         MESSAGE("Windows need set no-sign-compare")
         add_compile_options(-Wno-sign-compare)
     endif ()
-    add_subdirectory(libuv)
+    add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})
 
 # BDB
@@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE})
 # Build test
 # ================================================================================================
 if(${BUILD_DEPENDENCY_TESTS})
-    add_subdirectory(test)
+    add_subdirectory(test EXCLUDE_FROM_ALL)
 endif(${BUILD_DEPENDENCY_TESTS})

include/client/taos.h

@@ -121,13 +121,14 @@ typedef struct setConfRet {
 DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
+DLL_EXPORT int        taos_init(void);
 DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
-DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
+DLL_EXPORT TAOS      *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
-DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
-DLL_EXPORT void  taos_close(TAOS *taos);
+DLL_EXPORT TAOS      *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
+DLL_EXPORT void       taos_close(TAOS *taos);
 
-const char *taos_data_type(int type);
+const char            *taos_data_type(int type);
 
 DLL_EXPORT TAOS_STMT  *taos_stmt_init(TAOS *taos);
 DLL_EXPORT int         taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);

include/common/tmsg.h

@@ -438,6 +438,7 @@ typedef struct {
 
 int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
 int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
+void    tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
 
 typedef struct {
   int16_t colId;     // column id
@@ -1332,7 +1333,7 @@ int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicR
 
 typedef struct {
   int64_t consumerId;
-} SMqConsumerLostMsg;
+} SMqConsumerLostMsg, SMqConsumerRecoverMsg;
 
 typedef struct {
   int64_t consumerId;
@@ -1537,6 +1538,10 @@ int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq);
 
 typedef struct SVDropStbReq {
   // data
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
+
 } SVDropStbReq;
 
 typedef struct SVCreateStbRsp {
@@ -2116,7 +2121,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapp
   if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
   if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
 
-  pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols);
+  pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols);
   if (pSW->pSchema == NULL) return -1;
   for (int32_t i = 0; i < pSW->nCols; i++) {
     if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;

include/common/tmsgdef.h

@@ -145,10 +145,11 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)

include/dnode/mnode/sdb/sdb.h

@@ -18,6 +18,11 @@
 
 #include "os.h"
 
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -135,7 +140,7 @@ typedef enum {
 typedef struct SSdb SSdb;
 typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
-typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj);
+typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
@@ -326,9 +331,29 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
+void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
+typedef struct SSdb {
+  SMnode     *pMnode;
+  char       *currDir;
+  char       *syncDir;
+  char       *tmpDir;
+  int64_t     lastCommitVer;
+  int64_t     curVer;
+  int64_t     tableVer[SDB_MAX];
+  int64_t     maxId[SDB_MAX];
+  EKeyType    keyTypes[SDB_MAX];
+  SHashObj   *hashObjs[SDB_MAX];
+  SRWLatch    locks[SDB_MAX];
+  SdbInsertFp insertFps[SDB_MAX];
+  SdbUpdateFp updateFps[SDB_MAX];
+  SdbDeleteFp deleteFps[SDB_MAX];
+  SdbDeployFp deployFps[SDB_MAX];
+  SdbEncodeFp encodeFps[SDB_MAX];
+  SdbDecodeFp decodeFps[SDB_MAX];
+} SSdb;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/executor/executor.h

@@ -32,6 +32,7 @@ typedef struct SReadHandle {
   void* reader;
   void* meta;
   void* config;
+  void* vnode;
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1

include/os/os.h

@@ -59,6 +59,7 @@ extern "C" {
 #include <winsock.h>
 #endif
 
+#define __typeof(a) auto
 
 #endif
 

include/os/osMath.h

@@ -23,27 +23,21 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
+#define TSWAP(a, b)             \
+  do {                          \
+    __typeof(a) __tmp = (a); \
+    (a) = (b);               \
+    (b) = __tmp;                \
+  } while (0)
+
 #ifdef WINDOWS
 
-  #define TSWAP(a, b, c) \
-    do {                \
-      c __tmp = (c)(a); \
-      (a) = (c)(b);     \
-      (b) = __tmp;      \
-    } while (0)
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
   #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 
-  #define TSWAP(a, b, c)       \
-    do {                       \
-      __typeof(a) __tmp = (a); \
-      (a) = (b);               \
-      (b) = __tmp;             \
-    } while (0)
-
   #define TMAX(a, b)             \
     ({                           \
       __typeof(a) __a = (a);     \
@@ -51,12 +45,12 @@ extern "C" {
       (__a > __b) ? __a : __b;   \
     })
 
-  #define TMIN(a, b)             \
-    ({                           \
-      __typeof(a) __a = (a);     \
-      __typeof(b) __b = (b);     \
-      (__a < __b) ? __a : __b;   \
-    })
+#define TMIN(a, b)             \
+  ({                           \
+    __typeof(a) __a = (a);     \
+    __typeof(b) __b = (b);     \
+    (__a < __b) ? __a : __b;   \
+  })
 
 #define TRANGE(a, b, c) \
   ({                    \

include/os/osSocket.h

@@ -19,53 +19,53 @@
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-  #define socket SOCKET_FUNC_TAOS_FORBID
-  #define bind   BIND_FUNC_TAOS_FORBID
-  #define listen LISTEN_FUNC_TAOS_FORBID
-  #define accept ACCEPT_FUNC_TAOS_FORBID
-  #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
-  #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
-  #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
-  #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
-  #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
+#define socket       SOCKET_FUNC_TAOS_FORBID
+#define bind         BIND_FUNC_TAOS_FORBID
+#define listen       LISTEN_FUNC_TAOS_FORBID
+#define accept       ACCEPT_FUNC_TAOS_FORBID
+#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+#define epoll_ctl    EPOLL_CTL_FUNC_TAOS_FORBID
+#define epoll_wait   EPOLL_WAIT_FUNC_TAOS_FORBID
+#define inet_addr    INET_ADDR_FUNC_TAOS_FORBID
+#define inet_ntoa    INET_NTOA_FUNC_TAOS_FORBID
 #endif
 
 #if defined(WINDOWS)
-  #if BYTE_ORDER == LITTLE_ENDIAN
-    #include <stdlib.h>
-    #define htobe16(x) _byteswap_ushort(x)
-    #define htole16(x) (x)
-    #define be16toh(x) _byteswap_ushort(x)
-    #define le16toh(x) (x)
-                      
-    #define htobe32(x) _byteswap_ulong(x)
-    #define htole32(x) (x)
-    #define be32toh(x) _byteswap_ulong(x)
-    #define le32toh(x) (x)
-                      
-    #define htobe64(x) _byteswap_uint64(x)
-    #define htole64(x) (x)
-    #define be64toh(x) _byteswap_uint64(x)
-    #define le64toh(x) (x)
-  #else
-    #error byte order not supported
-  #endif
-
-  #define __BYTE_ORDER    BYTE_ORDER
-  #define __BIG_ENDIAN    BIG_ENDIAN
-  #define __LITTLE_ENDIAN LITTLE_ENDIAN
-  #define __PDP_ENDIAN    PDP_ENDIAN
+#if BYTE_ORDER == LITTLE_ENDIAN
+#include <stdlib.h>
+#define htobe16(x) _byteswap_ushort(x)
+#define htole16(x) (x)
+#define be16toh(x) _byteswap_ushort(x)
+#define le16toh(x) (x)
+
+#define htobe32(x) _byteswap_ulong(x)
+#define htole32(x) (x)
+#define be32toh(x) _byteswap_ulong(x)
+#define le32toh(x) (x)
+
+#define htobe64(x) _byteswap_uint64(x)
+#define htole64(x) (x)
+#define be64toh(x) _byteswap_uint64(x)
+#define le64toh(x) (x)
+#else
+#error byte order not supported
+#endif
+
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
 
 #else
-  #include <netinet/in.h>
-  #include <sys/socket.h>
-
-  #if defined(_TD_DARWIN_64)
-    #include <osEok.h>
-  #else
-    #include <netinet/in.h>
-    #include <sys/epoll.h>
-  #endif
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+#if defined(_TD_DARWIN_64)
+#include <osEok.h>
+#else
+#include <netinet/in.h>
+#include <sys/epoll.h>
+#endif
 #endif
 
 #ifdef __cplusplus
@@ -73,24 +73,24 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  typedef int socklen_t;
-  #define TAOS_EPOLL_WAIT_TIME 100
-  typedef SOCKET eventfd_t;
-  #define eventfd(a, b) -1
-  #define EpollClose(pollFd) epoll_close(pollFd)
-  #ifndef EPOLLWAKEUP
-    #define EPOLLWAKEUP (1u << 29)
-  #endif
+typedef int socklen_t;
+#define TAOS_EPOLL_WAIT_TIME 100
+typedef SOCKET eventfd_t;
+#define eventfd(a, b)      -1
+#define EpollClose(pollFd) epoll_close(pollFd)
+#ifndef EPOLLWAKEUP
+#define EPOLLWAKEUP (1u << 29)
+#endif
 #elif defined(_TD_DARWIN_64)
-  #define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
-  #define EpollClose(pollFd) epoll_close(pollFd)
+#define TAOS_EPOLL_WAIT_TIME 500
+typedef int32_t SOCKET;
+typedef SOCKET  EpollFd;
+#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
-  #define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
-  #define EpollClose(pollFd) taosCloseSocket(pollFd)
+#define TAOS_EPOLL_WAIT_TIME 500
+typedef int32_t SOCKET;
+typedef SOCKET  EpollFd;
+#define EpollClose(pollFd)   taosCloseSocket(pollFd)
 #endif
 
 #if defined(_TD_DARWIN_64)
@@ -119,8 +119,8 @@ extern "C" {
 #define __PDP_ENDIAN    PDP_ENDIAN
 #endif
 
-typedef int32_t SocketFd;
-typedef SocketFd  EpollFd;
+typedef int32_t  SocketFd;
+typedef SocketFd EpollFd;
 
 typedef struct TdSocket {
 #if SOCKET_WITH_LOCK
@@ -128,16 +128,17 @@ typedef struct TdSocket {
 #endif
   int      refId;
   SocketFd fd;
-} *TdSocketPtr, TdSocket;
+} * TdSocketPtr, TdSocket;
 
 typedef struct TdSocketServer *TdSocketServerPtr;
-typedef struct TdSocket *TdSocketPtr;
-typedef struct TdEpoll *TdEpollPtr;
+typedef struct TdSocket *      TdSocketPtr;
+typedef struct TdEpoll *       TdEpollPtr;
 
-int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr,
+                           int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -154,30 +155,32 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
 int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
-void taosWinSocketInit();
+void    taosWinSocketInit();
+
+int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
 
-TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
-TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
+TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
+TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
-int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
-TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
-
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen);
-
-void     taosBlockSIGPIPE();
-uint32_t taosGetIpv4FromFqdn(const char *);
-int32_t  taosGetFqdn(char *);
-void     tinet_ntoa(char *ipstr, uint32_t ip);
-uint32_t ip2uint(const char *const ip_addr);
-void     taosIgnSIGPIPE();
-void     taosSetMaskSIGPIPE();
-uint32_t taosInetAddr(const char *ipAddr);
+int32_t           taosKeepTcpAlive(TdSocketPtr pSocket);
+TdSocketPtr       taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
+
+int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
+
+void        taosBlockSIGPIPE();
+uint32_t    taosGetIpv4FromFqdn(const char *);
+int32_t     taosGetFqdn(char *);
+void        tinet_ntoa(char *ipstr, uint32_t ip);
+uint32_t    ip2uint(const char *const ip_addr);
+void        taosIgnSIGPIPE();
+void        taosSetMaskSIGPIPE();
+uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
 TdEpollPtr taosCreateEpoll(int32_t size);
-int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
-int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
-int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
+int32_t    taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
+int32_t    taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
+int32_t    taosCloseEpoll(TdEpollPtr *ppEpoll);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -62,6 +62,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_APP_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0014)
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0015)
 #define TSDB_CODE_RPC_INVALID_VERSION           TAOS_DEF_ERROR_CODE(0, 0x0016)
+#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017)
 
 //common & util
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0100)
@@ -268,14 +269,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOPIC_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03E1)
 #define TSDB_CODE_MND_TOO_MANY_TOPICS           TAOS_DEF_ERROR_CODE(0, 0x03E2)
 #define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x03E3)
-#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E4)
-#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E5)
-#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB    TAOS_DEF_ERROR_CODE(0, 0x03E6)
-#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
-#define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
-#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
-#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
-#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
+#define TSDB_CODE_MND_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x03E4)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E5)
+#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E6)
+#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E7)
+#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)

include/util/tarray.h

@@ -205,7 +205,6 @@ SArray* taosArrayDup(const SArray* pSrc);
  */
 SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
 
-
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -272,6 +271,8 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
 int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
 void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
 
+char* taosShowStrArray(const SArray* pArray);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -131,6 +131,7 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_PERFS_TABLE_TOPICS        "topics"
 #define TSDB_PERFS_TABLE_CONSUMERS     "consumers"
 #define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
+#define TSDB_PERFS_TABLE_OFFSETS       "offsets"
 
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"

include/util/tencode.h

@@ -79,31 +79,52 @@ typedef struct {
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
-#define TCODER_MALLOC(PCODER, SIZE)                                         \
-  ({                                                                        \
-    void*      ptr = NULL;                                                  \
-    SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
-    if (pMem) {                                                             \
-      pMem->next = (PCODER)->mList;                                         \
-      (PCODER)->mList = pMem;                                               \
-      ptr = (void*)&pMem[1];                                                \
-    }                                                                       \
-    ptr;                                                                    \
-  })
-
-#define tEncodeSize(E, S, SIZE)                                \
-  ({                                                           \
+// #define TCODER_MALLOC(PCODER, SIZE)                                         \
+//   ({                                                                        \
+//     void*      ptr = NULL;                                                  \
+//     SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
+//     if (pMem) {                                                             \
+//       pMem->next = (PCODER)->mList;                                         \
+//       (PCODER)->mList = pMem;                                               \
+//       ptr = (void*)&pMem[1];                                                \
+//     }                                                                       \
+//     ptr;                                                                    \
+//   })
+static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) {
+  void*      ptr = NULL;
+  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size);
+  if (pMem) {
+    pMem->next = pCoder->mList;
+    pCoder->mList = pMem;
+    ptr = (void*)&pMem[1];
+  }
+  return ptr;
+}
+
+#define tEncodeSize(E, S, SIZE, RET)                           \
+  do{                                                          \
     SCoder coder = {0};                                        \
-    int    ret = 0;                                            \
     tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
     if ((E)(&coder, S) == 0) {                                 \
       SIZE = coder.pos;                                        \
     } else {                                                   \
-      ret = -1;                                                \
+      RET = -1;                                                \
     }                                                          \
     tCoderClear(&coder);                                       \
-    ret;                                                       \
-  })
+  }while(0)
+// #define tEncodeSize(E, S, SIZE)                                \
+//   ({                                                           \
+//     SCoder coder = {0};                                        \
+//     int    ret = 0;                                            \
+//     tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
+//     if ((E)(&coder, S) == 0) {                                 \
+//       SIZE = coder.pos;                                        \
+//     } else {                                                   \
+//       ret = -1;                                                \
+//     }                                                          \
+//     tCoderClear(&coder);                                       \
+//     ret;                                                       \
+//   })
 
 void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
 void tCoderClear(SCoder* pCoder);

source/client/inc/clientInt.h

@@ -254,8 +254,6 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int taos_init();
-
 void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);

source/client/src/clientImpl.c

@@ -103,6 +103,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
 
   if (port) {
     epSet.epSet.eps[0].port = port;
+    epSet.epSet.eps[1].port = port;
   }
 
   char* key = getClusterKey(user, secretEncrypt, ip, port);
@@ -187,8 +188,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
       setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
     }
 
-    TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
-    TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
+    TSWAP(pRequest->dbList, (*pQuery)->pDbList);
+    TSWAP(pRequest->tableList, (*pQuery)->pTableList);
   }
 
   return code;
@@ -245,7 +246,6 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   ASSERT(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
-  // TODO handle memory leak
   if (pResInfo->fields != NULL) {
     taosMemoryFree(pResInfo->fields);
   }
@@ -358,8 +358,15 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   SQuery*      pQuery = NULL;
 
   int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
-  if (TSDB_CODE_SUCCESS == code) {
-    code = parseSql(pRequest, false, &pQuery, NULL);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    return NULL;
+  }
+
+  code = parseSql(pRequest, false, &pQuery, NULL);
+  if (code != TSDB_CODE_SUCCESS) {
+    pRequest->code = code;
+    return pRequest;
   }
 
   return launchQueryImpl(pRequest, pQuery, code, false);
@@ -410,7 +417,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
     pRequest = launchQuery(pTscObj, sql, sqlLen);
-    if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
+    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
     }
 

source/client/src/taos.def

 taos_cleanup
 taos_options
 taos_set_config
+taos_init
 taos_connect
 taos_connect_l
 taos_connect_auth

source/client/src/tmq.c

@@ -666,7 +666,6 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   code = param.rspErr;
   if (code != 0) goto FAIL;
 
-  // TODO: add max retry cnt
   while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
     tscDebug("not ready, retry");
     taosMsleep(500);
@@ -683,7 +682,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   code = 0;
 FAIL:
   if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
-  if (code != 0) {
+  if (code != 0 && buf) {
     taosMemoryFree(buf);
   }
   return code;
@@ -1265,6 +1264,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
     return (TAOS_RES*)rspObj;
   }
 
+  // in no topic status also need process delayed task
   if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
     return NULL;
   }
@@ -1285,6 +1285,9 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
         return NULL;
       }
       tsem_timewait(&tmq->rspSem, leftTime * 1000);
+    } else {
+      // use tsem_timewait instead of tsem_wait to avoid unexpected stuck
+      tsem_timewait(&tmq->rspSem, 500 * 1000);
     }
   }
 }

source/common/src/tdatablock.c

@@ -117,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
   int32_t type = pColumnInfoData->info.type;
   if (IS_VAR_DATA_TYPE(type)) {
     int32_t dataLen = varDataTLen(pData);
-    if(type == TSDB_DATA_TYPE_JSON) {
-      if(*pData == TSDB_DATA_TYPE_NULL) {
+    if (type == TSDB_DATA_TYPE_JSON) {
+      if (*pData == TSDB_DATA_TYPE_NULL) {
         dataLen = 0;
-      }else if(*pData == TSDB_DATA_TYPE_NCHAR) {
-        dataLen = varDataTLen(pData+CHAR_BYTES);
-      }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData + CHAR_BYTES);
+      } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
         dataLen = LONG_BYTES;
-      }else if(*pData == TSDB_DATA_TYPE_BOOL) {
+      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
         dataLen = CHAR_BYTES;
       }
       dataLen += CHAR_BYTES;
     }
+
     SVarColAttr* pAttr = &pColumnInfoData->varmeta;
     if (pAttr->allocLen < pAttr->length + dataLen) {
       uint32_t newSize = pAttr->allocLen;
-      if (newSize == 0) {
+      if (newSize <= 1) {
         newSize = 8;
       }
 
@@ -224,12 +225,16 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     // Handle the bitmap
     char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
     if (p == NULL) {
-      // TODO
+      return TSDB_CODE_OUT_OF_MEMORY;
     }
 
     pColumnInfoData->varmeta.offset = (int32_t*)p;
     for (int32_t i = 0; i < numOfRow2; ++i) {
-      pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
+      if (pSource->varmeta.offset[i] == -1) {
+        pColumnInfoData->varmeta.offset[i + numOfRow1] = -1;
+      } else {
+        pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
+      }
     }
 
     // copy data
@@ -238,7 +243,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     if (pColumnInfoData->varmeta.allocLen < len + oldLen) {
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen);
       if (tmp == NULL) {
-        return TSDB_CODE_VND_OUT_OF_MEMORY;
+        return TSDB_CODE_OUT_OF_MEMORY;
       }
 
       pColumnInfoData->pData = tmp;

source/common/src/tmsg.c

@@ -1302,6 +1302,11 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *
   return 0;
 }
 
+void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
+  taosHashCleanup(pRsp->readDbs);
+  taosHashCleanup(pRsp->writeDbs);
+}
+
 int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@@ -3226,7 +3231,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq
 int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
   if (tStartDecode(decoder) < 0) return -1;
   if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
-  pReq->offsets = (SMqOffset *)TCODER_MALLOC(decoder, sizeof(SMqOffset) * pReq->num);
+  pReq->offsets = (SMqOffset *)tCoderMalloc(decoder, sizeof(SMqOffset) * pReq->num);
   if (pReq->offsets == NULL) return -1;
   for (int32_t i = 0; i < pReq->num; i++) {
     tDecodeSMqOffset(decoder, &pReq->offsets[i]);
@@ -3509,7 +3514,7 @@ int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) {
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1;
-  pRsp->pRsps = (SVCreateTbRsp *)TCODER_MALLOC(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps);
+  pRsp->pRsps = (SVCreateTbRsp *)tCoderMalloc(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps);
   for (int32_t i = 0; i < pRsp->nRsps; i++) {
     if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1;
   }
@@ -3738,7 +3743,7 @@ int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) {
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1;
-  pReq->pReqs = (SVCreateTbReq *)TCODER_MALLOC(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs);
+  pReq->pReqs = (SVCreateTbReq *)tCoderMalloc(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs);
   if (pReq->pReqs == NULL) return -1;
   for (int iReq = 0; iReq < pReq->nReqs; iReq++) {
     if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;

source/common/src/ttypes.c

@@ -651,35 +651,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf
   switch (type) {
     case TSDB_DATA_TYPE_INT:
     case TSDB_DATA_TYPE_UINT: {
-      TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
+      TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_UBIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t);
+      TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight));
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
-      TSWAP(*(double *)(pLeft), *(double *)(pRight), double);
+      TSWAP(*(double *)(pLeft), *(double *)(pRight));
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT:
     case TSDB_DATA_TYPE_USMALLINT: {
-      TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t);
+      TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_FLOAT: {
-      TSWAP(*(float *)(pLeft), *(float *)(pRight), float);
+      TSWAP(*(float *)(pLeft), *(float *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_UTINYINT: {
-      TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t);
+      TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight));
       break;
     }
 

source/dnode/mgmt/exe/dmMain.c

@@ -69,8 +69,8 @@ static void dmSetSignalHandle() {
 static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
   int32_t cmdEnvIndex = 0;
   if (argc < 2) return 0;
-  global.envCmd = taosMemoryMalloc(argc-1);
-  memset(global.envCmd, 0, argc-1);
+  global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*));
+  memset(global.envCmd, 0, (argc-1)*sizeof(char*));
   for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
       if (i < argc - 1) {

source/dnode/mgmt/implement/src/dmEps.c

@@ -120,7 +120,7 @@ int32_t dmReadEps(SDnode *pDnode) {
       goto PRASE_DNODE_OVER;
     }
 
-    dnodeEp.id = dnodeId->valueint;
+    dnodeEp.id = did->valueint;
 
     cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
     if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
@@ -156,11 +156,6 @@ PRASE_DNODE_OVER:
   if (root != NULL) cJSON_Delete(root);
   if (pFile != NULL) taosCloseFile(&pFile);
 
-  if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
-    dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
-    return -1;
-  }
-
   if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) {
     SDnodeEp dnodeEp = {0};
     dnodeEp.isMnode = 1;
@@ -170,6 +165,11 @@ PRASE_DNODE_OVER:
 
   dmResetEps(pDnode, pDnode->data.dnodeEps);
 
+  if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
+    dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
+    return -1;
+  }
+
   terrno = code;
   return code;
 }
@@ -291,13 +291,17 @@ static void dmPrintEps(SDnode *pDnode) {
 
 static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) {
   bool changed = false;
+  if (dnodeId == 0) return changed;
   taosRLockLatch(&pDnode->data.latch);
 
   SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t));
   if (pDnodeEp != NULL) {
-    char epstr[TSDB_EP_LEN + 1];
+    char epstr[TSDB_EP_LEN + 1] = {0};
     snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    changed = strcmp(ep, epstr) != 0;
+    changed = (strcmp(ep, epstr) != 0);
+    if (changed) {
+      dError("dnode:%d, localEp %s different from %s", dnodeId, ep, epstr);
+    }
   }
 
   taosRUnLockLatch(&pDnode->data.latch);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -114,7 +114,6 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->tsdbCfg.keep2 = 3650;  // pCreate->daysToKeep0;
   pCfg->tsdbCfg.keep0 = 3650;  // pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep1 = 3650;  // pCreate->daysToKeep0;
-  pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -23,10 +23,8 @@ extern "C" {
 #endif
 
 enum {
-  // MQ_CONSUMER_STATUS__INIT = 1,
   MQ_CONSUMER_STATUS__MODIFY = 1,
   MQ_CONSUMER_STATUS__MODIFY_IN_REB,
-  // MQ_CONSUMER_STATUS__IDLE,
   MQ_CONSUMER_STATUS__READY,
   MQ_CONSUMER_STATUS__LOST,
   MQ_CONSUMER_STATUS__LOST_IN_REB,

source/dnode/mnode/impl/inc/mndDef.h

@@ -89,6 +89,7 @@ typedef enum {
   TRN_TYPE_DROP_STREAM = 1020,
   TRN_TYPE_ALTER_STREAM = 1021,
   TRN_TYPE_CONSUMER_LOST = 1022,
+  TRN_TYPE_CONSUMER_RECOVER = 1023,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -126,8 +127,6 @@ typedef enum {
   DND_REASON_OTHERS
 } EDndReason;
 
-typedef void (*TransCbFp)(SMnode* pMnode, void* param);
-
 typedef struct {
   int32_t    id;
   ETrnStage  stage;
@@ -150,8 +149,10 @@ typedef struct {
   int64_t    dbUid;
   char       dbname[TSDB_DB_FNAME_LEN];
   char       lastError[TSDB_TRANS_ERROR_LEN];
-  TransCbFp  transCbFp;
-  void*      transCbParam;
+  int32_t    startFunc;
+  int32_t    stopFunc;
+  int32_t    paramLen;
+  void*      param;
 } STrans;
 
 typedef struct {
@@ -463,12 +464,14 @@ enum {
   CONSUMER_UPDATE__ADD,
   CONSUMER_UPDATE__REMOVE,
   CONSUMER_UPDATE__LOST,
+  CONSUMER_UPDATE__RECOVER,
   CONSUMER_UPDATE__MODIFY,
 };
 
 typedef struct {
   int64_t consumerId;
   char    cgroup[TSDB_CGROUP_LEN];
+  char    appId[TSDB_CGROUP_LEN];
   int8_t  updateType;  // used only for update
   int32_t epoch;
   int32_t status;
@@ -479,6 +482,17 @@ typedef struct {
   SArray*  currentTopics;     // SArray<char*>
   SArray*  rebNewTopics;      // SArray<char*>
   SArray*  rebRemovedTopics;  // SArray<char*>
+
+  // subscribed by user
+  SArray* assignedTopics;  // SArray<char*>
+
+  // data for display
+  int32_t pid;
+  SEpSet  ep;
+  int64_t upTime;
+  int64_t subscribeTime;
+  int64_t rebalanceTime;
+
 } SMqConsumerObj;
 
 SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);

source/dnode/mnode/impl/inc/mndTrans.h

@@ -33,6 +33,15 @@ typedef struct {
   void   *pCont;
 } STransAction;
 
+typedef enum {
+  TEST_TRANS_START_FUNC = 1,
+  TEST_TRANS_STOP_FUNC = 2,
+  CONSUME_TRANS_START_FUNC = 3,
+  CONSUME_TRANS_STOP_FUNC = 4,
+} ETrnFuncType;
+
+typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
+
 int32_t mndInitTrans(SMnode *pMnode);
 void    mndCleanupTrans(SMnode *pMnode);
 
@@ -44,7 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
 int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
 void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
-void    mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param);
+void    mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen);
 void    mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -37,6 +37,8 @@
 
 static int8_t mqInRebFlag = 0;
 
+static const char *mndConsumerStatusName(int status);
+
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
@@ -48,6 +50,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
 static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
 static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
 static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
+static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg);
 
 int32_t mndInitConsumer(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_CONSUMER,
@@ -62,6 +65,11 @@ int32_t mndInitConsumer(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
+
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer);
+
   return sdbSetTable(pMnode->pSdb, table);
 }
 
@@ -91,6 +99,30 @@ FAIL:
   return -1;
 }
 
+static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg) {
+  SMnode                *pMnode = pMsg->pNode;
+  SMqConsumerRecoverMsg *pRecoverMsg = pMsg->rpcMsg.pCont;
+  SMqConsumerObj        *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
+  ASSERT(pConsumer);
+
+  SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
+  pConsumerNew->updateType = CONSUMER_UPDATE__RECOVER;
+
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_RECOVER, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto FAIL;
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+FAIL:
+  tDeleteSMqConsumerObj(pConsumerNew);
+  mndTransDrop(pTrans);
+  return -1;
+}
+
 static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1);
   if (pRebSub == NULL) {
@@ -216,8 +248,15 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   // 1. check consumer status
   int32_t status = atomic_load_32(&pConsumer->status);
 
-  if (status == MQ_CONSUMER_STATUS__LOST) {
-    // TODO: recover consumer
+  if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
+    SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
+
+    pRecoverMsg->consumerId = consumerId;
+    SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
+    pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER;
+    pRpcMsg->pCont = pRecoverMsg;
+    pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg);
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
   }
 
   if (status != MQ_CONSUMER_STATUS__READY) {
@@ -366,10 +405,14 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   if (pConsumerOld == NULL) {
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
-    /*pConsumerNew->waitingRebTopics = newSub;*/
     pConsumerNew->rebNewTopics = newSub;
     subscribe.topicNames = NULL;
 
+    for (int32_t i = 0; i < newTopicNum; i++) {
+      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
+    }
+
     STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
     if (pTrans == NULL) goto SUBSCRIBE_OVER;
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
@@ -389,7 +432,11 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
       goto SUBSCRIBE_OVER;
     }
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
-    /*pConsumerOld->waitingRebTopics = newSub;*/
+
+    for (int32_t i = 0; i < newTopicNum; i++) {
+      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
+    }
 
     int32_t oldTopicNum = 0;
     if (pConsumerOld->currentTopics) {
@@ -532,6 +579,7 @@ CM_DECODE_OVER:
 
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId);
+  pConsumer->subscribeTime = pConsumer->upTime;
   return 0;
 }
 
@@ -557,17 +605,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
     pNewConsumer->rebRemovedTopics = tmp;
 
+    tmp = pOldConsumer->assignedTopics;
+    pOldConsumer->assignedTopics = pNewConsumer->assignedTopics;
+    pNewConsumer->assignedTopics = tmp;
+
+    pOldConsumer->subscribeTime = pNewConsumer->upTime;
+
     pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
+
     int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
-    pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+    /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
     for (int32_t i = 0; i < sz; i++) {
       char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
-      taosArrayPush(pNewConsumer->rebRemovedTopics, &topic);
+      taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
+    ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+
+    int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
+      taosArrayPush(pOldConsumer->rebNewTopics, &topic);
+    }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
+    pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
     ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
     ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
@@ -612,6 +688,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
     ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
@@ -668,6 +747,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   }
 
@@ -688,3 +770,104 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
   SSdb *pSdb = pMnode->pSdb;
   sdbRelease(pSdb, pConsumer);
 }
+
+static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
+  SMnode         *pMnode = pReq->pNode;
+  SSdb           *pSdb = pMnode->pSdb;
+  int32_t         numOfRows = 0;
+  SMqConsumerObj *pConsumer = NULL;
+
+  while (numOfRows < rowsCapacity) {
+    pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
+    if (pShow->pIter == NULL) break;
+
+    SColumnInfoData *pColInfo;
+    int32_t          cols = 0;
+
+    taosRLockLatch(&pConsumer->lock);
+
+    // consumer id
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
+
+    // group id
+    char groupId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(groupId), pConsumer->cgroup, TSDB_CGROUP_LEN);
+    varDataSetLen(groupId, strlen(varDataVal(groupId)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)groupId, false);
+
+    // app id
+    char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN);
+    varDataSetLen(appId, strlen(varDataVal(appId)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)appId, false);
+
+    // status
+    char status[20 + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
+    varDataSetLen(status, strlen(varDataVal(status)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)status, false);
+
+    // subscribed topics
+    char  topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
+    char *showStr = taosShowStrArray(pConsumer->assignedTopics);
+    tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);
+    taosMemoryFree(showStr);
+    varDataSetLen(topics, strlen(varDataVal(topics)));
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)topics, false);
+
+    // pid
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true);
+
+    // end point
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);
+
+    // up time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
+
+    // subscribe time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
+
+    // rebalance time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
+
+    taosRUnLockLatch(&pConsumer->lock);
+    sdbRelease(pSdb, pConsumer);
+
+    numOfRows++;
+  }
+
+  pShow->numOfRows += numOfRows;
+  return numOfRows;
+}
+
+static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}
+
+static const char *mndConsumerStatusName(int status) {
+  switch (status) {
+    case MQ_CONSUMER_STATUS__READY:
+      return "ready";
+    case MQ_CONSUMER_STATUS__LOST:
+    case MQ_CONSUMER_STATUS__LOST_REBD:
+    case MQ_CONSUMER_STATUS__LOST_IN_REB:
+      return "lost";
+    case MQ_CONSUMER_STATUS__MODIFY:
+    case MQ_CONSUMER_STATUS__MODIFY_IN_REB:
+      return "rebalancing";
+    default:
+      return "unknown";
+  }
+}

source/dnode/mnode/impl/src/mndDef.c

@@ -34,15 +34,20 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L
   pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
   pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
   pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->assignedTopics = taosArrayInit(0, sizeof(void *));
 
-  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) {
+  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL ||
+      pConsumer->assignedTopics == NULL) {
     taosArrayDestroy(pConsumer->currentTopics);
     taosArrayDestroy(pConsumer->rebNewTopics);
     taosArrayDestroy(pConsumer->rebRemovedTopics);
+    taosArrayDestroy(pConsumer->assignedTopics);
     taosMemoryFree(pConsumer);
     return NULL;
   }
 
+  pConsumer->upTime = taosGetTimestampMs();
+
   return pConsumer;
 }
 
@@ -56,6 +61,9 @@ void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
   if (pConsumer->rebRemovedTopics) {
     taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
   }
+  if (pConsumer->assignedTopics) {
+    taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
+  }
 }
 
 int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
@@ -67,6 +75,12 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
   tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
   tlen += taosEncodeFixedI32(buf, pConsumer->status);
 
+  tlen += taosEncodeFixedI32(buf, pConsumer->pid);
+  tlen += taosEncodeSEpSet(buf, &pConsumer->ep);
+  tlen += taosEncodeFixedI64(buf, pConsumer->upTime);
+  tlen += taosEncodeFixedI64(buf, pConsumer->subscribeTime);
+  tlen += taosEncodeFixedI64(buf, pConsumer->rebalanceTime);
+
   // current topics
   if (pConsumer->currentTopics) {
     sz = taosArrayGetSize(pConsumer->currentTopics);
@@ -103,6 +117,18 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
     tlen += taosEncodeFixedI32(buf, 0);
   }
 
+  // lost topics
+  if (pConsumer->assignedTopics) {
+    sz = taosArrayGetSize(pConsumer->assignedTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->assignedTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
   return tlen;
 }
 
@@ -114,6 +140,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
   buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
   buf = taosDecodeFixedI32(buf, &pConsumer->status);
 
+  buf = taosDecodeFixedI32(buf, &pConsumer->pid);
+  buf = taosDecodeSEpSet(buf, &pConsumer->ep);
+  buf = taosDecodeFixedI64(buf, &pConsumer->upTime);
+  buf = taosDecodeFixedI64(buf, &pConsumer->subscribeTime);
+  buf = taosDecodeFixedI64(buf, &pConsumer->rebalanceTime);
+
   // current topics
   buf = taosDecodeFixedI32(buf, &sz);
   pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
@@ -141,6 +173,15 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
     taosArrayPush(pConsumer->rebRemovedTopics, &topic);
   }
 
+  // reb removed topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->assignedTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->assignedTopics, &topic);
+  }
+
   return (void *)buf;
 }
 
@@ -329,6 +370,7 @@ int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEnt
   tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
   return tlen;
 }
+
 void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
   buf = taosDecodeFixedI32(buf, &pEntry->epoch);
   buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -142,6 +142,7 @@ static const SInfosTableSchema userTblsSchema[] = {
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
 };
 
 static const SInfosTableSchema userTblDistSchema[] = {
@@ -164,7 +165,6 @@ static const SInfosTableSchema userUsersSchema[] = {
     {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
 };
 
 static const SInfosTableSchema grantsSchema[] = {
@@ -199,23 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = {
     {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
 };
 
-static const SInfosTableSchema consumerSchema[] = {
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    // ep
-    // up time
-    // topics
-};
-
-static const SInfosTableSchema subscribeSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
 static const SInfosTableSchema smaSchema[] = {
     {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
@@ -282,8 +265,6 @@ static const SInfosTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
     {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
     {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
-    {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
-    {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
     {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},
     {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
     {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)},

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -49,13 +49,15 @@ static const SPerfsTableSchema topicSchema[] = {
 
 static const SPerfsTableSchema consumerSchema[] = {
     {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
 
 static const SPerfsTableSchema subscriptionSchema[] = {
@@ -63,6 +65,12 @@ static const SPerfsTableSchema subscriptionSchema[] = {
     {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
+};
+
+static const SPerfsTableSchema offsetSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
@@ -74,6 +82,7 @@ static const SPerfsTableMeta perfsMeta[] = {
     {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
     {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
     {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
+    {TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
 };
 
 // connection/application/

source/dnode/mnode/impl/src/mndProfile.c

@@ -196,7 +196,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
     goto CONN_OVER;
   }
   if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
-    mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass);
+    mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->user, connReq.passwd, pUser->pass);
     code = TSDB_CODE_RPC_AUTH_FAILURE;
     goto CONN_OVER;
   }

source/dnode/mnode/impl/src/mndScheduler.c

@@ -489,7 +489,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
     if (levelNum != 1) {
       qDestroyQueryPlan(pPlan);
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+      terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
       return -1;
     }
 
@@ -498,7 +498,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     int32_t opNum = LIST_LENGTH(inner->pNodeList);
     if (opNum != 1) {
       qDestroyQueryPlan(pPlan);
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+      terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
       return -1;
     }
     plan = nodesListGetNode(inner->pNodeList, 0);

source/dnode/mnode/impl/src/mndStb.c

@@ -395,7 +395,9 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
     }
   }
   // get length
-  if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) {
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret);
+  if (ret < 0) {
     return NULL;
   }
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -261,7 +261,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
 
 static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) {
-    terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
+    terrno = TSDB_CODE_MND_INVALID_TOPIC;
     return -1;
   }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans);
 static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
 static int32_t  mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
 static int32_t  mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
-static int32_t  mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
+static int32_t  mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc);
 
 static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
 static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
@@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER)
   }
 
+  SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER)
+  if (pTrans->param != NULL) {
+    SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER)
+  }
+
   SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER)
   SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER)
 
@@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
     action.pCont = NULL;
   }
 
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER)
+  if (pTrans->paramLen != 0) {
+    pTrans->param = taosMemoryMalloc(pTrans->paramLen);
+    SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER);
+  }
+
   SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER)
 
   terrno = 0;
@@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) {
   }
 }
 
+static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) {
+  mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen);
+}
+
+static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) {
+  mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen);
+}
+
+static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
+  switch (ftype) {
+    case TEST_TRANS_START_FUNC:
+      return mndTransTestStartFunc;
+    case TEST_TRANS_STOP_FUNC:
+      return mndTransTestStopFunc;
+    default:
+      return NULL;
+  }
+}
+
 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
   // pTrans->stage = TRN_STAGE_PREPARE;
   mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
+
+  if (pTrans->startFunc > 0) {
+    TransCbFp fp = mndTransGetCbFp(pTrans->startFunc);
+    if (fp) {
+      (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
+    }
+  }
+
   return 0;
 }
 
@@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) {
     pTrans->rpcRsp = NULL;
     pTrans->rpcRspLen = 0;
   }
+  if (pTrans->param != NULL) {
+    taosMemoryFree(pTrans->param);
+    pTrans->param = NULL;
+    pTrans->paramLen = 0;
+  }
 }
 
-static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
-  mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
+static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
+  mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
+         callFunc);
+  if (pTrans->stopFunc > 0 && callFunc) {
+    TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc);
+    if (fp) {
+      (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
+    }
+  }
+
   mndTransDropData(pTrans);
   return 0;
 }
@@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
     return NULL;
   }
 
-  mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans);
+  mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
 
@@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) {
 void mndTransDrop(STrans *pTrans) {
   if (pTrans != NULL) {
     mndTransDropData(pTrans);
-    mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
+    mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans);
     taosMemoryFreeClear(pTrans);
   }
 }
@@ -574,9 +629,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) {
   pTrans->rpcRspLen = contLen;
 }
 
-void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) {
-  pTrans->transCbFp = fp;
-  pTrans->transCbParam = param;
+void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) {
+  pTrans->startFunc = startFunc;
+  pTrans->stopFunc = stopFunc;
+  pTrans->param = param;
+  pTrans->paramLen = paramLen;
 }
 
 void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
@@ -712,8 +769,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
   pNew->rpcRefId = pTrans->rpcRefId;
   pNew->rpcRsp = pTrans->rpcRsp;
   pNew->rpcRspLen = pTrans->rpcRspLen;
-  pNew->transCbFp = pTrans->transCbFp;
-  pNew->transCbParam = pTrans->transCbParam;
   pTrans->rpcRsp = NULL;
   pTrans->rpcRspLen = 0;
 
@@ -1125,10 +1180,6 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
 
   mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
 
-  if (pTrans->transCbFp != NULL) {
-    (*pTrans->transCbFp)(pMnode, pTrans->transCbParam);
-  }
-
   return continueExec;
 }
 

source/dnode/mnode/impl/src/mndUser.c

@@ -96,36 +96,36 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
   int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
 
   SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
-  if (pRaw == NULL) goto USER_ENCODE_OVER;
+  if (pRaw == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER)
-  SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER)
-  SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER)
-  SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER)
-  SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER)
-  SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
 
   char *db = taosHashIterate(pUser->readDbs, NULL);
   while (db != NULL) {
-    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
+    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
     db = taosHashIterate(pUser->readDbs, db);
   }
 
   db = taosHashIterate(pUser->writeDbs, NULL);
   while (db != NULL) {
-    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
+    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
     db = taosHashIterate(pUser->writeDbs, db);
   }
 
-  SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER)
-  SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER)
+  SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
+  SDB_SET_DATALEN(pRaw, dataPos, _OVER)
 
   terrno = 0;
 
-USER_ENCODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr());
     sdbFreeRaw(pRaw);
@@ -140,55 +140,55 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
   int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
 
   if (sver != USER_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
-    goto USER_DECODE_OVER;
+    goto _OVER;
   }
 
   SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
-  if (pRow == NULL) goto USER_DECODE_OVER;
+  if (pRow == NULL) goto _OVER;
 
   SUserObj *pUser = sdbGetRowObj(pRow);
-  if (pUser == NULL) goto USER_DECODE_OVER;
-
-  pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
-  pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
-  if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER;
+  if (pUser == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER)
-  SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER)
-  SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
 
   int32_t numOfReadDbs = 0;
   int32_t numOfWriteDbs = 0;
-  SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER)
-  SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
+  pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  pUser->writeDbs =
+      taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
 
   for (int32_t i = 0; i < numOfReadDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
-    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
+    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
     int32_t len = strlen(db) + 1;
     taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN);
   }
 
   for (int32_t i = 0; i < numOfWriteDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
-    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
+    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
     int32_t len = strlen(db) + 1;
     taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
   }
 
-  SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER)
+  SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
 
   terrno = 0;
 
-USER_DECODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
     taosHashCleanup(pUser->readDbs);
@@ -220,6 +220,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
   mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
   taosHashCleanup(pUser->readDbs);
   taosHashCleanup(pUser->writeDbs);
+  pUser->readDbs = NULL;
+  pUser->writeDbs = NULL;
   return 0;
 }
 
@@ -228,13 +230,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
   memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
   pOld->updateTime = pNew->updateTime;
 
-  void *tmp1 = pOld->readDbs;
-  pOld->readDbs = pNew->readDbs;
-  pNew->readDbs = tmp1;
-
-  void *tmp2 = pOld->writeDbs;
-  pOld->writeDbs = pNew->writeDbs;
-  pNew->writeDbs = tmp2;
+  TSWAP(pOld->readDbs, pNew->readDbs);
+  TSWAP(pOld->writeDbs, pNew->writeDbs);
 
   return 0;
 }
@@ -277,6 +274,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
   }
   sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
 
+  char *param = strdup("====> test code to be deleted later <=====");
+  mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
@@ -296,41 +296,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to create", createReq.user);
 
   if (createReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   if (createReq.pass[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, createReq.user);
   if (pUser != NULL) {
     terrno = TSDB_CODE_MND_USER_ALREADY_EXIST;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   if (mndCheckCreateUserAuth(pOperUser) != 0) {
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-CREATE_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to create since %s", createReq.user, terrstr());
   }
@@ -341,13 +341,13 @@ CREATE_USER_OVER:
   return code;
 }
 
-static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) {
+static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) {
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg);
   if (pTrans == NULL) {
-    mError("user:%s, failed to update since %s", pOld->user, terrstr());
+    mError("user:%s, failed to alter since %s", pOld->user, terrstr());
     return -1;
   }
-  mDebug("trans:%d, used to update user:%s", pTrans->id, pOld->user);
+  mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user);
 
   SSdbRaw *pRedoRaw = mndUserActionEncode(pNew);
   if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
@@ -368,7 +368,8 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNo
 }
 
 static SHashObj *mndDupDbHash(SHashObj *pOld) {
-  SHashObj *pNew = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
+  SHashObj *pNew =
+      taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   if (pNew == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -379,8 +380,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) {
     int32_t len = strlen(db) + 1;
     if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) {
       taosHashCancelIterate(pOld, db);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
       taosHashCleanup(pNew);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
     }
     db = taosHashIterate(pOld, db);
@@ -399,38 +400,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to alter", alterReq.user);
 
   if (alterReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   if (alterReq.pass[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, alterReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   memcpy(&newUser, pUser, sizeof(SUserObj));
   newUser.readDbs = mndDupDbHash(pUser->readDbs);
   newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
   if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   int32_t len = strlen(alterReq.dbname) + 1;
@@ -440,56 +441,56 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
   if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) {
     char pass[TSDB_PASSWORD_LEN + 1] = {0};
     taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass);
-    memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN);
+    memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN);
   } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) {
     newUser.superUser = alterReq.superUser;
   } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) {
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
     if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) {
     if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
     taosHashClear(newUser.readDbs);
   } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) {
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
     if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
     if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) {
     taosHashClear(newUser.writeDbs);
   } else {
     terrno = TSDB_CODE_MND_INVALID_ALTER_OPER;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   newUser.updateTime = taosGetTimestampMs();
 
   if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) {
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
-  code = mndUpdateUser(pMnode, pUser, &newUser, pReq);
+  code = mndAlterUser(pMnode, pUser, &newUser, pReq);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-ALTER_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
   }
@@ -537,36 +538,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to drop", dropReq.user);
 
   if (dropReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, dropReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   if (mndCheckDropUserAuth(pOperUser) != 0) {
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   code = mndDropUser(pMnode, pReq, pUser);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-DROP_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to drop since %s", dropReq.user, terrstr());
   }
@@ -586,7 +587,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
 
   if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   mTrace("user:%s, start to get auth", authReq.user);
@@ -594,7 +595,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   pUser = mndAcquireUser(pMnode, authReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   memcpy(authRsp.user, pUser->user, TSDB_USER_LEN);
@@ -622,7 +623,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   void   *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp);
@@ -631,10 +632,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   pReq->rspLen = contLen;
   code = 0;
 
-GET_AUTH_OVER:
+_OVER:
   mndReleaseUser(pMnode, pUser);
-  taosHashCleanup(authRsp.readDbs);
-  taosHashCleanup(authRsp.writeDbs);
+  tFreeSGetUserAuthRsp(&authRsp);
 
   return code;
 }
@@ -671,11 +671,6 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
 
-    cols++;
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]);
-    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
-
     numOfRows++;
     sdbRelease(pSdb, pUser);
   }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) {
                      .encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
                      .decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
                      .insertFp = (SdbInsertFp)mndVgroupActionInsert,
-                     .updateFp = (SdbUpdateFp)mndVgroupActionDelete,
-                     .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate};
+                     .updateFp = (SdbUpdateFp)mndVgroupActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndVgroupActionDelete};
 
   mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);

source/dnode/mnode/impl/src/mnode.c

@@ -43,7 +43,7 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 
-#define MQ_TIMER_MS    3000
+#define MQ_TIMER_MS    2000
 #define TRNAS_TIMER_MS 6000
 
 static void *mndBuildTimerMsg(int32_t *pContLen) {
@@ -418,7 +418,6 @@ int64_t mndGenerateUid(char *name, int32_t len) {
   } while (true);
 }
 
-
 int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
                           SMonGrantInfo *pGrantInfo) {
   if (!mndIsMaster(pMnode)) return -1;
@@ -528,4 +527,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
 int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
   pLoad->syncState = pMnode->syncMgmt.state;
   return 0;
-}
+}
\ No newline at end of file

source/dnode/mnode/impl/test/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(func)
 add_subdirectory(mnode)
 add_subdirectory(profile)
 add_subdirectory(qnode)
+add_subdirectory(sdb)
 add_subdirectory(show)
 add_subdirectory(sma)
 add_subdirectory(snode)

source/dnode/mnode/impl/test/sdb/CMakeLists.txt

+aux_source_directory(. MNODE_SDB_TEST_SRC)
+add_executable(sdbTest ${MNODE_SDB_TEST_SRC})
+target_link_libraries(
+    sdbTest
+    PUBLIC sut
+    PUBLIC sdb
+)
+
+add_test(
+    NAME sdbTest
+    COMMAND sdbTest
+)

source/dnode/mnode/impl/test/sdb/sdbTest.cpp

+/**
+ * @file sdbTest.cpp
+ * @author slguan (slguan@taosdata.com)
+ * @brief MNODE module sdb tests
+ * @version 1.0
+ * @date 2022-04-27
+ *
+ * @copyright Copyright (c) 2022
+ *
+ */
+
+#include "sut.h"
+
+class MndTestShow : public ::testing::Test {
+ protected:
+  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); }
+  static void TearDownTestSuite() { test.Cleanup(); }
+
+  static Testbase test;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+};
+
+Testbase MndTestShow::test;
+
+TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
+  SShowReq showReq = {0};
+  showReq.type = TSDB_MGMT_TABLE_MAX;
+
+  int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSShowReq(pReq, contLen, &showReq);
+  tFreeSShowReq(&showReq);
+
+  SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
+  ASSERT_NE(pRsp, nullptr);
+  ASSERT_NE(pRsp->code, 0);
+}
+
+TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
+  SShowReq showReq = {0};
+  showReq.type = TSDB_MGMT_TABLE_START;
+
+  int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSShowReq(pReq, contLen, &showReq);
+  tFreeSShowReq(&showReq);
+
+  SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
+  ASSERT_NE(pRsp, nullptr);
+  ASSERT_NE(pRsp->code, 0);
+}
+
+TEST_F(MndTestShow, 03_ShowMsg_Conn) {
+  char passwd[] = "taosdata";
+  char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
+  taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
+
+  SConnectReq connectReq = {0};
+  connectReq.pid = 1234;
+  strcpy(connectReq.app, "mnode_test_show");
+  strcpy(connectReq.db, "");
+  strcpy(connectReq.user, "root");
+  strcpy(connectReq.passwd, secretEncrypt);
+
+  int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
+  void*   pReq = rpcMallocCont(contLen);
+  tSerializeSConnectReq(pReq, contLen, &connectReq);
+
+  SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
+  ASSERT_NE(pRsp, nullptr);
+  ASSERT_EQ(pRsp->code, 0);
+
+  test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", "");
+  // EXPECT_EQ(test.GetShowRows(), 1);
+}
+
+TEST_F(MndTestShow, 04_ShowMsg_Cluster) {
+  test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", "");
+  EXPECT_EQ(test.GetShowRows(), 1);
+}

source/dnode/mnode/sdb/inc/sdbInt.h

@@ -19,10 +19,6 @@
 #include "os.h"
 
 #include "sdb.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tlog.h"
-#include "tmsg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,26 +49,6 @@ typedef struct SSdbRow {
   char       pObj[];
 } SSdbRow;
 
-typedef struct SSdb {
-  SMnode     *pMnode;
-  char       *currDir;
-  char       *syncDir;
-  char       *tmpDir;
-  int64_t     lastCommitVer;
-  int64_t     curVer;
-  int64_t     tableVer[SDB_MAX];
-  int64_t     maxId[SDB_MAX];
-  EKeyType    keyTypes[SDB_MAX];
-  SHashObj   *hashObjs[SDB_MAX];
-  SRWLatch    locks[SDB_MAX];
-  SdbInsertFp insertFps[SDB_MAX];
-  SdbUpdateFp updateFps[SDB_MAX];
-  SdbDeleteFp deleteFps[SDB_MAX];
-  SdbDeployFp deployFps[SDB_MAX];
-  SdbEncodeFp encodeFps[SDB_MAX];
-  SdbDecodeFp decodeFps[SDB_MAX];
-} SSdb;
-
 const char *sdbTableName(ESdbType type);
 void        sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
 

source/dnode/mnode/sdb/src/sdb.c

@@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) {
       SSdbRow *pRow = *ppRow;
       if (pRow == NULL) continue;
 
-      sdbFreeRow(pSdb, pRow);
+      sdbFreeRow(pSdb, pRow, true);
       ppRow = taosHashIterate(hash, ppRow);
     }
   }

source/dnode/mnode/sdb/src/sdbHash.c

@@ -137,7 +137,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
   if (pOldRow != NULL) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
     return terrno;
   }
@@ -148,7 +148,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
 
   if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
     return terrno;
   }
@@ -164,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
       taosWLockLatch(pLock);
       taosHashRemove(hash, pRow->pObj, keySize);
       taosWUnLockLatch(pLock);
-      sdbFreeRow(pSdb, pRow);
+      sdbFreeRow(pSdb, pRow, false);
       terrno = code;
       return terrno;
     }
@@ -202,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
     code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
   }
 
-  sdbFreeRow(pSdb, pNewRow);
+  sdbFreeRow(pSdb, pNewRow, false);
 
   pSdb->tableVer[pOldRow->type]++;
   return code;
@@ -215,7 +215,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
   if (ppOldRow == NULL || *ppOldRow == NULL) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
     return terrno;
   }
@@ -228,7 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   taosWUnLockLatch(pLock);
 
   pSdb->tableVer[pOldRow->type]++;
-  sdbFreeRow(pSdb, pRow);
+  sdbFreeRow(pSdb, pRow, false);
 
   sdbCheck(pSdb, pOldRow);
   // sdbRelease(pSdb, pOldRow->pObj);
@@ -322,7 +322,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) {
   int32_t ref = atomic_load_32(&pRow->refCount);
   sdbPrintOper(pSdb, pRow, "check");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, true);
   }
 
   taosRUnLockLatch(pLock);
@@ -340,7 +340,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
   int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
   sdbPrintOper(pSdb, pRow, "release");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, true);
   }
 
   taosRUnLockLatch(pLock);

source/dnode/mnode/sdb/src/sdbRaw.c

@@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
     return -1;
   }
 
-  memcpy(pRaw->pData + dataPos, pVal, valLen);
+  if (pVal != NULL) {
+    memcpy(pRaw->pData + dataPos, pVal, valLen);
+  }
   return 0;
 }
 

source/dnode/mnode/sdb/src/sdbRow.c

@@ -36,11 +36,11 @@ void *sdbGetRowObj(SSdbRow *pRow) {
   return pRow->pObj;
 }
 
-void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
+void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) {
   // remove attached object such as trans
   SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
   if (deleteFp != NULL) {
-    (*deleteFp)(pSdb, pRow->pObj);
+    (*deleteFp)(pSdb, pRow->pObj, callFunc);
   }
 
   sdbPrintOper(pSdb, pRow, "free");

source/dnode/vnode/CMakeLists.txt

@@ -27,10 +27,9 @@ target_sources(
     "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
     "src/tsdb/tsdbCommit2.c"
-    "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbFile.c"
     "src/tsdb/tsdbFS.c"
-    "src/tsdb/tsdbMain.c"
+    "src/tsdb/tsdbOpen.c"
     "src/tsdb/tsdbMemTable.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"

source/dnode/vnode/inc/vnode.h

@@ -68,6 +68,7 @@ void    vnodeStop(SVnode *pVnode);
 
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
+void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
 
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
@@ -76,6 +77,7 @@ typedef struct SMetaEntry  SMetaEntry;
 
 void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
 void metaReaderClear(SMetaReader *pReader);
+int  metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
 int  metaReadNext(SMetaReader *pReader);
 
 #if 1  // refact APIs below (TODO)
@@ -133,23 +135,18 @@ int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *p
 // need to reposition
 
 // structs
-struct SMetaCfg {
-  uint64_t lruSize;
-};
-
 struct STsdbCfg {
-  int8_t   precision;
-  int8_t   update;
-  int8_t   compression;
-  int8_t   slLevel;
-  int32_t  days;
-  int32_t  minRows;
-  int32_t  maxRows;
-  int32_t  keep2;
-  int32_t  keep0;
-  int32_t  keep1;
-  uint64_t lruCacheSize;
-  SArray  *retentions;
+  int8_t  precision;
+  int8_t  update;
+  int8_t  compression;
+  int8_t  slLevel;
+  int32_t days;
+  int32_t minRows;
+  int32_t maxRows;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
+  SArray *retentions;
 };
 
 struct SVnodeCfg {
@@ -160,8 +157,6 @@ struct SVnodeCfg {
   int32_t  szCache;
   uint64_t szBuf;
   bool     isHeap;
-  uint32_t ttl;
-  uint32_t keep;
   int8_t   streamMode;
   bool     isWeak;
   STsdbCfg tsdbCfg;

source/dnode/vnode/src/inc/meta.h

@@ -77,21 +77,25 @@ typedef struct {
   tb_uid_t uid;
 } STbDbKey;
 
-typedef struct __attribute__((__packed__)) {
+#pragma pack(push, 1)
+typedef struct {
   tb_uid_t uid;
   int32_t  sver;
 } SSkmDbKey;
+#pragma pack(pop)
 
 typedef struct {
   tb_uid_t suid;
   tb_uid_t uid;
 } SCtbIdxKey;
 
-typedef struct __attribute__((__packed__)) {
+#pragma pack(push, 1)
+typedef struct {
   tb_uid_t suid;
   int16_t  cid;
   char     data[];
 } STagIdxKey;
+#pragma pack(pop)
 
 typedef struct {
   int64_t  dtime;

source/dnode/vnode/src/inc/tsdb.h

@@ -46,12 +46,45 @@ int  tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKe
 
 // tsdbCommit ================
 
-#if 1
+// tsdbFS ================
+typedef struct STsdbFS STsdbFS;
 
-typedef struct SSmaStat SSmaStat;
+// tsdbSma ================
 typedef struct SSmaEnv  SSmaEnv;
 typedef struct SSmaEnvs SSmaEnvs;
 
+// structs
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+struct SSmaEnvs {
+  int16_t  nTSma;
+  int16_t  nRSma;
+  SSmaEnv *pTSmaEnv;
+  SSmaEnv *pRSmaEnv;
+};
+
+struct STsdb {
+  char          *path;
+  SVnode        *pVnode;
+  bool           repoLocked;
+  TdThreadMutex  mutex;
+  STsdbCfg       config;
+  STsdbMemTable *mem;
+  STsdbMemTable *imem;
+  SRtn           rtn;
+  STsdbFS       *fs;
+  SSmaEnvs       smaEnvs;
+};
+
+#if 1  // ======================================
+
+typedef struct SSmaStat SSmaStat;
+
 struct STable {
   uint64_t  tid;
   uint64_t  uid;
@@ -97,13 +130,6 @@ typedef struct {
   uint8_t   state;
 } SDFile;
 
-struct SSmaEnvs {
-  int16_t  nTSma;
-  int16_t  nRSma;
-  SSmaEnv *pTSmaEnv;
-  SSmaEnv *pRSmaEnv;
-};
-
 typedef struct {
   int      fid;
   int8_t   state;  // -128~127
@@ -112,13 +138,6 @@ typedef struct {
   SDFile   files[TSDB_FILE_MAX];
 } SDFileSet;
 
-typedef struct {
-  int   minFid;
-  int   midFid;
-  int   maxFid;
-  TSKEY minKey;
-} SRtn;
-
 struct STbData {
   tb_uid_t   uid;
   TSKEY      keyMin;
@@ -155,7 +174,7 @@ typedef struct {
   SArray     *sf;    // sma data file array    v2f1900.index_name_1
 } SFSStatus;
 
-typedef struct {
+struct STsdbFS {
   TdThreadRwlock lock;
 
   SFSStatus *cstatus;        // current status
@@ -163,23 +182,9 @@ typedef struct {
   SHashObj  *metaCacheComp;  // meta cache for compact
   bool       intxn;
   SFSStatus *nstatus;  // new status
-} STsdbFS;
-
-struct STsdb {
-  char          *path;
-  SVnode        *pVnode;
-  int32_t        vgId;
-  bool           repoLocked;
-  TdThreadMutex  mutex;
-  STsdbCfg       config;
-  STsdbMemTable *mem;
-  STsdbMemTable *imem;
-  SRtn           rtn;
-  STsdbFS       *fs;
-  SSmaEnvs       smaEnvs;
 };
 
-#define REPO_ID(r)        ((r)->vgId)
+#define REPO_ID(r)        TD_VID((r)->pVnode)
 #define REPO_CFG(r)       (&(r)->config)
 #define REPO_FS(r)        ((r)->fs)
 #define REPO_META(r)      ((r)->pVnode->pMeta)
@@ -507,12 +512,6 @@ static FORCE_INLINE void *taosTZfree(void *ptr) {
 
 // tsdbCommit
 
-typedef struct {
-  uint64_t uid;
-  int64_t  offset;
-  int64_t  size;
-} SKVRecord;
-
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
 
 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
@@ -891,66 +890,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
   return 0;
 }
 
-// tsdbSma
-// #define TSDB_SMA_TEST // remove after test finished
-
-// struct SSmaEnv {
-//   TdThreadRwlock lock;
-//   SDiskID        did;
-//   TDBEnv         dbEnv;  // TODO: If it's better to put it in smaIndex level?
-//   char          *path;   // relative path
-//   SSmaStat      *pStat;
-// };
-
-// #define SMA_ENV_LOCK(env)       ((env)->lock)
-// #define SMA_ENV_DID(env)        ((env)->did)
-// #define SMA_ENV_ENV(env)        ((env)->dbEnv)
-// #define SMA_ENV_PATH(env)       ((env)->path)
-// #define SMA_ENV_STAT(env)       ((env)->pStat)
-// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
-
-// void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-// #if 0
-// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
-// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-// #endif
-
-// // internal func
-// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
-//   int32_t len = 0;
-//   len += taosEncodeFixedI64(pData, tsKey);
-//   len += taosEncodeFixedI64(pData, groupId);
-//   return len;
-// }
-
-// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockRdlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockWrlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockUnlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
 typedef struct SSmaKey SSmaKey;
 
 struct SSmaKey {

source/dnode/vnode/src/inc/vnodeInt.h

@@ -75,7 +75,6 @@ int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq*
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
-int             metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
 int             metaGetTbNum(SMeta* pMeta);
 SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);

source/dnode/vnode/src/meta/metaTable.c

@@ -88,7 +88,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
 
   // preprocess req
   pReq->uid = tGenIdPI64();
-  pReq->ctime = taosGetTimestampSec();
+  pReq->ctime = taosGetTimestampMs();
 
   // validate req
   metaReaderInit(&mr, pMeta, 0);
@@ -158,7 +158,9 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   pKey = &tbDbKey;
   kLen = sizeof(tbDbKey);
 
-  if (tEncodeSize(metaEncodeEntry, pME, vLen) < 0) {
+  int32_t ret = 0;
+  tEncodeSize(metaEncodeEntry, pME, vLen, ret);
+  if (ret < 0) {
     goto _err;
   }
 
@@ -250,7 +252,9 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   skmDbKey.sver = pSW->sver;
 
   // encode schema
-  if (tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen) < 0) return -1;
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSSchemaWrapper, pSW, vLen, ret);
+  if (ret < 0) return -1;
   pVal = taosMemoryMalloc(vLen);
   if (pVal == NULL) {
     rcode = -1;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -752,334 +752,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
   return 0;
 }
 
-// // =================== Commit Meta Data
-// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) {
-//   STsdbFS *  pfs = REPO_FS(pRepo);
-//   SMFile *   pOMFile = pfs->cstatus->pmf;
-//   SDiskID    did;
-
-//   // Create/Open a meta file or open the existing file
-//   if (pOMFile == NULL) {
-//     // Create a new meta file
-//     did.level = TFS_PRIMARY_LEVEL;
-//     did.id = TFS_PRIMARY_ID;
-//     tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
-
-//     if (open && tsdbCreateMFile(pMf, true) < 0) {
-//       tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-
-//     tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf));
-//   } else {
-//     tsdbInitMFileEx(pMf, pOMFile);
-//     if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) {
-//       tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-//   }
-
-//   return 0;
-// }
-
-// static int tsdbCommitMeta(STsdbRepo *pRepo) {
-//   STsdbFS *  pfs = REPO_FS(pRepo);
-//   SMemTable *pMem = pRepo->imem;
-//   SMFile *   pOMFile = pfs->cstatus->pmf;
-//   SMFile     mf;
-//   SActObj *  pAct = NULL;
-//   SActCont * pCont = NULL;
-//   SListNode *pNode = NULL;
-
-//   ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0);
-
-//   if (listNEles(pMem->actList) <= 0) {
-//     // no meta data to commit, just keep the old meta file
-//     tsdbUpdateMFile(pfs, pOMFile);
-//     if (tsTsdbMetaCompactRatio > 0) {
-//       if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) {
-//         return -1;
-//       }
-//       int ret = tsdbCompactMetaFile(pRepo, pfs, &mf);
-//       if (ret < 0) tsdbError("compact meta file error");
-
-//       return ret;
-//     }
-//     return 0;
-//   } else {
-//     if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) {
-//       return -1;
-//     }
-//   }
-
-//   // Loop to write
-//   while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
-//     pAct = (SActObj *)pNode->data;
-//     if (pAct->act == TSDB_UPDATE_META) {
-//       pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
-//       if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) {
-//         tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(&mf);
-//         (void)tsdbApplyMFileChange(&mf, pOMFile);
-//         // TODO: need to reload metaCache
-//         return -1;
-//       }
-//     } else if (pAct->act == TSDB_DROP_META) {
-//       if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
-//         tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(&mf);
-//         tsdbApplyMFileChange(&mf, pOMFile);
-//         // TODO: need to reload metaCache
-//         return -1;
-//       }
-//     } else {
-//       ASSERT(false);
-//     }
-//   }
-
-//   if (tsdbUpdateMFileHeader(&mf) < 0) {
-//     tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
-//     tsdbApplyMFileChange(&mf, pOMFile);
-//     // TODO: need to reload metaCache
-//     return -1;
-//   }
-
-//   TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbUpdateMFile(pfs, &mf);
-
-//   if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) {
-//     tsdbError("compact meta file error");
-//   }
-
-//   return 0;
-// }
-
-// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) {
-//   int tlen = 0;
-//   tlen += taosEncodeFixedU64(buf, pRecord->uid);
-//   tlen += taosEncodeFixedI64(buf, pRecord->offset);
-//   tlen += taosEncodeFixedI64(buf, pRecord->size);
-
-//   return tlen;
-// }
-
-// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) {
-//   buf = taosDecodeFixedU64(buf, &(pRecord->uid));
-//   buf = taosDecodeFixedI64(buf, &(pRecord->offset));
-//   buf = taosDecodeFixedI64(buf, &(pRecord->size));
-
-//   return buf;
-// }
-
-// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) {
-//   char      buf[64] = "\0";
-//   void *    pBuf = buf;
-//   SKVRecord rInfo;
-//   int64_t   offset;
-
-//   // Seek to end of meta file
-//   offset = tsdbSeekMFile(pMFile, 0, SEEK_END);
-//   if (offset < 0) {
-//     return -1;
-//   }
-
-//   rInfo.offset = offset;
-//   rInfo.uid = uid;
-//   rInfo.size = contLen;
-
-//   int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo);
-//   if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) {
-//     return -1;
-//   }
-
-//   if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) {
-//     return -1;
-//   }
-
-//   tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
-
-//   SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache;
-
-//   pMFile->info.nRecords++;
-
-//   SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid));
-//   if (pRecord != NULL) {
-//     pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
-//   } else {
-//     pMFile->info.nRecords++;
-//   }
-//   taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
-
-//   return 0;
-// }
-
-// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
-//   SKVRecord rInfo = {0};
-//   char      buf[128] = "\0";
-
-//   SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
-//   if (pRecord == NULL) {
-//     tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid);
-//     return -1;
-//   }
-
-//   rInfo.offset = -pRecord->offset;
-//   rInfo.uid = pRecord->uid;
-//   rInfo.size = pRecord->size;
-
-//   void *pBuf = buf;
-//   tsdbEncodeKVRecord(&pBuf, &rInfo);
-
-//   if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) {
-//     return -1;
-//   }
-
-//   pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord));
-//   pMFile->info.nDels++;
-//   pMFile->info.nRecords--;
-//   pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-
-//   taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid));
-//   return 0;
-// }
-
-// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) {
-//   float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords);
-//   float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size);
-//   float compactRatio = (float)(tsTsdbMetaCompactRatio)/100;
-
-//   if (delPercent < compactRatio && tombPercent < compactRatio) {
-//     return 0;
-//   }
-
-//   if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
-//     tsdbError("open meta file %s compact fail", pMFile->f.rname);
-//     return -1;
-//   }
-
-//   tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64
-//   ",size:%" PRId64,
-//     tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size);
-
-//   SMFile mf;
-//   SDiskID did;
-
-//   // first create tmp meta file
-//   did.level = TFS_PRIMARY_LEVEL;
-//   did.id = TFS_PRIMARY_ID;
-//   tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1);
-
-//   if (tsdbCreateMFile(&mf, true) < 0) {
-//     tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-
-//   tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
-
-//   // second iterator metaCache
-//   int code = -1;
-//   int64_t maxBufSize = 1024;
-//   SKVRecord *pRecord;
-//   void *pBuf = NULL;
-
-//   pBuf = taosMemoryMalloc((size_t)maxBufSize);
-//   if (pBuf == NULL) {
-//     goto _err;
-//   }
-
-//   // init Comp
-//   assert(pfs->metaCacheComp == NULL);
-//   pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-//   if (pfs->metaCacheComp == NULL) {
-//     goto _err;
-//   }
-
-//   pRecord = taosHashIterate(pfs->metaCache, NULL);
-//   while (pRecord) {
-//     if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
-//       tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                 tstrerror(terrno));
-//       goto _err;
-//     }
-//     if (pRecord->size > maxBufSize) {
-//       maxBufSize = pRecord->size;
-//       void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize);
-//       if (tmp == NULL) {
-//         goto _err;
-//       }
-//       pBuf = tmp;
-//     }
-//     int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
-//     if (nread < 0) {
-//       tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//         tstrerror(terrno));
-//       goto _err;
-//     }
-
-//     if (nread < pRecord->size) {
-//       tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
-//                 REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
-//       goto _err;
-//     }
-
-//     if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) {
-//       tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid,
-//                 tstrerror(terrno));
-//       goto _err;
-//     }
-
-//     pRecord = taosHashIterate(pfs->metaCache, pRecord);
-//   }
-//   code = 0;
-
-// _err:
-//   if (code == 0) TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbCloseMFile(pMFile);
-
-//   if (code == 0) {
-//     // rename meta.tmp -> meta
-//     tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf),
-//     TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname,
-//     TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN);
-//     // update current meta file info
-//     pfs->nstatus->pmf = NULL;
-//     tsdbUpdateMFile(pfs, &mf);
-
-//     taosHashCleanup(pfs->metaCache);
-//     pfs->metaCache = pfs->metaCacheComp;
-//     pfs->metaCacheComp = NULL;
-//   } else {
-//     // remove meta.tmp file
-//     taosRemoveFile(mf.f.aname);
-//     taosHashCleanup(pfs->metaCacheComp);
-//     pfs->metaCacheComp = NULL;
-//   }
-
-//   taosMemoryFreeClear(pBuf);
-
-//   ASSERT(mf.info.nDels == 0);
-//   ASSERT(mf.info.tombSize == 0);
-
-//   tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64,
-//     code,mf.info.nRecords,mf.info.size);
-//   return code;
-// }
-
-// // =================== Commit Time-Series Data
-// #if 0
-// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
-//   for (int i = 0; i < nIters; i++) {
-//     TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
-//     if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true;
-//   }
-//   return false;
-// }
-// #endif
-
+// =================== Commit Time-Series Data
 static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
   SCommitIter *pIter = pCommith->iters + tid;
   TSKEY        nextKey = tsdbNextIterKey(pIter->pIter);

source/dnode/vnode/src/tsdb/tsdbCompact.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#if 0
-#include "tsdb.h"
-
-typedef struct {
-  STable *    pTable;
-  SBlockIdx * pBlkIdx;
-  SBlockIdx   bindex;
-  SBlockInfo *pInfo;
-} STableCompactH;
-
-typedef struct {
-  SRtn       rtn;
-  SFSIter    fsIter;
-  SArray *   tbArray;  // table array to cache table obj and block indexes
-  SReadH     readh;
-  SDFileSet  wSet;
-  SArray *   aBlkIdx;
-  SArray *   aSupBlk;
-  SDataCols *pDataCols;
-} SCompactH;
-
-#define TSDB_COMPACT_WSET(pComph)      (&((pComph)->wSet))
-#define TSDB_COMPACT_REPO(pComph)      TSDB_READ_REPO(&((pComph)->readh))
-#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
-#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
-#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
-#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
-#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
-#define TSDB_COMPACT_BUF(pComph)       TSDB_READ_BUF(&((pComph)->readh))
-#define TSDB_COMPACT_COMP_BUF(pComph)  TSDB_READ_COMP_BUF(&((pComph)->readh))
-
-static int  tsdbAsyncCompact(STsdbRepo *pRepo);
-static void tsdbStartCompact(STsdbRepo *pRepo);
-static void tsdbEndCompact(STsdbRepo *pRepo, int eno);
-static int  tsdbCompactMeta(STsdbRepo *pRepo);
-static int  tsdbCompactTSData(STsdbRepo *pRepo);
-static int  tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet);
-static bool tsdbShouldCompact(SCompactH *pComph);
-static int  tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo);
-static void tsdbDestroyCompactH(SCompactH *pComph);
-static int  tsdbInitCompTbArray(SCompactH *pComph);
-static void tsdbDestroyCompTbArray(SCompactH *pComph);
-static int  tsdbCacheFSetIndex(SCompactH *pComph);
-static int  tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet);
-static void tsdbCompactFSetEnd(SCompactH *pComph);
-static int  tsdbCompactFSetImpl(SCompactH *pComph);
-static int  tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
-                                      void **ppCBuf);
-
-enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
-int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); }
-
-void *tsdbCompactImpl(STsdbRepo *pRepo) {
-  // Check if there are files in TSDB FS to compact
-  if (REPO_FS(pRepo)->cstatus->pmf == NULL) {
-    tsdbInfo("vgId:%d no file to compact in FS", REPO_ID(pRepo));
-    return NULL;
-  }
-
-  tsdbStartCompact(pRepo);
-
-  if (tsdbCompactMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
-
-  if (tsdbCompactTSData(pRepo) < 0) {
-    tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
-
-  tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS);
-  return NULL;
-
-_err:
-  pRepo->code = terrno;
-  tsdbEndCompact(pRepo, terrno);
-  return NULL;
-}
-
-static int tsdbAsyncCompact(STsdbRepo *pRepo) {
-  if (pRepo->compactState != TSDB_NO_COMPACT) {
-    tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
-    return 0; 
-  } 
-  pRepo->compactState = TSDB_WAITING_COMPACT;   
-  tsem_wait(&(pRepo->readyToCommit));
-  return tsdbScheduleCommit(pRepo, COMPACT_REQ);
-}
-
-static void tsdbStartCompact(STsdbRepo *pRepo) {
-  assert(pRepo->compactState != TSDB_IN_COMPACT);
-  tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
-  tsdbStartFSTxn(pRepo, 0, 0);
-  pRepo->code = TSDB_CODE_SUCCESS;
-  pRepo->compactState = TSDB_IN_COMPACT;
-}
-
-static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
-  if (eno != TSDB_CODE_SUCCESS) {
-    tsdbEndFSTxnWithError(REPO_FS(pRepo));
-  } else {
-    tsdbEndFSTxn(pRepo);
-  }
-  pRepo->compactState = TSDB_NO_COMPACT;
-  tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
-  tsem_post(&(pRepo->readyToCommit));
-}
-
-static int tsdbCompactMeta(STsdbRepo *pRepo) {
-  STsdbFS *pfs = REPO_FS(pRepo);
-  tsdbUpdateMFile(pfs, pfs->cstatus->pmf);
-  return 0;
-}
-
-  static int tsdbCompactTSData(STsdbRepo *pRepo) {
-    SCompactH  compactH;
-    SDFileSet *pSet = NULL;
-
-    tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo));
-
-    // If no file, just return 0;
-    if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) {
-      tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo));
-      return 0;
-    }
-
-    if (tsdbInitCompactH(&compactH, pRepo) < 0) {
-      return -1;
-    }
-
-    while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) {
-      // Remove those expired files
-      if (pSet->fid < compactH.rtn.minFid) {
-        tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
-                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
-        continue;
-      }
-
-      if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) {
-        tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL);
-        tsdbUpdateDFileSet(REPO_FS(pRepo), pSet);
-        continue;
-      }
-
-      if (tsdbCompactFSet(&compactH, pSet) < 0) {
-        tsdbDestroyCompactH(&compactH);
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        return -1;
-      }
-    }
-
-    tsdbDestroyCompactH(&compactH);
-    tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo));
-    return 0;
-  }
-
-  static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    SDiskID    did;
-
-    tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet),
-              TSDB_FSET_ID(pSet));
-
-    if (tsdbCompactFSetInit(pComph, pSet) < 0) {
-      return -1;
-    }
-
-    if (!tsdbShouldCompact(pComph)) {
-      tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid);
-      if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) {
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-    } else {
-      // Create new fset as compacted fset
-      if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) {
-        terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet),
-                      FS_TXN_VERSION(REPO_FS(pRepo)));
-      if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) {
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      if (tsdbCompactFSetImpl(pComph) < 0) {
-        tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-        tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-      tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph));
-      tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid);
-    }
-
-    tsdbCompactFSetEnd(pComph);
-    return 0;
-  }
-
-  static bool tsdbShouldCompact(SCompactH *pComph) {
-    STsdbRepo *     pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg *      pCfg = REPO_CFG(pRepo);
-    SReadH *        pReadh = &(pComph->readh);
-    STableCompactH *pTh;
-    SBlock *        pBlock;
-    int             defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
-    SDFile *        pDataF = TSDB_READ_DATA_FILE(pReadh);
-    SDFile *        pLastF = TSDB_READ_LAST_FILE(pReadh);
-
-    int     tblocks = 0;       // total blocks
-    int     nSubBlocks = 0;    // # of blocks with sub-blocks
-    int     nSmallBlocks = 0;  // # of blocks with rows < defaultRows
-    int64_t tsize = 0;
-
-    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
-      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
-
-      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
-
-      for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) {
-        tblocks++;
-        pBlock = pTh->pInfo->blocks + bidx;
-
-        if (pBlock->numOfRows < defaultRows) {
-          nSmallBlocks++;
-        }
-
-        if (pBlock->numOfSubBlocks > 1) {
-          nSubBlocks++;
-          for (int k = 0; k < pBlock->numOfSubBlocks; k++) {
-            SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k;
-            tsize = tsize + iBlock->len;
-          }
-        } else if (pBlock->numOfSubBlocks == 1) {
-          tsize += pBlock->len;
-        } else {
-          ASSERT(0);
-        }
-      }
-    }
-
-    return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) ||
-            (tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85));
-  }
-
-  static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo) {
-    STsdbCfg *pCfg = REPO_CFG(pRepo);
-
-    memset(pComph, 0, sizeof(*pComph));
-
-    TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph));
-
-    tsdbGetRtnSnap(pRepo, &(pComph->rtn));
-    tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
-
-    if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) {
-      return -1;
-    }
-
-    if (tsdbInitCompTbArray(pComph) < 0) {
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
-    if (pComph->aBlkIdx == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
-    if (pComph->aSupBlk == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
-    if (pComph->pDataCols == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static void tsdbDestroyCompactH(SCompactH *pComph) {
-    pComph->pDataCols = tdFreeDataCols(pComph->pDataCols);
-    pComph->aSupBlk = taosArrayDestroy(pComph->aSupBlk);
-    pComph->aBlkIdx = taosArrayDestroy(pComph->aBlkIdx);
-    tsdbDestroyCompTbArray(pComph);
-    tsdbDestroyReadH(&(pComph->readh));
-    tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-  }
-
-  static int tsdbInitCompTbArray(SCompactH *pComph) {  // Init pComp->tbArray
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-    if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
-
-    pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH));
-    if (pComph->tbArray == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbUnlockRepoMeta(pRepo);
-      return -1;
-    }
-
-    // Note here must start from 0
-    for (int i = 0; i < pMeta->maxTables; i++) {
-      STableCompactH ch = {0};
-      if (pMeta->tables[i] != NULL) {
-        tsdbRefTable(pMeta->tables[i]);
-        ch.pTable = pMeta->tables[i];
-      }
-
-      if (taosArrayPush(pComph->tbArray, &ch) == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        tsdbUnlockRepoMeta(pRepo);
-        return -1;
-      }
-    }
-
-    if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
-    return 0;
-  }
-
-  static void tsdbDestroyCompTbArray(SCompactH *pComph) {
-    STableCompactH *pTh;
-
-    if (pComph->tbArray == NULL) return;
-
-    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
-      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
-      if (pTh->pTable) {
-        tsdbUnRefTable(pTh->pTable);
-      }
-
-      pTh->pInfo = taosTZfree(pTh->pInfo);
-    }
-
-    pComph->tbArray = taosArrayDestroy(pComph->tbArray);
-  }
-
-  static int tsdbCacheFSetIndex(SCompactH *pComph) {
-    SReadH *pReadH = &(pComph->readh);
-
-    if (tsdbLoadBlockIdx(pReadH) < 0) {
-      return -1;
-    }
-
-    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
-      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
-      pTh->pBlkIdx = NULL;
-
-      if (pTh->pTable == NULL) continue;
-      if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) {
-        return -1;
-      }
-
-      if (pReadH->pBlkIdx == NULL) continue;
-      pTh->bindex = *(pReadH->pBlkIdx);
-      pTh->pBlkIdx = &(pTh->bindex);
-
-      if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) {
-        return -1;
-      }
-
-      if (tsdbLoadBlockInfo(pReadH, (void *)(pTh->pInfo)) < 0) {
-        return -1;
-      }
-    }
-
-    return 0;
-  }
-
-  static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) {
-    taosArrayClear(pComph->aBlkIdx);
-    taosArrayClear(pComph->aSupBlk);
-
-    if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) {
-      return -1;
-    }
-
-    if (tsdbCacheFSetIndex(pComph) < 0) {
-      tsdbCloseAndUnsetFSet(&(pComph->readh));
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); }
-
-  static int tsdbCompactFSetImpl(SCompactH *pComph) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg * pCfg = REPO_CFG(pRepo);
-    SReadH *   pReadh = &(pComph->readh);
-    SBlockIdx  blkIdx;
-    void **    ppBuf = &(TSDB_COMPACT_BUF(pComph));
-    void **    ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph));
-    int        defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
-
-    taosArrayClear(pComph->aBlkIdx);
-
-    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
-      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
-      STSchema *      pSchema;
-
-      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
-
-      pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1);
-      taosArrayClear(pComph->aSupBlk);
-      if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) ||
-          (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-      tdFreeSchema(pSchema);
-
-      // Loop to compact each block data
-      for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) {
-        SBlock *pBlock = pTh->pInfo->blocks + i;
-
-        // Load the block data
-        if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) {
-          return -1;
-        }
-
-        // Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file
-        if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) {
-          if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf) < 0) {
-            return -1;
-          }
-        } else {
-          int ridx = 0;
-
-          while (true) {
-            if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
-            int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
-
-            tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);
-
-            if (pComph->pDataCols->numOfRows < defaultRows) {
-              break;
-            }
-
-            if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
-              return -1;
-            }
-            tdResetDataCols(pComph->pDataCols);
-          }
-        }
-      }
-
-      if (pComph->pDataCols->numOfRows > 0 &&
-          tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
-        return -1;
-      }
-
-      if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) <
-          0) {
-        return -1;
-      }
-
-      if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-    }
-
-    if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) {
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
-                                      void **ppCBuf) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg * pCfg = REPO_CFG(pRepo);
-    SDFile *   pDFile;
-    bool       isLast;
-    SBlock     block;
-
-    ASSERT(pDataCols->numOfRows > 0);
-
-    if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
-      pDFile = TSDB_COMPACT_LAST_FILE(pComph);
-      isLast = true;
-    } else {
-      pDFile = TSDB_COMPACT_DATA_FILE(pComph);
-      isLast = false;
-    }
-
-    if (tsdbWriteBlockImpl(pRepo, pTable, pDFile, pDataCols, &block, isLast, true, ppBuf, ppCBuf) < 0) {
-      return -1;
-    }
-
-    if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      return -1;
-    }
-
-    return 0;
-}
-
-#endif

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -246,62 +246,6 @@ void *tsdbFreeFS(STsdbFS *pfs) {
   return NULL;
 }
 
-// static int tsdbProcessExpiredFS(STsdb *pRepo) {
-//   tsdbStartFSTxn(pRepo, 0, 0);
-//   // if (tsdbCreateMeta(pRepo) < 0) {
-//   //   tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//   //   return -1;
-//   // }
-
-//   if (tsdbApplyRtn(pRepo) < 0) {
-//     tsdbEndFSTxnWithError(REPO_FS(pRepo));
-//     tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-//   if (tsdbEndFSTxn(pRepo) < 0) {
-//     tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static int tsdbCreateMeta(STsdb *pRepo) {
-//   STsdbFS *pfs = REPO_FS(pRepo);
-//   SMFile * pOMFile = pfs->cstatus->pmf;
-//   SMFile   mf;
-//   SDiskID  did;
-
-//   if (pOMFile != NULL) {
-//     // keep the old meta file
-//     tsdbUpdateMFile(pfs, pOMFile);
-//     return 0;
-//   }
-
-//   // Create a new meta file
-//   did.level = TFS_PRIMARY_LEVEL;
-//   did.id = TFS_PRIMARY_ID;
-//   tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
-
-//   if (tsdbCreateMFile(&mf, true) < 0) {
-//     tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-
-//   tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
-
-//   if (tsdbUpdateMFileHeader(&mf) < 0) {
-//     tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
-//     tsdbApplyMFileChange(&mf, pOMFile);
-//     return -1;
-//   }
-
-//   TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbUpdateMFile(pfs, &mf);
-
-//   return 0;
-// }
-
 int tsdbOpenFS(STsdb *pRepo) {
   STsdbFS *pfs = REPO_FS(pRepo);
   char     current[TSDB_FILENAME_LEN] = "\0";
@@ -644,7 +588,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
 }
 
 static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
-  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), pRepo->vgId,
+  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
            tsdbTxnFname[ftype]);
 }
 
@@ -769,142 +713,6 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) {
   return 0;
 }
 
-// int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta) {
-//   char      tbuf[128];
-//   STsdbFS * pfs = REPO_FS(pRepo);
-//   SMFile    mf;
-//   SMFile *  pMFile = &mf;
-//   void *    pBuf = NULL;
-//   SKVRecord rInfo;
-//   int64_t   maxBufSize = 0;
-//   SMFInfo   minfo;
-
-//   taosHashClear(pfs->metaCache);
-
-//   // No meta file, just return
-//   if (pfs->cstatus->pmf == NULL) return 0;
-
-//   mf = pfs->cstatus->mf;
-//   // Load cache first
-//   if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
-//     return -1;
-//   }
-
-//   if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
-//     tsdbCloseMFile(pMFile);
-//     return -1;
-//   }
-
-//   while (true) {
-//     int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
-//     if (tsize == 0) break;
-
-//     if (tsize < 0) {
-//       tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-
-//     if (tsize < sizeof(SKVRecord)) {
-//       tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
-//                 TSDB_FILE_FULL_NAME(pMFile));
-//       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//       tsdbCloseMFile(pMFile);
-//       return -1;
-//     }
-
-//     void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
-//     ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
-//     // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
-
-//     if (rInfo.offset < 0) {
-//       taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
-// #if 0
-//       pStore->info.size += sizeof(SKVRecord);
-//       pStore->info.nRecords--;
-//       pStore->info.nDels++;
-//       pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-// #endif
-//     } else {
-//       ASSERT(rInfo.offset > 0 && rInfo.size > 0);
-//       if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
-//         tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
-//                   TSDB_FILE_FULL_NAME(pMFile));
-//         terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       maxBufSize = TMAX(maxBufSize, rInfo.size);
-
-//       if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
-//         tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-// #if 0
-//       pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
-//       pStore->info.nRecords++;
-// #endif
-//     }
-//   }
-
-//   if (recoverMeta) {
-//     pBuf = taosMemoryMalloc((size_t)maxBufSize);
-//     if (pBuf == NULL) {
-//       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-//       tsdbCloseMFile(pMFile);
-//       return -1;
-//     }
-
-//     SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
-//     while (pRecord) {
-//       if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
-//         tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
-//       if (nread < 0) {
-//         tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       if (nread < pRecord->size) {
-//         tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
-//                   REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
-//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
-//         tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       pRecord = taosHashIterate(pfs->metaCache, pRecord);
-//     }
-
-//     tsdbOrgMeta(pRepo);
-//   }
-
-//   tsdbCloseMFile(pMFile);
-//   taosMemoryFreeClear(pBuf);
-//   return 0;
-// }
-
 static int tsdbScanRootDir(STsdb *pRepo) {
   char            rootDir[TSDB_FILENAME_LEN];
   char            bname[TSDB_FILENAME_LEN];
@@ -983,127 +791,6 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) {
   return false;
 }
 
-// static int tsdbRestoreMeta(STsdb *pRepo) {
-//   char         rootDir[TSDB_FILENAME_LEN];
-//   char         bname[TSDB_FILENAME_LEN];
-//   STfsDir *       tdir = NULL;
-//   const STfsFile *pf = NULL;
-//   const char * pattern = "^meta(-ver[0-9]+)?$";
-//   regex_t      regex;
-//   STsdbFS *    pfs = REPO_FS(pRepo);
-
-//   regcomp(&regex, pattern, REG_EXTENDED);
-
-//   tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
-
-//   tsdbGetRootDir(REPO_ID(pRepo), rootDir);
-
-//   tdir = tfsOpendir(rootDir);
-//   if (tdir == NULL) {
-//     tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
-//     regfree(&regex);
-//     return -1;
-//   }
-
-//   while ((pf = tfsReaddir(tdir))) {
-//     tfsBasename(pf, bname);
-
-//     if (strcmp(bname, "data") == 0) {
-//       // Skip the data/ directory
-//       continue;
-//     }
-
-//     if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
-//       // Skip current.t file
-//       tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname);
-//       (void)tfsremove(pf);
-//       continue;
-//     }
-
-//     int code = regexec(&regex, bname, 0, NULL, 0);
-//     if (code == 0) {
-//       // Match
-//       if (pfs->cstatus->pmf != NULL) {
-//         tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
-//                   TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname);
-//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//         tfsClosedir(tdir);
-//         regfree(&regex);
-//         return -1;
-//       } else {
-//         uint32_t _version = 0;
-//         if (strcmp(bname, "meta") != 0) {
-//           sscanf(bname, "meta-ver%" PRIu32, &_version);
-//           pfs->cstatus->meta.version = _version;
-//         }
-
-//         pfs->cstatus->pmf = &(pfs->cstatus->mf);
-//         pfs->cstatus->pmf->f = *pf;
-//         TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
-
-//         if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
-//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//           tfsClosedir(tdir);
-//           regfree(&regex);
-//           return -1;
-//         }
-
-//         if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
-//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//           tsdbCloseMFile(pfs->cstatus->pmf);
-//           tfsClosedir(tdir);
-//           regfree(&regex);
-//           return -1;
-//         }
-
-//         if (tsdbForceKeepFile) {
-//           struct stat tfstat;
-
-//           // Get real file size
-//           if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) {
-//             terrno = TAOS_SYSTEM_ERROR(errno);
-//             tsdbCloseMFile(pfs->cstatus->pmf);
-//             tfsClosedir(tdir);
-//             regfree(&regex);
-//             return -1;
-//           }
-
-//           if (pfs->cstatus->pmf->info.size != tfstat.st_size) {
-//             int64_t tfsize = pfs->cstatus->pmf->info.size;
-//             pfs->cstatus->pmf->info.size = tfstat.st_size;
-//             tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
-//                      TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size);
-//           }
-//         }
-
-//         tsdbCloseMFile(pfs->cstatus->pmf);
-//       }
-//     } else if (code == REG_NOMATCH) {
-//       // Not match
-//       tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname);
-//       tfsremove(pf);
-//       continue;
-//     } else {
-//       // Has other error
-//       tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
-//       terrno = TAOS_SYSTEM_ERROR(code);
-//       tfsClosedir(tdir);
-//       regfree(&regex);
-//       return -1;
-//     }
-//   }
-
-//   if (pfs->cstatus->pmf) {
-//     tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
-//   } else {
-//     tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
-//   }
-
-//   tfsClosedir(tdir);
-//   regfree(&regex);
-//   return 0;
-// }
-
 static int tsdbRestoreDFileSet(STsdb *pRepo) {
   char            dataDir[TSDB_FILENAME_LEN];
   char            bname[TSDB_FILENAME_LEN];

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -33,271 +33,6 @@ static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
 static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
 static int   tsdbRollBackDFile(SDFile *pDFile);
 
-#if 0
-// ============== SMFile
-void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
-  char fname[TSDB_FILENAME_LEN];
-
-  TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
-
-  memset(&(pMFile->info), 0, sizeof(pMFile->info));
-  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
-
-  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
-  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
-}
-
-void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
-  *pMFile = *pOMFile;
-  TSDB_FILE_SET_CLOSED(pMFile);
-}
-
-int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
-  int tlen = 0;
-
-  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
-  tlen += tfsEncodeFile(buf, &(pMFile->f));
-
-  return tlen;
-}
-
-void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
-  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
-  buf = tfsDecodeFile(buf, &(pMFile->f));
-  TSDB_FILE_SET_CLOSED(pMFile);
-
-  return buf;
-}
-
-int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
-  int tlen = 0;
-
-  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
-  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));
-
-  return tlen;
-}
-
-void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
-  char *aname;
-  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
-  buf = taosDecodeString(buf, &aname);
-  strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
-  TSDB_FILE_SET_CLOSED(pMFile);
-
-  taosMemoryFreeClear(aname);
-
-  return buf;
-}
-
-int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
-  if (from == NULL && to == NULL) return 0;
-
-  if (from != NULL) {
-    if (to == NULL) {
-      return tsdbRemoveMFile(from);
-    } else {
-      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
-        if (from->info.size > to->info.size) {
-          tsdbRollBackMFile(to);
-        }
-      } else {
-        return tsdbRemoveMFile(from);
-      }
-    }
-  }
-
-  return 0;
-}
-
-int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
-  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);
-
-  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
-  if (pMFile->fd < 0) {
-    if (errno == ENOENT) {
-      // Try to create directory recursively
-      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
-      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
-        taosMemoryFreeClear(s);
-        return -1;
-      }
-      taosMemoryFreeClear(s);
-
-      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
-      if (pMFile->fd < 0) {
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-    } else {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-  }
-
-  if (!updateHeader) {
-    return 0;
-  }
-
-  pMFile->info.size += TSDB_FILE_HEAD_SIZE;
-
-  if (tsdbUpdateMFileHeader(pMFile) < 0) {
-    tsdbCloseMFile(pMFile);
-    tsdbRemoveMFile(pMFile);
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbUpdateMFileHeader(SMFile *pMFile) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
-    return -1;
-  }
-
-  void *ptr = buf;
-  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
-  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
-    return -1;
-  }
-
-  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  tsdbDecodeMFInfo(buf, pInfo);
-  return 0;
-}
-
-int tsdbScanAndTryFixMFile(STsdb *pRepo) {
-  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
-  struct stat mfstat;
-  SMFile      mf;
-
-  if (pMFile == NULL) {
-    // No meta file, no need to scan
-    return 0;
-  }
-
-  tsdbInitMFileEx(&mf, pMFile);
-
-  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
-    tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
-              TSDB_FILE_FULL_NAME(pMFile));
-    pRepo->state |= TSDB_STATE_BAD_META;
-    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
-    return 0;
-  }
-
-  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (pMFile->info.size < mfstat.st_size) {
-    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
-      return -1;
-    }
-
-    if (taosFtruncate(mf.fd, mf.info.size) < 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      tsdbCloseMFile(&mf);
-      return -1;
-    }
-
-    if (tsdbUpdateMFileHeader(&mf) < 0) {
-      tsdbCloseMFile(&mf);
-      return -1;
-    }
-
-    tsdbCloseMFile(&mf);
-    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-             mfstat.st_size, pMFile->info.size);
-  } else if (pMFile->info.size > mfstat.st_size) {
-    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
-              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
-    pRepo->state |= TSDB_STATE_BAD_META;
-    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return 0;
-  } else {
-    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
-  }
-
-  return 0;
-}
-
-int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
-  int tlen = 0;
-
-  tlen += taosEncodeVariantI64(buf, pInfo->size);
-  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
-  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
-  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
-  tlen += taosEncodeFixedU32(buf, pInfo->magic);
-
-  return tlen;
-}
-
-void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
-  buf = taosDecodeVariantI64(buf, &(pInfo->size));
-  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
-  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
-
-  return buf;
-}
-
-static int tsdbRollBackMFile(SMFile *pMFile) {
-  SMFile mf;
-
-  tsdbInitMFileEx(&mf, pMFile);
-
-  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
-    return -1;
-  }
-
-  if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    tsdbCloseMFile(&mf);
-    return -1;
-  }
-
-  if (tsdbUpdateMFileHeader(&mf) < 0) {
-    tsdbCloseMFile(&mf);
-    return -1;
-  }
-
-  TSDB_FILE_FSYNC(&mf);
-
-  tsdbCloseMFile(&mf);
-  return 0;
-}
-
-#endif
-
 // ============== Operations on SDFile
 void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
   char fname[TSDB_FILENAME_LEN];
@@ -310,7 +45,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
   pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
   pDFile->info.fver = tsdbGetDFSVersion(ftype);
 
-  tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname);
+  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname);
   tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
 }
 

source/dnode/vnode/src/tsdb/tsdbOpen.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+
+int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb) {
+  STsdb *pTsdb = NULL;
+  int    slen = 0;
+
+  *ppTsdb = NULL;
+  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_TSDB_DIR) + 3;
+
+  // create handle
+  pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
+  if (pTsdb == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pTsdb->path = (char *)&pTsdb[1];
+  sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
+          VNODE_TSDB_DIR);
+  pTsdb->pVnode = pVnode;
+  pTsdb->repoLocked = false;
+  tdbMutexInit(&pTsdb->mutex, NULL);
+  pTsdb->config = pVnode->config.tsdbCfg;
+  pTsdb->fs = tsdbNewFS(&pTsdb->config);
+
+  // create dir (TODO: use tfsMkdir)
+  taosMkDir(pTsdb->path);
+
+  // open tsdb
+  if (tsdbOpenFS(pTsdb) < 0) {
+    goto _err;
+  }
+
+  tsdbDebug("vgId: %d tsdb is opened", TD_VID(pVnode));
+
+  *ppTsdb = pTsdb;
+  return 0;
+
+_err:
+  taosMemoryFree(pTsdb);
+  return -1;
+}
+
+int tsdbClose(STsdb *pTsdb) {
+  if (pTsdb) {
+    tsdbCloseFS(pTsdb);
+    tsdbFreeFS(pTsdb->fs);
+    taosMemoryFree(pTsdb);
+  }
+  return 0;
+}
+
+int tsdbLockRepo(STsdb *pTsdb) {
+  int code = taosThreadMutexLock(&pTsdb->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  pTsdb->repoLocked = true;
+  return 0;
+}
+
+int tsdbUnlockRepo(STsdb *pTsdb) {
+  ASSERT(IS_REPO_LOCKED(pTsdb));
+  pTsdb->repoLocked = false;
+  int code = taosThreadMutexUnlock(&pTsdb->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -454,7 +454,7 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond) {
   if (emptyQueryTimewindow(pTsdbReadHandle)) {
     if (pCond->order != pTsdbReadHandle->order) {
       pTsdbReadHandle->order = pCond->order;
-      TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t);
+      TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey);
     }
 
     return;
@@ -924,7 +924,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
   pHandle->cur.mixBlock = true;
 
   if (!ASCENDING_TRAVERSE(pHandle->order)) {
-    TSWAP(win->skey, win->ekey, TSKEY);
+    TSWAP(win->skey, win->ekey);
   }
 
   return true;
@@ -1203,7 +1203,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
       // update the last key value
       pCheckInfo->lastKey = cur->win.ekey + step;
       if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-        TSWAP(cur->win.skey, cur->win.ekey, TSKEY);
+        TSWAP(cur->win.skey, cur->win.ekey);
       }
 
       cur->mixBlock = true;
@@ -1701,7 +1701,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa
   int32_t end = endPos;
 
   if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-    TSWAP(start, end, int32_t);
+    TSWAP(start, end);
   }
 
   assert(pTsdbReadHandle->outputCapacity >= (end - start + 1));
@@ -1932,7 +1932,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
        ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order)));
 
   if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
-    TSWAP(cur->win.skey, cur->win.ekey, TSKEY);
+    TSWAP(cur->win.skey, cur->win.ekey);
   }
 
   moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -23,8 +23,6 @@ const SVnodeCfg vnodeCfgDefault = {
     .szCache = 256,
     .szBuf = 96 * 1024 * 1024,
     .isHeap = false,
-    .ttl = 0,
-    .keep = 0,
     .streamMode = 0,
     .isWeak = 0,
     .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI,
@@ -58,8 +56,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "szCache", pCfg->szCache) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
@@ -72,7 +68,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
-  if (tjsonAddIntegerToObject(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
@@ -109,8 +104,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "szCache", pCfg->szCache) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "szBuf", pCfg->szBuf) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeap) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
@@ -123,7 +116,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
-  if (tjsonGetNumberValue(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -138,3 +138,13 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
   pLoad->numOfBatchInsertSuccessReqs = 4;
   return 0;
 }
+
+void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) {
+  if (dbname) {
+    *dbname = pVnode->config.dbname;
+  }
+
+  if (vgId) {
+    *vgId = TD_VID(pVnode);
+  }
+}
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -142,7 +142,7 @@ _err:
 
 int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   vTrace("message in vnode query queue is processing");
-  SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config};
+  SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode};
 
   switch (pMsg->msgType) {
     case TDMT_VND_QUERY:
@@ -369,7 +369,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq,
   tCoderClear(&coder);
 
   // prepare rsp
-  tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen);
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVCreateTbBatchRsp, &rsp, pRsp->contLen, ret);
   pRsp->pCont = rpcMallocCont(pRsp->contLen);
   if (pRsp->pCont == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;

source/libs/executor/inc/executorimpl.h

@@ -380,7 +380,7 @@ typedef struct SStreamBlockScanInfo {
 typedef struct SSysTableScanInfo {
   union {
     void* pTransporter;
-    void* readHandle;
+    SReadHandle readHandle;
   };
 
   SRetrieveMetaTableRsp* pRsp;

source/libs/function/src/tudf.c

@@ -213,7 +213,7 @@ enum {
 
 int32_t getUdfdPipeName(char* pipeName, int32_t size) {
   char    dnodeId[8] = {0};
-  size_t  dnodeIdSize;
+  size_t  dnodeIdSize = sizeof(dnodeId);
   int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize);
   if (err != 0) {
     dnodeId[0] = '1';

source/libs/parser/src/parInsertData.c

@@ -159,9 +159,10 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star
 int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) {
   SCoder coder = {0};
   char* pBuf;
-   int32_t len;
+  int32_t len;
 
-  tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len);
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret);
   if (pBlocks->nAllocSize - pBlocks->size < len) {
     pBlocks->nAllocSize += len + pBlocks->rowSize;
     char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize);