Merge remote-tracking branch 'origin/feature/vnode_refact1' into feature/3.0_liaohj

.devcontainer/Dockerfile

@@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
 # [Optional] Uncomment this section to install additional packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
-RUN apt-get update && apt-get -y install tree vim tmux
+RUN apt-get update && apt-get -y install tree vim tmux python3-pip

contrib/CMakeLists.txt

@@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
 # ================================================================================================
 # googletest
 if(${BUILD_TEST})
-    add_subdirectory(googletest)
+    add_subdirectory(googletest EXCLUDE_FROM_ALL)
     target_include_directories(
         gtest
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
@@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
 option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
 option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
-add_subdirectory(cJson)
+add_subdirectory(cJson EXCLUDE_FROM_ALL)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
@@ -152,7 +152,7 @@ target_include_directories(
 unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
 # lz4
-add_subdirectory(lz4/build/cmake)
+add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
@@ -160,7 +160,7 @@ target_include_directories(
 
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
-add_subdirectory(zlib)
+add_subdirectory(zlib EXCLUDE_FROM_ALL)
 target_include_directories(
     zlibstatic
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 # leveldb
 if(${BUILD_WITH_LEVELDB})
     option(LEVELDB_BUILD_TESTS "" OFF)
-    add_subdirectory(leveldb)
+    add_subdirectory(leveldb EXCLUDE_FROM_ALL)
     target_include_directories(
         leveldb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
@@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB})
     option(WITH_TOOLS "" OFF)
     option(WITH_LIBURING "" OFF)
     option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
-    add_subdirectory(rocksdb)
+    add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
     target_include_directories(
         rocksdb
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
@@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB})
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
     option(ENABLE_TEST "Enable the tests" OFF)
-    add_subdirectory(lucene)
+    add_subdirectory(lucene EXCLUDE_FROM_ALL)
     target_include_directories(
       lucene++ 
       PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
@@ -213,14 +213,14 @@ endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
 if(${BUILD_WITH_NURAFT})
-    add_subdirectory(nuraft)
+    add_subdirectory(nuraft EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_NURAFT})
 
 # pthread
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE release)
     add_definitions(-DPTW32_STATIC_LIB) 
-    add_subdirectory(pthread)
+    add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
     add_library(pthread STATIC IMPORTED GLOBAL)
     SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
@@ -228,12 +228,12 @@ endif()
 
 # iconv
 if(${BUILD_WITH_ICONV})
-    add_subdirectory(iconv)
+    add_subdirectory(iconv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_ICONV})
 
 # wingetopt
 if(${BUILD_WINGETOPT})
-    add_subdirectory(wingetopt)
+    add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
 endif(${BUILD_WINGETOPT})
 
 # msvcregex
@@ -293,7 +293,7 @@ if(${BUILD_WITH_UV})
         MESSAGE("Windows need set no-sign-compare")
         add_compile_options(-Wno-sign-compare)
     endif ()
-    add_subdirectory(libuv)
+    add_subdirectory(libuv EXCLUDE_FROM_ALL)
 endif(${BUILD_WITH_UV})
 
 # BDB
@@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE})
 # Build test
 # ================================================================================================
 if(${BUILD_DEPENDENCY_TESTS})
-    add_subdirectory(test)
+    add_subdirectory(test EXCLUDE_FROM_ALL)
 endif(${BUILD_DEPENDENCY_TESTS})

include/common/tmsg.h

@@ -177,6 +177,7 @@ typedef struct SField {
   char    name[TSDB_COL_NAME_LEN];
   uint8_t type;
   int32_t bytes;
+  int8_t  flags;
 } SField;
 
 typedef struct SRetention {
@@ -302,13 +303,11 @@ typedef struct {
   int32_t ttl;
   int32_t numOfColumns;
   int32_t numOfTags;
-  int32_t numOfSmas;
   int32_t commentLen;
   int32_t ast1Len;
   int32_t ast2Len;
   SArray* pColumns;  // array of SField
   SArray* pTags;     // array of SField
-  SArray* pSmas;     // array of SField
   char*   comment;
   char*   pAst1;
   char*   pAst2;
@@ -2384,9 +2383,9 @@ typedef struct {
   int32_t  epoch;
   uint64_t reqId;
   int64_t  consumerId;
-  int64_t  blockingTime;
+  int64_t  waitTime;
   int64_t  currentOffset;
-} SMqPollReqV2;
+} SMqPollReq;
 
 typedef struct {
   int32_t vgId;
@@ -2401,53 +2400,6 @@ typedef struct {
   SSchemaWrapper schema;
 } SMqSubTopicEp;
 
-typedef struct {
-  SMqRspHead head;
-  int64_t    reqOffset;
-  int64_t    rspOffset;
-  int32_t    skipLogNum;
-  int32_t    dataLen;
-  SArray*    blockPos;   // beginning pos for each SRetrieveTableRsp
-  void*      blockData;  // serialized batched SRetrieveTableRsp
-} SMqPollRspV2;
-
-static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
-  tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
-  tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
-  tlen += taosEncodeFixedI32(buf, pRsp->dataLen);
-  if (pRsp->dataLen != 0) {
-    int32_t sz = taosArrayGetSize(pRsp->blockPos);
-    tlen += taosEncodeFixedI32(buf, sz);
-    for (int32_t i = 0; i < sz; i++) {
-      int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
-      tlen += taosEncodeFixedI32(buf, blockPos);
-    }
-    tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) {
-  buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
-  buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
-  buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
-  buf = taosDecodeFixedI32(buf, &pRsp->dataLen);
-  if (pRsp->dataLen != 0) {
-    int32_t sz;
-    buf = taosDecodeFixedI32(buf, &sz);
-    pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t));
-    for (int32_t i = 0; i < sz; i++) {
-      int32_t blockPos;
-      buf = taosDecodeFixedI32(buf, &blockPos);
-      taosArrayPush(pRsp->blockPos, &blockPos);
-    }
-    buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen);
-  }
-  return (void*)buf;
-}
-
 typedef struct {
   SMqRspHead head;
   int64_t    reqOffset;

include/dnode/mnode/sdb/sdb.h

@@ -18,6 +18,11 @@
 
 #include "os.h"
 
+#include "thash.h"
+#include "tlockfree.h"
+#include "tlog.h"
+#include "tmsg.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -135,7 +140,7 @@ typedef enum {
 typedef struct SSdb SSdb;
 typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
-typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj);
+typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
@@ -326,9 +331,29 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
+void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
+typedef struct SSdb {
+  SMnode     *pMnode;
+  char       *currDir;
+  char       *syncDir;
+  char       *tmpDir;
+  int64_t     lastCommitVer;
+  int64_t     curVer;
+  int64_t     tableVer[SDB_MAX];
+  int64_t     maxId[SDB_MAX];
+  EKeyType    keyTypes[SDB_MAX];
+  SHashObj   *hashObjs[SDB_MAX];
+  SRWLatch    locks[SDB_MAX];
+  SdbInsertFp insertFps[SDB_MAX];
+  SdbUpdateFp updateFps[SDB_MAX];
+  SdbDeleteFp deleteFps[SDB_MAX];
+  SdbDeployFp deployFps[SDB_MAX];
+  SdbEncodeFp encodeFps[SDB_MAX];
+  SdbDecodeFp decodeFps[SDB_MAX];
+} SSdb;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "function.h"
+#include "querynodes.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -123,7 +123,7 @@ struct SCatalog;
 
 typedef struct SFmGetFuncInfoParam {
   struct SCatalog* pCtg;
-  void *pRpc;
+  void*            pRpc;
   const SEpSet*    pMgmtEps;
   char*            pErrBuf;
   int32_t          errBufLen;
@@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
+bool fmIsScanPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);

include/libs/nodes/plannodes.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 #include "tname.h"
 
 typedef struct SLogicNode {
@@ -33,16 +33,12 @@ typedef struct SLogicNode {
   int32_t            optimizedFlag;
 } SLogicNode;
 
-typedef enum EScanType {
-  SCAN_TYPE_TAG,
-  SCAN_TYPE_TABLE,
-  SCAN_TYPE_SYSTEM_TABLE,
-  SCAN_TYPE_STREAM
-} EScanType;
+typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
 
 typedef struct SScanLogicNode {
   SLogicNode         node;
   SNodeList*         pScanCols;
+  SNodeList*         pScanPseudoCols;
   struct STableMeta* pMeta;
   SVgroupsInfo*      pVgroupList;
   EScanType          scanType;
@@ -95,11 +91,7 @@ typedef struct SExchangeLogicNode {
   uint8_t    precision;
 } SExchangeLogicNode;
 
-typedef enum EWindowType {
-  WINDOW_TYPE_INTERVAL = 1,
-  WINDOW_TYPE_SESSION,
-  WINDOW_TYPE_STATE
-} EWindowType;
+typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
 
 typedef struct SWindowLogicNode {
   SLogicNode  node;
@@ -187,6 +179,7 @@ typedef struct SPhysiNode {
 typedef struct SScanPhysiNode {
   SPhysiNode node;
   SNodeList* pScanCols;
+  SNodeList* pScanPseudoCols;
   uint64_t   uid;  // unique id of the table
   int8_t     tableType;
   SName      tableName;
@@ -315,7 +308,7 @@ typedef struct SDataInserterNode {
   SDataSinkNode sink;
   int32_t       numOfTables;
   uint32_t      size;
-  char     *pData;
+  char*         pData;
 } SDataInserterNode;
 
 typedef struct SSubplan {
@@ -333,11 +326,7 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;     // data of the subplan flow into the datasink
 } SSubplan;
 
-typedef enum EExplainMode {
-  EXPLAIN_MODE_DISABLE = 1,
-  EXPLAIN_MODE_STATIC,
-  EXPLAIN_MODE_ANALYZE
-} EExplainMode;
+typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
 
 typedef struct SExplainInfo {
   EExplainMode mode;

include/libs/nodes/querynodes.h

@@ -293,7 +293,10 @@ typedef struct SExplainStmt {
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
-int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
+typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
+
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
+                            SNodeList** pCols);
 
 typedef bool (*FFuncClassifier)(int32_t funcId);
 int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);

include/libs/transport/trpc.h

@@ -59,9 +59,13 @@ typedef struct {
   void *   pNode;
 } SNodeMsg;
 
-typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
+typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
 typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
-typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
+///
+// // SRpcMsg code
+// REDIERE,
+// NOT READY, EpSet
+typedef bool (*RpcRfp)(int32_t code);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port

include/libs/wal/wal.h

@@ -72,8 +72,6 @@ extern "C" {
 #define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
 #define WAL_MAGIC        0xFAFBFCFDULL
 
-#define WAL_CUR_FAILED 1
-
 #pragma pack(push, 1)
 typedef enum {
   TAOS_WAL_NOLOG = 0,

source/client/src/clientImpl.c

@@ -246,6 +246,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
 
   pResInfo->numOfCols = numOfCols;
   // TODO handle memory leak
+  if (pResInfo->fields != NULL) {
+    taosMemoryFree(pResInfo->fields);
+  }
+  if (pResInfo->userFields != NULL) {
+    taosMemoryFree(pResInfo->userFields);
+  }
   pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
   pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
 

source/client/src/tmq.c

@@ -16,7 +16,6 @@
 #include "clientInt.h"
 #include "clientLog.h"
 #include "parser.h"
-#include "planner.h"
 #include "tdatablock.h"
 #include "tdef.h"
 #include "tglobal.h"
@@ -68,7 +67,7 @@ struct tmq_conf_t {
   char*          user;
   char*          pass;
   char*          db;
-  tmq_commit_cb* commit_cb;
+  tmq_commit_cb* commitCb;
 };
 
 struct tmq_t {
@@ -115,8 +114,8 @@ enum {
 
 enum {
   TMQ_CONSUMER_STATUS__INIT = 0,
-  TMQ_CONSUMER_STATUS__SUBSCRIBED,
   TMQ_CONSUMER_STATUS__READY,
+  TMQ_CONSUMER_STATUS__NO_TOPIC,
 };
 
 enum {
@@ -175,7 +174,6 @@ typedef struct {
   int32_t         epoch;
   int32_t         vgId;
   tsem_t          rspSem;
-  int32_t         sync;
 } SMqPollCbParam;
 
 typedef struct {
@@ -300,6 +298,7 @@ void tmqAssignDelayedHbTask(void* param, void* tmrId) {
   int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
   *pTaskType = TMQ_DELAYED_TASK__HB;
   taosWriteQitem(tmq->delayedTask, pTaskType);
+  tsem_post(&tmq->rspSem);
 }
 
 void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
@@ -307,6 +306,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
   int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
   *pTaskType = TMQ_DELAYED_TASK__COMMIT;
   taosWriteQitem(tmq->delayedTask, pTaskType);
+  tsem_post(&tmq->rspSem);
 }
 
 void tmqAssignDelayedReportTask(void* param, void* tmrId) {
@@ -314,6 +314,7 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) {
   int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
   *pTaskType = TMQ_DELAYED_TASK__REPORT;
   taosWriteQitem(tmq->delayedTask, pTaskType);
+  tsem_post(&tmq->rspSem);
 }
 
 int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
@@ -364,7 +365,6 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
   pParam->rspErr = code;
   tmq_t* tmq = pParam->tmq;
-  atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__SUBSCRIBED);
   tsem_post(&pParam->rspSem);
   return 0;
 }
@@ -385,14 +385,16 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
   }
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i);
-    tmq_list_append(*topics, strdup(topic->topicName));
+    tmq_list_append(*topics, topic->topicName);
   }
   return TMQ_RESP_ERR__SUCCESS;
 }
 
 tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) {
   tmq_list_t*    lst = tmq_list_new();
-  return tmq_subscribe(tmq, lst);
+  tmq_resp_err_t rsp = tmq_subscribe(tmq, lst);
+  tmq_list_destroy(lst);
+  return rsp;
 }
 
 #if 0
@@ -475,7 +477,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   strcpy(pTmq->groupId, conf->groupId);
   pTmq->autoCommit = conf->autoCommit;
   pTmq->autoCommitInterval = conf->autoCommitInterval;
-  pTmq->commit_cb = conf->commit_cb;
+  pTmq->commit_cb = conf->commitCb;
   pTmq->resetOffsetCfg = conf->resetOffset;
 
   // assign consumerId
@@ -655,6 +657,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
+  // avoid double free if msg is sent
+  buf = NULL;
+
   tsem_wait(&param.rspSem);
   tsem_destroy(&param.rspSem);
 
@@ -686,7 +691,7 @@ FAIL:
 
 void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
   //
-  conf->commit_cb = cb;
+  conf->commitCb = cb;
 }
 
 TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
@@ -798,7 +803,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
     // do not write into queue since updating epoch reset
     tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
             tmqEpoch);
-    /*tsem_post(&tmq->rspSem);*/
+    tsem_post(&tmq->rspSem);
     return 0;
   }
 
@@ -806,25 +811,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
     tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
   }
 
-#if 0
-  if (pParam->sync == 1) {
-    /**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/
-    *pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
-    if (*pParam->msg) {
-      memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
-      tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
-      if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
-        pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
-      }
-      taosWriteQitem(tmq->mqueue, *pParam->msg);
-      tsem_post(&pParam->rspSem);
-      return 0;
-    }
-    tsem_post(&pParam->rspSem);
-    return -1;
-  }
-#endif
-
   SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
   if (pRspWrapper == NULL) {
     tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
@@ -843,14 +829,14 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
            pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
 
   taosWriteQitem(tmq->mqueue, pRspWrapper);
-  /*tsem_post(&tmq->rspSem);*/
+  tsem_post(&tmq->rspSem);
 
   return 0;
 CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
-  /*tsem_post(&tmq->rspSem);*/
+  tsem_post(&tmq->rspSem);
   return -1;
 }
 
@@ -927,6 +913,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
   if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
   taosHashCleanup(pHash);
   tmq->clientTopics = newTopics;
+
+  if (taosArrayGetSize(tmq->clientTopics) == 0)
+    atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC);
+  else
+    atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
+
   atomic_store_32(&tmq->epoch, epoch);
   return set;
 }
@@ -955,9 +947,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
     /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
     /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
-    if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
-      atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
-    }
+    tmqUpdateEp(tmq, head->epoch, &rsp);
     tDeleteSMqAskEpRsp(&rsp);
   } else {
     SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
@@ -972,7 +962,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
 
     taosWriteQitem(tmq->mqueue, pWrapper);
-    /*tsem_post(&tmq->rspSem);*/
+    tsem_post(&tmq->rspSem);
     taosMemoryFree(pParam);
   }
 
@@ -1076,7 +1066,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
   return TMQ_RESP_ERR__FAIL;
 }
 
-SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
   int64_t reqOffset;
   if (pVg->currentOffset >= 0) {
     reqOffset = pVg->currentOffset;
@@ -1088,7 +1078,7 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
     reqOffset = tmq->resetOffsetCfg;
   }
 
-  SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2));
+  SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
   if (pReq == NULL) {
     return NULL;
   }
@@ -1101,14 +1091,14 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
   pReq->subKey[tlen] = TMQ_SEPARATOR;
   strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
 
-  pReq->blockingTime = blockingTime;
+  pReq->waitTime = waitTime;
   pReq->consumerId = tmq->consumerId;
   pReq->epoch = tmq->epoch;
   pReq->currentOffset = reqOffset;
   pReq->reqId = generateRequestId();
 
   pReq->head.vgId = htonl(pVg->vgId);
-  pReq->head.contLen = htonl(sizeof(SMqPollReqV2));
+  pReq->head.contLen = htonl(sizeof(SMqPollReq));
   return pReq;
 }
 
@@ -1130,7 +1120,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
-int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
+int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) {
   /*printf("call poll\n");*/
   for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
@@ -1151,17 +1141,17 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 #endif
       }
       atomic_store_32(&pVg->vgSkipCnt, 0);
-      SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
+      SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, waitTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        /*tsem_post(&tmq->rspSem);*/
+        tsem_post(&tmq->rspSem);
         return -1;
       }
       SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
       if (pParam == NULL) {
         taosMemoryFree(pReq);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        /*tsem_post(&tmq->rspSem);*/
+        tsem_post(&tmq->rspSem);
         return -1;
       }
       pParam->tmq = tmq;
@@ -1169,20 +1159,19 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       pParam->pTopic = pTopic;
       pParam->vgId = pVg->vgId;
       pParam->epoch = tmq->epoch;
-      pParam->sync = 0;
 
       SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
       if (sendInfo == NULL) {
         taosMemoryFree(pReq);
         taosMemoryFree(pParam);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        /*tsem_post(&tmq->rspSem);*/
+        tsem_post(&tmq->rspSem);
         return -1;
       }
 
       sendInfo->msgInfo = (SDataBuf){
           .pData = pReq,
-          .len = sizeof(SMqPollReqV2),
+          .len = sizeof(SMqPollReq),
           .handle = NULL,
       };
       sendInfo->requestId = pReq->reqId;
@@ -1222,7 +1211,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
   return 0;
 }
 
-SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
+SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) {
   while (1) {
     SMqRspWrapper* rspWrapper = NULL;
     taosGetQitem(tmq->qall, (void**)&rspWrapper);
@@ -1261,37 +1250,41 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
       taosFreeQitem(rspWrapper);
       if (pollIfReset && reset) {
         tscDebug("consumer %ld reset and repoll", tmq->consumerId);
-        tmqPollImpl(tmq, blockingTime);
+        tmqPollImpl(tmq, waitTime);
       }
     }
   }
 }
 
-TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
+TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
   SMqRspObj* rspObj;
   int64_t    startTime = taosGetTimestampMs();
 
-  rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
+  rspObj = tmqHandleAllRsp(tmq, wait_time, false);
   if (rspObj) {
     return (TAOS_RES*)rspObj;
   }
 
+  if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
+    return NULL;
+  }
+
   while (1) {
     tmqHandleAllDelayedTask(tmq);
-    tmqPollImpl(tmq, blocking_time);
-
-    /*tsem_wait(&tmq->rspSem);*/
+    tmqPollImpl(tmq, wait_time);
 
-    rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
+    rspObj = tmqHandleAllRsp(tmq, wait_time, false);
     if (rspObj) {
       return (TAOS_RES*)rspObj;
     }
-    if (blocking_time != 0) {
+    if (wait_time != 0) {
       int64_t endTime = taosGetTimestampMs();
-      if (endTime - startTime > blocking_time) {
+      int64_t leftTime = endTime - startTime;
+      if (leftTime > wait_time) {
         tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch);
         return NULL;
       }
+      tsem_timewait(&tmq->rspSem, leftTime * 1000);
     }
   }
 }

source/common/src/tmsg.c

@@ -517,7 +517,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
@@ -527,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
     if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+    if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
   }
 
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
@@ -534,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
     if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
-  }
-
-  for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
-    SField *pField = taosArrayGet(pReq->pSmas, i);
-    if (tEncodeI8(&encoder, pField->type) < 0) return -1;
-    if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
-    if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
+    if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
   }
 
   if (pReq->commentLen > 0) {
@@ -571,15 +565,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
 
   pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
   pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
-  pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField));
-  if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) {
+  if (pReq->pColumns == NULL || pReq->pTags == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
@@ -589,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
     if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
     if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+    if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
     if (taosArrayPush(pReq->pColumns, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -600,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
     if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
     if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
+    if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
     if (taosArrayPush(pReq->pTags, &field) == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
   }
 
-  for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
-    SField field = {0};
-    if (tDecodeI8(&decoder, &field.type) < 0) return -1;
-    if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
-    if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
-    if (taosArrayPush(pReq->pSmas, &field) == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-  }
-
   if (pReq->commentLen > 0) {
     pReq->comment = taosMemoryMalloc(pReq->commentLen);
     if (pReq->comment == NULL) return -1;
@@ -644,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
 void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
   taosArrayDestroy(pReq->pColumns);
   taosArrayDestroy(pReq->pTags);
-  taosArrayDestroy(pReq->pSmas);
   taosMemoryFreeClear(pReq->comment);
   taosMemoryFreeClear(pReq->pAst1);
   taosMemoryFreeClear(pReq->pAst2);
   pReq->pColumns = NULL;
   pReq->pTags = NULL;
-  pReq->pSmas = NULL;
 }
 
 int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {

source/dnode/mgmt/exe/dmMain.c

@@ -69,8 +69,8 @@ static void dmSetSignalHandle() {
 static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
   int32_t cmdEnvIndex = 0;
   if (argc < 2) return 0;
-  global.envCmd = taosMemoryMalloc(argc-1);
-  memset(global.envCmd, 0, argc-1);
+  global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*));
+  memset(global.envCmd, 0, (argc-1)*sizeof(char*));
   for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
       if (i < argc - 1) {

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -114,7 +114,6 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->tsdbCfg.keep2 = 3650;  // pCreate->daysToKeep0;
   pCfg->tsdbCfg.keep0 = 3650;  // pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep1 = 3650;  // pCreate->daysToKeep0;
-  pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;

source/dnode/mnode/impl/inc/mndDef.h

@@ -126,8 +126,6 @@ typedef enum {
   DND_REASON_OTHERS
 } EDndReason;
 
-typedef void (*TransCbFp)(SMnode* pMnode, void* param);
-
 typedef struct {
   int32_t    id;
   ETrnStage  stage;
@@ -150,8 +148,10 @@ typedef struct {
   int64_t    dbUid;
   char       dbname[TSDB_DB_FNAME_LEN];
   char       lastError[TSDB_TRANS_ERROR_LEN];
-  TransCbFp  transCbFp;
-  void*      transCbParam;
+  int32_t    startFunc;
+  int32_t    stopFunc;
+  int32_t    paramLen;
+  void*      param;
 } STrans;
 
 typedef struct {
@@ -358,13 +358,11 @@ typedef struct {
   int32_t  ttl;
   int32_t  numOfColumns;
   int32_t  numOfTags;
-  int32_t  numOfSmas;
   int32_t  commentLen;
   int32_t  ast1Len;
   int32_t  ast2Len;
   SSchema* pColumns;
   SSchema* pTags;
-  SSchema* pSmas;
   char*    comment;
   char*    pAst1;
   char*    pAst2;

source/dnode/mnode/impl/inc/mndInt.h

@@ -45,7 +45,7 @@ extern "C" {
 typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg);
 typedef int32_t (*MndInitFp)(SMnode *pMnode);
 typedef void (*MndCleanupFp)(SMnode *pMnode);
-typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
 typedef struct SQWorkerMgmt SQHandle;
 
@@ -84,7 +84,7 @@ typedef struct {
   int64_t timeseriesAllowed;
 } SGrantInfo;
 
-struct SMnode {
+typedef struct SMnode {
   int32_t       selfId;
   int64_t       clusterId;
   int8_t        replica;
@@ -109,7 +109,7 @@ struct SMnode {
   SGrantInfo    grant;
   MndMsgFp      msgFp[TDMT_MAX];
   SMsgCb        msgCb;
-};
+} SMnode;
 
 void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
 int64_t mndGenerateUid(char *name, int32_t len);

source/dnode/mnode/impl/inc/mndTrans.h

@@ -33,6 +33,15 @@ typedef struct {
   void   *pCont;
 } STransAction;
 
+typedef enum {
+  TEST_TRANS_START_FUNC = 1,
+  TEST_TRANS_STOP_FUNC = 2,
+  CONSUME_TRANS_START_FUNC = 3,
+  CONSUME_TRANS_STOP_FUNC = 4,
+} ETrnFuncType;
+
+typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
+
 int32_t mndInitTrans(SMnode *pMnode);
 void    mndCleanupTrans(SMnode *pMnode);
 
@@ -44,7 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
 int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
 void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
-void    mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param);
+void    mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen);
 void    mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);

source/dnode/mnode/impl/src/mndStb.c

@@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {}
 SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) +
-                 +pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
+  int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen +
+                 pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
   if (pRaw == NULL) goto _OVER;
 
@@ -91,7 +91,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
   SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER)
@@ -112,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
     SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
   }
 
-  for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
-    SSchema *pSchema = &pStb->pSmas[i];
-    SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER)
-    SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER)
-    SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER)
-    SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
-  }
-
   if (pStb->commentLen > 0) {
     SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
   }
@@ -178,15 +169,13 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER)
 
   pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
   pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema));
-  pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema));
-  if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) {
+  if (pStb->pColumns == NULL || pStb->pTags == NULL) {
     goto _OVER;
   }
 
@@ -206,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
     SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
   }
 
-  for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
-    SSchema *pSchema = &pStb->pSmas[i];
-    SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER)
-    SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER)
-    SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER)
-    SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
-  }
-
   if (pStb->commentLen > 0) {
     pStb->comment = taosMemoryCalloc(pStb->commentLen, 1);
     if (pStb->comment == NULL) goto _OVER;
@@ -291,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
     }
   }
 
-  if (pOld->numOfSmas < pNew->numOfSmas) {
-    void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema));
-    if (pSmas != NULL) {
-      taosMemoryFree(pOld->pSmas);
-      pOld->pSmas = pSmas;
-    } else {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
-      taosWUnLockLatch(&pOld->lock);
-    }
-  }
-
   if (pOld->commentLen < pNew->commentLen) {
     void *comment = taosMemoryMalloc(pNew->commentLen);
     if (comment != NULL) {
@@ -411,11 +380,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.schemaTag.nCols = pStb->numOfTags;
   req.schemaTag.pSchema = pStb->pTags;
 
-  // TODO: remove here
-  for (int iCol = 0; iCol < req.schema.nCols; iCol++) {
-    req.schema.pSchema[iCol].flags = SCHEMA_SMA_ON;
-  }
-
   if (req.rollup) {
     req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
     req.pRSmaParam.delay = pStb->delay;
@@ -674,7 +638,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
   stbObj.ttl = pCreate->ttl;
   stbObj.numOfColumns = pCreate->numOfColumns;
   stbObj.numOfTags = pCreate->numOfTags;
-  stbObj.numOfSmas = pCreate->numOfSmas;
   stbObj.commentLen = pCreate->commentLen;
   if (stbObj.commentLen > 0) {
     stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
@@ -707,8 +670,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
 
   stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema));
   stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema));
-  stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema));
-  if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) {
+  if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
@@ -718,6 +680,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     SSchema *pSchema = &stbObj.pColumns[i];
     pSchema->type = pField->type;
     pSchema->bytes = pField->bytes;
+    pSchema->flags = pField->flags;
     memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
     pSchema->colId = stbObj.nextColId;
     stbObj.nextColId++;
@@ -733,18 +696,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
     stbObj.nextColId++;
   }
 
-  for (int32_t i = 0; i < stbObj.numOfSmas; ++i) {
-    SField  *pField = taosArrayGet(pCreate->pSmas, i);
-    SSchema *pSchema = &stbObj.pSmas[i];
-    SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
-    if (pColSchema == NULL) {
-      mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
-      terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
-      return -1;
-    }
-    memcpy(pSchema, pColSchema, sizeof(SSchema));
-  }
-
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
   if (pTrans == NULL) goto _OVER;

source/dnode/mnode/impl/src/mndSync.c

@@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) {
 
   char path[PATH_MAX] = {0};
   snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
-  SWalCfg cfg = {.vgId = 1,
+  SWalCfg cfg = {
+      .vgId = 1,
       .fsyncPeriod = 0,
       .rollPeriod = -1,
       .segSize = -1,
       .retentionPeriod = -1,
       .retentionSize = -1,
-                 .level = TAOS_WAL_FSYNC};
+      .level = TAOS_WAL_FSYNC,
+  };
   pMgmt->pWal = walOpen(path, &cfg);
   if (pMgmt->pWal == NULL) return -1;
 
@@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
 
   int64_t first = walGetFirstVer(pWal);
   int64_t last = walGetLastVer(pWal);
-  mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
+  mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
 
   first = TMAX(lastSdbVer + 1, first);
   for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) {
     if (walReadWithHandle(pHandle, ver) < 0) {
-      mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver);
-      goto WAL_RESTORE_OVER;
+      mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr());
+      goto _OVER;
     }
 
     SWalHead *pHead = pHandle->pHead;
     int64_t   sdbVer = sdbUpdateVer(pSdb, 0);
     if (sdbVer + 1 != ver) {
       terrno = TSDB_CODE_SDB_INVALID_WAl_VER;
-      mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver);
-      goto WAL_RESTORE_OVER;
+      mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer);
+      goto _OVER;
     }
 
-    mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
+    mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
     if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) {
-      mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver);
-      goto WAL_RESTORE_OVER;
+      mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr());
+      goto _OVER;
     }
 
     sdbUpdateVer(pSdb, 1);
-    mDebug("wal:%" PRId64 ", is restored", ver);
+    mDebug("ver:%" PRId64 ", is restored", ver);
   }
 
   int64_t sdbVer = sdbUpdateVer(pSdb, 0);
-  mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer);
+  mDebug("restore wal finished, sdbver:%" PRId64, sdbVer);
 
   mndTransPullup(pMnode);
   sdbVer = sdbUpdateVer(pSdb, 0);
-  mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer);
+  mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer);
 
   if (sdbVer != lastSdbVer) {
     mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer);
     if (sdbWriteFile(pSdb) != 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walCommit(pWal, sdbVer) != 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walBeginSnapshot(pWal, sdbVer) < 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
 
     if (walEndSnapshot(pWal) < 0) {
-      goto WAL_RESTORE_OVER;
+      goto _OVER;
     }
   }
 
   code = 0;
 
-WAL_RESTORE_OVER:
+_OVER:
   walCloseReadHandle(pHandle);
   return code;
 }
@@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
   int64_t ver = sdbUpdateVer(pSdb, 1);
   if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) {
     sdbUpdateVer(pSdb, -1);
-    mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver);
+    mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr());
     return -1;
   }
 
-  mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver);
+  mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw);
   walCommit(pWal, ver);
   walFsync(pWal, true);
 

source/dnode/mnode/impl/src/mndTelem.c

@@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) {
   taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
   mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
 
-  mDebug("mnode telemetry is initialized");
   return 0;
 }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans);
 static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
 static int32_t  mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
 static int32_t  mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
-static int32_t  mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
+static int32_t  mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc);
 
 static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
 static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
@@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER)
   }
 
+  SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER)
+  if (pTrans->param != NULL) {
+    SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER)
+  }
+
   SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER)
   SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER)
 
@@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
     action.pCont = NULL;
   }
 
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER)
+  if (pTrans->paramLen != 0) {
+    pTrans->param = taosMemoryMalloc(pTrans->paramLen);
+    SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER);
+  }
+
   SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER)
 
   terrno = 0;
@@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) {
   }
 }
 
+static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) {
+  mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen);
+}
+
+static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) {
+  mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen);
+}
+
+static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
+  switch (ftype) {
+    case TEST_TRANS_START_FUNC:
+      return mndTransTestStartFunc;
+    case TEST_TRANS_STOP_FUNC:
+      return mndTransTestStopFunc;
+    default:
+      return NULL;
+  }
+}
+
 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
   // pTrans->stage = TRN_STAGE_PREPARE;
   mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
+
+  if (pTrans->startFunc > 0) {
+    TransCbFp fp = mndTransGetCbFp(pTrans->startFunc);
+    if (fp) {
+      (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
+    }
+  }
+
   return 0;
 }
 
@@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) {
     pTrans->rpcRsp = NULL;
     pTrans->rpcRspLen = 0;
   }
+  if (pTrans->param != NULL) {
+    taosMemoryFree(pTrans->param);
+    pTrans->param = NULL;
+    pTrans->paramLen = 0;
+  }
 }
 
-static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
-  mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
+static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
+  mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
+         callFunc);
+  if (pTrans->stopFunc > 0 && callFunc) {
+    TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc);
+    if (fp) {
+      (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
+    }
+  }
+
   mndTransDropData(pTrans);
   return 0;
 }
@@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
     return NULL;
   }
 
-  mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans);
+  mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans);
   return pTrans;
 }
 
@@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) {
 void mndTransDrop(STrans *pTrans) {
   if (pTrans != NULL) {
     mndTransDropData(pTrans);
-    mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
+    mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans);
     taosMemoryFreeClear(pTrans);
   }
 }
@@ -574,9 +629,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) {
   pTrans->rpcRspLen = contLen;
 }
 
-void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) {
-  pTrans->transCbFp = fp;
-  pTrans->transCbParam = param;
+void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) {
+  pTrans->startFunc = startFunc;
+  pTrans->stopFunc = stopFunc;
+  pTrans->param = param;
+  pTrans->paramLen = paramLen;
 }
 
 void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
@@ -712,8 +769,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
   pNew->rpcRefId = pTrans->rpcRefId;
   pNew->rpcRsp = pTrans->rpcRsp;
   pNew->rpcRspLen = pTrans->rpcRspLen;
-  pNew->transCbFp = pTrans->transCbFp;
-  pNew->transCbParam = pTrans->transCbParam;
   pTrans->rpcRsp = NULL;
   pTrans->rpcRspLen = 0;
 
@@ -1125,10 +1180,6 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
 
   mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
 
-  if (pTrans->transCbFp != NULL) {
-    (*pTrans->transCbFp)(pMnode, pTrans->transCbParam);
-  }
-
   return continueExec;
 }
 

source/dnode/mnode/impl/src/mndUser.c

@@ -21,8 +21,8 @@
 #include "mndTrans.h"
 #include "tbase64.h"
 
-#define TSDB_USER_VER_NUMBER   1
-#define TSDB_USER_RESERVE_SIZE 64
+#define USER_VER_NUMBER   1
+#define USER_RESERVE_SIZE 64
 
 static int32_t  mndCreateDefaultUsers(SMnode *pMnode);
 static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
@@ -35,7 +35,7 @@ static int32_t  mndProcessCreateUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessAlterUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessDropUserReq(SNodeMsg *pReq);
 static int32_t  mndProcessGetUserAuthReq(SNodeMsg *pReq);
-static int32_t  mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextUser(SMnode *pMnode, void *pIter);
 
 int32_t mndInitUser(SMnode *pMnode) {
@@ -93,39 +93,39 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
 
   int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
   int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
-  int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
+  int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size);
-  if (pRaw == NULL) goto USER_ENCODE_OVER;
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
+  if (pRaw == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER)
-  SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER)
-  SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER)
-  SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER)
-  SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER)
-  SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
 
   char *db = taosHashIterate(pUser->readDbs, NULL);
   while (db != NULL) {
-    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
+    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
     db = taosHashIterate(pUser->readDbs, db);
   }
 
   db = taosHashIterate(pUser->writeDbs, NULL);
   while (db != NULL) {
-    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER);
+    SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
     db = taosHashIterate(pUser->writeDbs, db);
   }
 
-  SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER)
-  SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER)
+  SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
+  SDB_SET_DATALEN(pRaw, dataPos, _OVER)
 
   terrno = 0;
 
-USER_ENCODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr());
     sdbFreeRaw(pRaw);
@@ -140,55 +140,54 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
   int8_t sver = 0;
-  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
 
-  if (sver != TSDB_USER_VER_NUMBER) {
+  if (sver != USER_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
-    goto USER_DECODE_OVER;
+    goto _OVER;
   }
 
   SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
-  if (pRow == NULL) goto USER_DECODE_OVER;
+  if (pRow == NULL) goto _OVER;
 
   SUserObj *pUser = sdbGetRowObj(pRow);
-  if (pUser == NULL) goto USER_DECODE_OVER;
-
-  pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
-  pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
-  if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER;
+  if (pUser == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER)
-  SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER)
-  SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
 
   int32_t numOfReadDbs = 0;
   int32_t numOfWriteDbs = 0;
-  SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER)
-  SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
+  pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
+  pUser->writeDbs = taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
+  if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
 
   for (int32_t i = 0; i < numOfReadDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
-    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
+    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
     int32_t len = strlen(db) + 1;
     taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN);
   }
 
   for (int32_t i = 0; i < numOfWriteDbs; ++i) {
     char db[TSDB_DB_FNAME_LEN] = {0};
-    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER)
+    SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
     int32_t len = strlen(db) + 1;
     taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
   }
 
-  SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER)
+  SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
 
   terrno = 0;
 
-USER_DECODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
     taosHashCleanup(pUser->readDbs);
@@ -220,6 +219,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
   mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
   taosHashCleanup(pUser->readDbs);
   taosHashCleanup(pUser->writeDbs);
+  pUser->readDbs = NULL;
+  pUser->writeDbs = NULL;
   return 0;
 }
 
@@ -228,13 +229,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
   memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
   pOld->updateTime = pNew->updateTime;
 
-  void *tmp1 = pOld->readDbs;
-  pOld->readDbs = pNew->readDbs;
-  pNew->readDbs = tmp1;
-
-  void *tmp2 = pOld->writeDbs;
-  pOld->writeDbs = pNew->writeDbs;
-  pNew->writeDbs = tmp2;
+  TSWAP(pOld->readDbs, pNew->readDbs, (void *));
+  TSWAP(pOld->writeDbs, pNew->writeDbs, (void *));
 
   return 0;
 }
@@ -277,6 +273,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
   }
   sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
 
+  char *param = strdup("====> test code to be deleted later <=====");
+  mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
@@ -296,41 +295,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to create", createReq.user);
 
   if (createReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   if (createReq.pass[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, createReq.user);
   if (pUser != NULL) {
     terrno = TSDB_CODE_MND_USER_ALREADY_EXIST;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   if (mndCheckCreateUserAuth(pOperUser) != 0) {
-    goto CREATE_USER_OVER;
+    goto _OVER;
   }
 
   code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-CREATE_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to create since %s", createReq.user, terrstr());
   }
@@ -399,38 +398,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to alter", alterReq.user);
 
   if (alterReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   if (alterReq.pass[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, alterReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   memcpy(&newUser, pUser, sizeof(SUserObj));
   newUser.readDbs = mndDupDbHash(pUser->readDbs);
   newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
   if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   int32_t len = strlen(alterReq.dbname) + 1;
@@ -446,50 +445,50 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
   } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) {
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
     if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) {
     if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
     taosHashClear(newUser.readDbs);
   } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) {
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
     if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
     if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) {
       terrno = TSDB_CODE_MND_DB_NOT_EXIST;
-      goto ALTER_USER_OVER;
+      goto _OVER;
     }
   } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) {
     taosHashClear(newUser.writeDbs);
   } else {
     terrno = TSDB_CODE_MND_INVALID_ALTER_OPER;
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   newUser.updateTime = taosGetTimestampMs();
 
   if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) {
-    goto ALTER_USER_OVER;
+    goto _OVER;
   }
 
   code = mndUpdateUser(pMnode, pUser, &newUser, pReq);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-ALTER_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
   }
@@ -537,36 +536,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) {
 
   if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   mDebug("user:%s, start to drop", dropReq.user);
 
   if (dropReq.user[0] == 0) {
     terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, dropReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   pOperUser = mndAcquireUser(pMnode, pReq->user);
   if (pOperUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   if (mndCheckDropUserAuth(pOperUser) != 0) {
-    goto DROP_USER_OVER;
+    goto _OVER;
   }
 
   code = mndDropUser(pMnode, pReq, pUser);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-DROP_USER_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to drop since %s", dropReq.user, terrstr());
   }
@@ -586,7 +585,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
 
   if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   mTrace("user:%s, start to get auth", authReq.user);
@@ -594,7 +593,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   pUser = mndAcquireUser(pMnode, authReq.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_USER_NOT_EXIST;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   memcpy(authRsp.user, pUser->user, TSDB_USER_LEN);
@@ -622,7 +621,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   void   *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto GET_AUTH_OVER;
+    goto _OVER;
   }
 
   tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp);
@@ -631,7 +630,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
   pReq->rspLen = contLen;
   code = 0;
 
-GET_AUTH_OVER:
+_OVER:
   mndReleaseUser(pMnode, pUser);
   taosHashCleanup(authRsp.readDbs);
   taosHashCleanup(authRsp.writeDbs);
@@ -639,7 +638,7 @@ GET_AUTH_OVER:
   return code;
 }
 
-static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode   *pMnode = pReq->pNode;
   SSdb     *pSdb = pMnode->pSdb;
   int32_t   numOfRows = 0;
@@ -652,29 +651,29 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB
     if (pShow->pIter == NULL) break;
 
     cols = 0;
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
 
     char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]);
 
-    colDataAppend(pColInfo, numOfRows, (const char*) name, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
 
-    const char* src = pUser->superUser? "super":"normal";
-    char b[10+VARSTR_HEADER_SIZE] = {0};
+    const char *src = pUser->superUser ? "super" : "normal";
+    char        b[10 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src));
-    colDataAppend(pColInfo, numOfRows, (const char*) b, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)b, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
 
     cols++;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]);
-    colDataAppend(pColInfo, numOfRows, (const char*) name, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)name, false);
 
     numOfRows++;
     sdbRelease(pSdb, pUser);

source/dnode/mnode/impl/src/mndVgroup.c

@@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) {
                      .encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
                      .decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
                      .insertFp = (SdbInsertFp)mndVgroupActionInsert,
-                     .updateFp = (SdbUpdateFp)mndVgroupActionDelete,
-                     .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate};
+                     .updateFp = (SdbUpdateFp)mndVgroupActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndVgroupActionDelete};
 
   mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -114,18 +114,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen)
   SMCreateStbReq createReq = {0};
   createReq.numOfColumns = 3;
   createReq.numOfTags = 1;
-  createReq.numOfSmas = 1;
   createReq.igExists = 0;
   createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
   createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
-  createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField));
   strcpy(createReq.name, stbname);
 
   PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts");
   PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1");
   PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2");
   PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1");
-  PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1");
 
   int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
   void*   pHead = rpcMallocCont(tlen);
@@ -190,7 +187,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3
 }
 
 TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
-  #if 0
+#if 0
   const char* dbname = "1.d1";
   const char* stbname = "1.d1.stb";
   const char* smaname = "1.d1.sma";
@@ -264,7 +261,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateBSmaStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
+    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
   }
 
@@ -280,7 +277,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildDropStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
-    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname);
+    test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 0);
   }
 

source/dnode/mnode/sdb/inc/sdbInt.h

@@ -19,10 +19,6 @@
 #include "os.h"
 
 #include "sdb.h"
-#include "thash.h"
-#include "tlockfree.h"
-#include "tlog.h"
-#include "tmsg.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,26 +49,6 @@ typedef struct SSdbRow {
   char       pObj[];
 } SSdbRow;
 
-typedef struct SSdb {
-  SMnode     *pMnode;
-  char       *currDir;
-  char       *syncDir;
-  char       *tmpDir;
-  int64_t     lastCommitVer;
-  int64_t     curVer;
-  int64_t     tableVer[SDB_MAX];
-  int64_t     maxId[SDB_MAX];
-  EKeyType    keyTypes[SDB_MAX];
-  SHashObj   *hashObjs[SDB_MAX];
-  SRWLatch    locks[SDB_MAX];
-  SdbInsertFp insertFps[SDB_MAX];
-  SdbUpdateFp updateFps[SDB_MAX];
-  SdbDeleteFp deleteFps[SDB_MAX];
-  SdbDeployFp deployFps[SDB_MAX];
-  SdbEncodeFp encodeFps[SDB_MAX];
-  SdbDecodeFp decodeFps[SDB_MAX];
-} SSdb;
-
 const char *sdbTableName(ESdbType type);
 void        sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
 

source/dnode/mnode/sdb/src/sdb.c

@@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) {
       SSdbRow *pRow = *ppRow;
       if (pRow == NULL) continue;
 
-      sdbFreeRow(pSdb, pRow);
+      sdbFreeRow(pSdb, pRow, true);
       ppRow = taosHashIterate(hash, ppRow);
     }
   }
@@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) {
   return 0;
 }
 
-int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) {
-  pSdb->curVer += val;
+int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); }
\ No newline at end of file
-  return pSdb->curVer;
-}
\ No newline at end of file

source/dnode/mnode/sdb/src/sdbFile.c

@@ -28,7 +28,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
     if (fp == NULL) continue;
 
     if ((*fp)(pSdb->pMnode) != 0) {
-      mError("failed to deploy sdb:%d since %s", i, terrstr());
+      mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr());
       return -1;
     }
   }

source/dnode/mnode/sdb/src/sdbHash.c

@@ -137,7 +137,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
   if (pOldRow != NULL) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
     return terrno;
   }
@@ -148,7 +148,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
 
   if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
     return terrno;
   }
@@ -164,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
       taosWLockLatch(pLock);
       taosHashRemove(hash, pRow->pObj, keySize);
       taosWUnLockLatch(pLock);
-      sdbFreeRow(pSdb, pRow);
+      sdbFreeRow(pSdb, pRow, false);
       terrno = code;
       return terrno;
     }
@@ -202,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
     code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
   }
 
-  sdbFreeRow(pSdb, pNewRow);
+  sdbFreeRow(pSdb, pNewRow, false);
 
   pSdb->tableVer[pOldRow->type]++;
   return code;
@@ -215,7 +215,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
   if (ppOldRow == NULL || *ppOldRow == NULL) {
     taosWUnLockLatch(pLock);
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, false);
     terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
     return terrno;
   }
@@ -228,7 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
   taosWUnLockLatch(pLock);
 
   pSdb->tableVer[pOldRow->type]++;
-  sdbFreeRow(pSdb, pRow);
+  sdbFreeRow(pSdb, pRow, false);
 
   sdbCheck(pSdb, pOldRow);
   // sdbRelease(pSdb, pOldRow->pObj);
@@ -322,7 +322,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) {
   int32_t ref = atomic_load_32(&pRow->refCount);
   sdbPrintOper(pSdb, pRow, "check");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, true);
   }
 
   taosRUnLockLatch(pLock);
@@ -340,7 +340,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
   int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
   sdbPrintOper(pSdb, pRow, "release");
   if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
-    sdbFreeRow(pSdb, pRow);
+    sdbFreeRow(pSdb, pRow, true);
   }
 
   taosRUnLockLatch(pLock);

source/dnode/mnode/sdb/src/sdbRaw.c

@@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
     return -1;
   }
 
+  if (pVal != NULL) {
     memcpy(pRaw->pData + dataPos, pVal, valLen);
+  }
   return 0;
 }
 

source/dnode/mnode/sdb/src/sdbRow.c

@@ -36,11 +36,11 @@ void *sdbGetRowObj(SSdbRow *pRow) {
   return pRow->pObj;
 }
 
-void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
+void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) {
   // remove attached object such as trans
   SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
   if (deleteFp != NULL) {
-    (*deleteFp)(pSdb, pRow->pObj);
+    (*deleteFp)(pSdb, pRow->pObj, callFunc);
   }
 
   sdbPrintOper(pSdb, pRow, "free");

source/dnode/vnode/CMakeLists.txt

@@ -19,7 +19,6 @@ target_sources(
     "src/meta/metaOpen.c"
     "src/meta/metaIdx.c"
     "src/meta/metaTable.c"
-    "src/meta/metaTDBImpl.c"
     "src/meta/metaQuery.c"
     "src/meta/metaCommit.c"
     "src/meta/metaEntry.c"
@@ -28,12 +27,10 @@ target_sources(
     "src/tsdb/tsdbTDBImpl.c"
     "src/tsdb/tsdbCommit.c"
     "src/tsdb/tsdbCommit2.c"
-    "src/tsdb/tsdbCompact.c"
     "src/tsdb/tsdbFile.c"
     "src/tsdb/tsdbFS.c"
-    "src/tsdb/tsdbMain.c"
+    "src/tsdb/tsdbOpen.c"
     "src/tsdb/tsdbMemTable.c"
-    "src/tsdb/tsdbOptions.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
     "src/tsdb/tsdbScan.c"

source/dnode/vnode/inc/vnode.h

@@ -75,8 +75,9 @@ typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;
 typedef struct SMetaEntry  SMetaEntry;
 
-void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags);
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
 void metaReaderClear(SMetaReader *pReader);
+int  metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
 int  metaReadNext(SMetaReader *pReader);
 
 #if 1  // refact APIs below (TODO)
@@ -112,7 +113,7 @@ bool         tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
 void         tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
 int32_t      tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
 SArray      *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
-void         tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond);
+void         tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond);
 void         tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
 int32_t      tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 int32_t      tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
@@ -134,10 +135,6 @@ int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *p
 // need to reposition
 
 // structs
-struct SMetaCfg {
-  uint64_t lruSize;
-};
-
 struct STsdbCfg {
   int8_t  precision;
   int8_t  update;
@@ -146,10 +143,9 @@ struct STsdbCfg {
   int32_t days;
   int32_t minRows;
   int32_t maxRows;
-  int32_t  keep2;
   int32_t keep0;
   int32_t keep1;
-  uint64_t lruCacheSize;
+  int32_t keep2;
   SArray *retentions;
 };
 
@@ -161,8 +157,6 @@ struct SVnodeCfg {
   int32_t  szCache;
   uint64_t szBuf;
   bool     isHeap;
-  uint32_t ttl;
-  uint32_t keep;
   int8_t   streamMode;
   bool     isWeak;
   STsdbCfg tsdbCfg;

source/dnode/vnode/src/inc/meta.h

@@ -16,13 +16,14 @@
 #ifndef _TD_VNODE_META_H_
 #define _TD_VNODE_META_H_
 
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 typedef struct SMetaIdx    SMetaIdx;
 typedef struct SMetaDB     SMetaDB;
-typedef struct SMCtbCursor SMCtbCursor;
 typedef struct SMSmaCursor SMSmaCursor;
 
 // metaDebug ==================
@@ -36,22 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor;
 // clang-format on
 
 // metaOpen ==================
-int metaOpen(SVnode* pVnode, SMeta** ppMeta);
-int metaClose(SMeta* pMeta);
 
 // metaEntry ==================
 int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME);
 int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME);
 
 // metaTable ==================
-int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
-int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
 
 // metaQuery ==================
 int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid);
-int metaGetTableEntryByUid(SMetaReader* pReader, tb_uid_t uid);
-int metaGetTableEntryByName(SMetaReader* pReader, const char* name);
 
 // metaIdx ==================
 int  metaOpenIdx(SMeta* pMeta);
@@ -60,8 +55,6 @@ int  metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
 int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 
 // metaCommit ==================
-int metaBegin(SMeta* pMeta);
-
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
 
 struct SMeta {
@@ -106,27 +99,12 @@ typedef struct {
 } STtlIdxKey;
 
 #if 1
-#define META_SUPER_TABLE  TD_SUPER_TABLE
-#define META_CHILD_TABLE  TD_CHILD_TABLE
-#define META_NORMAL_TABLE TD_NORMAL_TABLE
 
 // int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
 int          metaDropTable(SMeta* pMeta, tb_uid_t uid);
-int             metaCommit(SMeta* pMeta);
-int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
-int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
-SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
-STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
-void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
-STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
-SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
-int             metaGetTbNum(SMeta* pMeta);
 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
 void         metaCloseSmaCursor(SMSmaCursor* pSmaCur);
 int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
-SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
-void            metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
-tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
 
 #ifndef META_REFACT
 // SMetaDB

source/dnode/vnode/src/inc/tq.h

@@ -16,6 +16,8 @@
 #ifndef _TD_VNODE_TQ_H_
 #define _TD_VNODE_TQ_H_
 
+#include "vnodeInt.h"
+
 #include "executor.h"
 #include "os.h"
 #include "tcache.h"
@@ -237,17 +239,7 @@ int  tqInit();
 void tqCleanUp();
 
 // open in each vnode
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
-void tqClose(STQ*);
 // required by vnode
-int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
-int tqCommit(STQ*);
-
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
-int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
-int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
-int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
 
 int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**);
 int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**);

source/dnode/vnode/src/inc/tsdb.h

@@ -16,6 +16,8 @@
 #ifndef _TD_VNODE_TSDB_H_
 #define _TD_VNODE_TSDB_H_
 
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -43,14 +45,46 @@ int  tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKe
                            TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
 
 // tsdbCommit ================
-int tsdbBegin(STsdb *pTsdb);
 
-#if 1
+// tsdbFS ================
+typedef struct STsdbFS STsdbFS;
 
-typedef struct SSmaStat SSmaStat;
+// tsdbSma ================
 typedef struct SSmaEnv  SSmaEnv;
 typedef struct SSmaEnvs SSmaEnvs;
 
+// structs
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+struct SSmaEnvs {
+  int16_t  nTSma;
+  int16_t  nRSma;
+  SSmaEnv *pTSmaEnv;
+  SSmaEnv *pRSmaEnv;
+};
+
+struct STsdb {
+  char          *path;
+  SVnode        *pVnode;
+  bool           repoLocked;
+  TdThreadMutex  mutex;
+  STsdbCfg       config;
+  STsdbMemTable *mem;
+  STsdbMemTable *imem;
+  SRtn           rtn;
+  STsdbFS       *fs;
+  SSmaEnvs       smaEnvs;
+};
+
+#if 1  // ======================================
+
+typedef struct SSmaStat SSmaStat;
+
 struct STable {
   uint64_t  tid;
   uint64_t  uid;
@@ -60,16 +94,9 @@ struct STable {
 #define TABLE_TID(t) (t)->tid
 #define TABLE_UID(t) (t)->uid
 
-int     tsdbOpen(SVnode *pVnode, STsdb **ppTsdb);
-int     tsdbClose(STsdb *pTsdb);
-int     tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp);
 int     tsdbPrepareCommit(STsdb *pTsdb);
-int     tsdbCommit(STsdb *pTsdb);
 int32_t tsdbInitSma(STsdb *pTsdb);
-int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg);
 int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg);
-int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version);
-int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg);
 int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
 int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
 void    tsdbCleanupReadHandle(tsdbReaderT queryHandle);
@@ -103,13 +130,6 @@ typedef struct {
   uint8_t   state;
 } SDFile;
 
-struct SSmaEnvs {
-  int16_t  nTSma;
-  int16_t  nRSma;
-  SSmaEnv *pTSmaEnv;
-  SSmaEnv *pRSmaEnv;
-};
-
 typedef struct {
   int      fid;
   int8_t   state;  // -128~127
@@ -118,13 +138,6 @@ typedef struct {
   SDFile   files[TSDB_FILE_MAX];
 } SDFileSet;
 
-typedef struct {
-  int   minFid;
-  int   midFid;
-  int   maxFid;
-  TSKEY minKey;
-} SRtn;
-
 struct STbData {
   tb_uid_t   uid;
   TSKEY      keyMin;
@@ -161,7 +174,7 @@ typedef struct {
   SArray     *sf;    // sma data file array    v2f1900.index_name_1
 } SFSStatus;
 
-typedef struct {
+struct STsdbFS {
   TdThreadRwlock lock;
 
   SFSStatus *cstatus;        // current status
@@ -169,23 +182,9 @@ typedef struct {
   SHashObj  *metaCacheComp;  // meta cache for compact
   bool       intxn;
   SFSStatus *nstatus;  // new status
-} STsdbFS;
-
-struct STsdb {
-  char          *path;
-  SVnode        *pVnode;
-  int32_t        vgId;
-  bool           repoLocked;
-  TdThreadMutex  mutex;
-  STsdbCfg       config;
-  STsdbMemTable *mem;
-  STsdbMemTable *imem;
-  SRtn           rtn;
-  STsdbFS       *fs;
-  SSmaEnvs       smaEnvs;
 };
 
-#define REPO_ID(r)        ((r)->vgId)
+#define REPO_ID(r)        TD_VID((r)->pVnode)
 #define REPO_CFG(r)       (&(r)->config)
 #define REPO_FS(r)        ((r)->fs)
 #define REPO_META(r)      ((r)->pVnode->pMeta)
@@ -237,12 +236,6 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
   return TD_ROW_KEY(row);
 }
 
-// tsdbOptions
-extern const STsdbCfg defautlTsdbOptions;
-
-int  tsdbValidateOptions(const STsdbCfg *);
-void tsdbOptionsCopy(STsdbCfg *pDest, const STsdbCfg *pSrc);
-
 // tsdbReadImpl
 typedef struct SReadH SReadH;
 
@@ -519,12 +512,6 @@ static FORCE_INLINE void *taosTZfree(void *ptr) {
 
 // tsdbCommit
 
-typedef struct {
-  uint64_t uid;
-  int64_t  offset;
-  int64_t  size;
-} SKVRecord;
-
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
 
 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
@@ -903,66 +890,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) {
   return 0;
 }
 
-// tsdbSma
-// #define TSDB_SMA_TEST // remove after test finished
-
-// struct SSmaEnv {
-//   TdThreadRwlock lock;
-//   SDiskID        did;
-//   TDBEnv         dbEnv;  // TODO: If it's better to put it in smaIndex level?
-//   char          *path;   // relative path
-//   SSmaStat      *pStat;
-// };
-
-// #define SMA_ENV_LOCK(env)       ((env)->lock)
-// #define SMA_ENV_DID(env)        ((env)->did)
-// #define SMA_ENV_ENV(env)        ((env)->dbEnv)
-// #define SMA_ENV_PATH(env)       ((env)->path)
-// #define SMA_ENV_STAT(env)       ((env)->pStat)
-// #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
-
-// void  tsdbDestroySmaEnv(SSmaEnv *pSmaEnv);
-// void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv);
-// #if 0
-// int32_t tsdbGetTSmaStatus(STsdb *pTsdb, STSma *param, void *result);
-// int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin);
-// #endif
-
-// // internal func
-// static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
-//   int32_t len = 0;
-//   len += taosEncodeFixedI64(pData, tsKey);
-//   len += taosEncodeFixedI64(pData, groupId);
-//   return len;
-// }
-
-// static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockRdlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockWrlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
-//   int code = taosThreadRwlockUnlock(&(pEnv->lock));
-//   if (code != 0) {
-//     terrno = TAOS_SYSTEM_ERROR(code);
-//     return -1;
-//   }
-//   return 0;
-// }
-
 typedef struct SSmaKey SSmaKey;
 
 struct SSmaKey {

source/dnode/vnode/src/inc/tsdbSma.h

@@ -16,9 +16,7 @@
 #ifndef _TD_VNODE_TSDB_SMA_H_
 #define _TD_VNODE_TSDB_SMA_H_
 
-#include "os.h"
-#include "thash.h"
-#include "tmsg.h"
+#include "tsdb.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -32,12 +30,6 @@ struct STbDdlH {
   __tb_ddl_fn_t fp;
 };
 
-typedef struct {
-  tb_uid_t  suid;
-  SArray   *tbUids;
-  SHashObj *uidHash;
-} STbUidStore;
-
 static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
   ASSERT(*pStore == NULL);
   *pStore = taosMemoryCalloc(1, sizeof(STbUidStore));

source/dnode/vnode/src/inc/vnd.h

@@ -16,6 +16,10 @@
 #ifndef _TD_VND_H_
 #define _TD_VND_H_
 
+#include "sync.h"
+#include "syncTools.h"
+#include "vnodeInt.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -61,8 +65,6 @@ struct SVBufPool {
 int  vnodeOpenBufPool(SVnode* pVnode, int64_t size);
 int  vnodeCloseBufPool(SVnode* pVnode);
 void vnodeBufPoolReset(SVBufPool* pPool);
-void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
-void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
 
 // vnodeQuery ====================
 int  vnodeQueryOpen(SVnode* pVnode);
@@ -79,6 +81,20 @@ int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
 int vnodeSyncCommit(SVnode* pVnode);
 int vnodeAsyncCommit(SVnode* pVnode);
 
+// vnodeCommit ====================
+int32_t   vnodeSyncOpen(SVnode* pVnode, char* path);
+int32_t   vnodeSyncStart(SVnode* pVnode);
+void      vnodeSyncClose(SVnode* pVnode);
+void      vnodeSyncSetQ(SVnode* pVnode, void* qHandle);
+void      vnodeSyncSetRpc(SVnode* pVnode, void* rpcHandle);
+int32_t   vnodeSyncEqMsg(void* qHandle, SRpcMsg* pMsg);
+int32_t   vnodeSendMsg(void* rpcHandle, const SEpSet* pEpSet, SRpcMsg* pMsg);
+void      vnodeSyncCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+void      vnodeSyncPreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+void      vnodeSyncRollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+int32_t   vnodeSyncGetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+SSyncFSM* syncVnodeMakeFsm();
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -59,6 +59,54 @@ typedef struct SQWorkerMgmt SQHandle;
 #define VNODE_TQ_DIR   "tq"
 #define VNODE_WAL_DIR  "wal"
 
+// vnd.h
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
+
+// meta
+typedef struct SMCtbCursor SMCtbCursor;
+typedef struct STbUidStore STbUidStore;
+
+int             metaOpen(SVnode* pVnode, SMeta** ppMeta);
+int             metaClose(SMeta* pMeta);
+int             metaBegin(SMeta* pMeta);
+int             metaCommit(SMeta* pMeta);
+int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
+int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
+SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
+STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
+int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
+int             metaGetTbNum(SMeta* pMeta);
+SMCtbCursor*    metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
+void            metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
+tb_uid_t        metaCtbCursorNext(SMCtbCursor* pCtbCur);
+SArray*         metaGetSmaTbUids(SMeta* pMeta, bool isDup);
+void*           metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
+STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
+int32_t         metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
+int32_t         metaDropTSma(SMeta* pMeta, int64_t indexUid);
+
+// tsdb
+int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb);
+int     tsdbClose(STsdb* pTsdb);
+int     tsdbBegin(STsdb* pTsdb);
+int     tsdbCommit(STsdb* pTsdb);
+int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
+int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
+int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
+int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
+
+// tq
+STQ*    tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
+void    tqClose(STQ*);
+int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
+int     tqCommit(STQ*);
+int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
+int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
+int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
+
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
@@ -106,6 +154,12 @@ struct SVnode {
   SQHandle*  pQuery;
 };
 
+struct STbUidStore {
+  tb_uid_t  suid;
+  SArray*   tbUids;
+  SHashObj* uidHash;
+};
+
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
 typedef struct STbDdlH STbDdlH;
@@ -113,18 +167,6 @@ typedef struct STbDdlH STbDdlH;
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
-#include "vnd.h"
-
-#include "meta.h"
-
-#include "tsdb.h"
-
-#include "tq.h"
-
-#include "vnodeSync.h"
-
-#include "tsdbSma.h"
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeSync.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_SYNC_H_
-#define _TD_VNODE_SYNC_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
-int32_t vnodeSyncStart(SVnode *pVnode);
-void    vnodeSyncClose(SVnode *pVnode);
-
-void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
-void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
-
-int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
-int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
-
-void    vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-void    vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-void    vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
-
-SSyncFSM *syncVnodeMakeFsm();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_SYNC_H_*/

source/dnode/vnode/src/meta/metaCommit.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
 static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBufPoolMalloc((SVBufPool *)pPool, size); }
 static FORCE_INLINE void  metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }

source/dnode/vnode/src/meta/metaEntry.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
 int metaEncodeEntry(SCoder *pCoder, const SMetaEntry *pME) {
   if (tStartEncode(pCoder) < 0) return -1;

source/dnode/vnode/src/meta/metaIdx.c

@@ -16,7 +16,7 @@
 #ifdef USE_INVERTED_INDEX
 #include "index.h"
 #endif
-#include "vnodeInt.h"
+#include "meta.h"
 
 struct SMetaIdx {
 #ifdef USE_INVERTED_INDEX

source/dnode/vnode/src/meta/metaOpen.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
 static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);

source/dnode/vnode/src/meta/metaQuery.c

@@ -13,12 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
-void metaReaderInit(SMetaReader *pReader, SVnode *pVnode, int32_t flags) {
+void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) {
   memset(pReader, 0, sizeof(*pReader));
   pReader->flags = flags;
-  pReader->pMeta = pVnode->pMeta;
+  pReader->pMeta = pMeta;
 }
 
 void metaReaderClear(SMetaReader *pReader) {
@@ -32,6 +32,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u
 
   // query table.db
   if (tdbDbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pReader->pBuf, &pReader->szBuf) < 0) {
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     goto _err;
   }
 
@@ -54,6 +55,7 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) {
 
   // query uid.idx
   if (tdbDbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) {
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 
@@ -67,6 +69,7 @@ int metaGetTableEntryByName(SMetaReader *pReader, const char *name) {
 
   // query name.idx
   if (tdbDbGet(pMeta->pNameIdx, name, strlen(name) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
     return -1;
   }
 
@@ -91,7 +94,7 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) {
     return NULL;
   }
 
-  metaReaderInit(&pTbCur->mr, pMeta->pVnode, 0);
+  metaReaderInit(&pTbCur->mr, pMeta, 0);
 
   tdbDbcOpen(pMeta->pUidIdx, &pTbCur->pDbc);
 
@@ -122,7 +125,7 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
     }
 
     metaGetTableEntryByVersion(&pTbCur->mr, *(int64_t *)pTbCur->pVal, *(tb_uid_t *)pTbCur->pKey);
-    if (pTbCur->mr.me.type == META_SUPER_TABLE) {
+    if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
       continue;
     }
 
@@ -234,7 +237,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
   STSchemaBuilder sb = {0};
   SSchema        *pSchema;
 
-  metaReaderInit(&mr, pMeta->pVnode, 0);
+  metaReaderInit(&mr, pMeta, 0);
   metaGetTableEntryByUid(&mr, uid);
 
   if (mr.me.type == TSDB_CHILD_TABLE) {

source/dnode/vnode/src/meta/metaTable.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "meta.h"
 
 static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
@@ -37,7 +37,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaReader mr = {0};
 
   // validate req
-  metaReaderInit(&mr, pMeta->pVnode, 0);
+  metaReaderInit(&mr, pMeta, 0);
   if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
 // TODO: just for pass case
 #if 0
@@ -91,7 +91,7 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) {
   pReq->ctime = taosGetTimestampMs();
 
   // validate req
-  metaReaderInit(&mr, pMeta->pVnode, 0);
+  metaReaderInit(&mr, pMeta, 0);
   if (metaGetTableEntryByName(&mr, pReq->name) == 0) {
     terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST;
     metaReaderClear(&mr);

source/dnode/vnode/src/tq/tq.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 int32_t tqInit() {
   //
@@ -78,7 +78,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
 
     SMqDataBlkRsp rsp = {0};
     rsp.reqOffset = pExec->pushHandle.reqOffset;
-    rsp.blockData = taosArrayInit(0, sizeof(int32_t));
+    rsp.blockData = taosArrayInit(0, sizeof(void*));
     rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
 
     if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
@@ -176,7 +176,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
     atomic_store_ptr(&pExec->pushHandle.handle, NULL);
     taosWUnLockLatch(&pExec->pushHandle.lock);
 
-    vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
             TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
             rsp.reqOffset, rsp.rspOffset);
 
@@ -210,35 +210,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
 
   tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
 
-#if 0
-  void* pIter = taosHashIterate(pTq->tqPushMgr->pHash, NULL);
-  while (pIter != NULL) {
-    STqPusher* pusher = *(STqPusher**)pIter;
-    if (pusher->type == TQ_PUSHER_TYPE__STREAM) {
-      STqStreamPusher* streamPusher = (STqStreamPusher*)pusher;
-      // repack
-      STqStreamToken* token = taosMemoryMalloc(sizeof(STqStreamToken));
-      if (token == NULL) {
-        taosHashCancelIterate(pTq->tqPushMgr->pHash, pIter);
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        return -1;
-      }
-      token->type = TQ_STREAM_TOKEN__DATA;
-      token->data = msg;
-      // set input
-      // exec
-    }
-    // send msg to ep
-  }
-  // iterate hash
-  // process all msg
-  // if waiting
-  // memcpy and send msg to fetch thread
-  // TODO: add reference
-  // if handle waiting, launch query and response to consumer
-  //
-  // if no waiting handle, return
-#endif
   return 0;
 }
 
@@ -375,9 +346,9 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 }
 
 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
-  SMqPollReqV2* pReq = pMsg->pCont;
+  SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
-  int64_t       waitTime = pReq->blockingTime;
+  int64_t     waitTime = pReq->waitTime;
   int32_t     reqEpoch = pReq->epoch;
   int64_t     fetchOffset;
 
@@ -390,7 +361,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
           TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey));
@@ -418,7 +389,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
     if (consumerEpoch > reqEpoch) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
               consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
@@ -426,7 +397,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     taosThreadMutexLock(&pExec->pWalReader->mutex);
 
     if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
               TD_VID(pTq->pVnode), fetchOffset);
       taosThreadMutexUnlock(&pExec->pWalReader->mutex);
       break;
@@ -448,7 +419,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
 
 #if 0
@@ -476,7 +447,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 #endif
 
-    vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
             TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
 
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -591,7 +562,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
 
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+  tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
           TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset);
 
   // TODO destroy
@@ -618,7 +589,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
-  vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
+  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
          TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   SMqPollRspV2 rspV2 = {0};
@@ -660,7 +631,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     return 0;
   }
 
-  vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
+  tqDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
          TD_VID(pTq->pVnode));
 
   rspV2.reqOffset = pReq->currentOffset;
@@ -671,7 +642,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     // TODO
     consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
              consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
@@ -680,11 +651,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
-      vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
+      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
              TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
-    vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
+    tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
            TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
@@ -709,7 +680,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       }
 
       if (taosArrayGetSize(pRes) == 0) {
-        vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
+        tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
                pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
         fetchOffset++;
         rspV2.skipLogNum++;
@@ -770,7 +741,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = msgLen;
       pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
+      tqDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
              pHead->msgType, consumerId, pReq->epoch);
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
@@ -804,7 +775,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
+  tqDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
          pReq->epoch);
   /*}*/
 

source/dnode/vnode/src/tq/tqCommit.c

@@ -13,4 +13,4 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -12,7 +12,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "vnodeInt.h"
+#include "tq.h"
 // #include <fcntl.h>
 // #include <string.h>
 // #include <unistd.h>
@@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
   }
   strcpy(pMeta->dirPath, path);
 
-  char *name = taosMemoryMalloc(pathLen + 10) ;
+  char* name = taosMemoryMalloc(pathLen + 10);
 
   strcpy(name, path);
   if (!taosDirExist(name) && taosMkDir(name) != 0) {

source/dnode/vnode/src/tq/tqOffset.c

@@ -14,7 +14,7 @@
  */
 #define _DEFAULT_SOURCE
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 enum ETqOffsetPersist {
   TQ_OFFSET_PERSIST__LAZY = 1,

source/dnode/vnode/src/tq/tqRead.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tq.h"
 
 STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
   STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle));

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 #define TSDB_MAX_SUBBLOCKS 8
 
@@ -752,334 +752,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) {
   return 0;
 }
 
-// // =================== Commit Meta Data
-// static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) {
-//   STsdbFS *  pfs = REPO_FS(pRepo);
-//   SMFile *   pOMFile = pfs->cstatus->pmf;
-//   SDiskID    did;
-
-//   // Create/Open a meta file or open the existing file
-//   if (pOMFile == NULL) {
-//     // Create a new meta file
-//     did.level = TFS_PRIMARY_LEVEL;
-//     did.id = TFS_PRIMARY_ID;
-//     tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
-
-//     if (open && tsdbCreateMFile(pMf, true) < 0) {
-//       tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-
-//     tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf));
-//   } else {
-//     tsdbInitMFileEx(pMf, pOMFile);
-//     if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) {
-//       tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-//   }
-
-//   return 0;
-// }
-
-// static int tsdbCommitMeta(STsdbRepo *pRepo) {
-//   STsdbFS *  pfs = REPO_FS(pRepo);
-//   SMemTable *pMem = pRepo->imem;
-//   SMFile *   pOMFile = pfs->cstatus->pmf;
-//   SMFile     mf;
-//   SActObj *  pAct = NULL;
-//   SActCont * pCont = NULL;
-//   SListNode *pNode = NULL;
-
-//   ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0);
-
-//   if (listNEles(pMem->actList) <= 0) {
-//     // no meta data to commit, just keep the old meta file
-//     tsdbUpdateMFile(pfs, pOMFile);
-//     if (tsTsdbMetaCompactRatio > 0) {
-//       if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) {
-//         return -1;
-//       }
-//       int ret = tsdbCompactMetaFile(pRepo, pfs, &mf);
-//       if (ret < 0) tsdbError("compact meta file error");
-
-//       return ret;
-//     }
-//     return 0;
-//   } else {
-//     if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) {
-//       return -1;
-//     }
-//   }
-
-//   // Loop to write
-//   while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
-//     pAct = (SActObj *)pNode->data;
-//     if (pAct->act == TSDB_UPDATE_META) {
-//       pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
-//       if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) {
-//         tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(&mf);
-//         (void)tsdbApplyMFileChange(&mf, pOMFile);
-//         // TODO: need to reload metaCache
-//         return -1;
-//       }
-//     } else if (pAct->act == TSDB_DROP_META) {
-//       if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
-//         tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(&mf);
-//         tsdbApplyMFileChange(&mf, pOMFile);
-//         // TODO: need to reload metaCache
-//         return -1;
-//       }
-//     } else {
-//       ASSERT(false);
-//     }
-//   }
-
-//   if (tsdbUpdateMFileHeader(&mf) < 0) {
-//     tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
-//     tsdbApplyMFileChange(&mf, pOMFile);
-//     // TODO: need to reload metaCache
-//     return -1;
-//   }
-
-//   TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbUpdateMFile(pfs, &mf);
-
-//   if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) {
-//     tsdbError("compact meta file error");
-//   }
-
-//   return 0;
-// }
-
-// int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) {
-//   int tlen = 0;
-//   tlen += taosEncodeFixedU64(buf, pRecord->uid);
-//   tlen += taosEncodeFixedI64(buf, pRecord->offset);
-//   tlen += taosEncodeFixedI64(buf, pRecord->size);
-
-//   return tlen;
-// }
-
-// void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) {
-//   buf = taosDecodeFixedU64(buf, &(pRecord->uid));
-//   buf = taosDecodeFixedI64(buf, &(pRecord->offset));
-//   buf = taosDecodeFixedI64(buf, &(pRecord->size));
-
-//   return buf;
-// }
-
-// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) {
-//   char      buf[64] = "\0";
-//   void *    pBuf = buf;
-//   SKVRecord rInfo;
-//   int64_t   offset;
-
-//   // Seek to end of meta file
-//   offset = tsdbSeekMFile(pMFile, 0, SEEK_END);
-//   if (offset < 0) {
-//     return -1;
-//   }
-
-//   rInfo.offset = offset;
-//   rInfo.uid = uid;
-//   rInfo.size = contLen;
-
-//   int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo);
-//   if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) {
-//     return -1;
-//   }
-
-//   if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) {
-//     return -1;
-//   }
-
-//   tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
-
-//   SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache;
-
-//   pMFile->info.nRecords++;
-
-//   SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid));
-//   if (pRecord != NULL) {
-//     pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord));
-//   } else {
-//     pMFile->info.nRecords++;
-//   }
-//   taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
-
-//   return 0;
-// }
-
-// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
-//   SKVRecord rInfo = {0};
-//   char      buf[128] = "\0";
-
-//   SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
-//   if (pRecord == NULL) {
-//     tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid);
-//     return -1;
-//   }
-
-//   rInfo.offset = -pRecord->offset;
-//   rInfo.uid = pRecord->uid;
-//   rInfo.size = pRecord->size;
-
-//   void *pBuf = buf;
-//   tsdbEncodeKVRecord(&pBuf, &rInfo);
-
-//   if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) {
-//     return -1;
-//   }
-
-//   pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord));
-//   pMFile->info.nDels++;
-//   pMFile->info.nRecords--;
-//   pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-
-//   taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid));
-//   return 0;
-// }
-
-// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) {
-//   float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords);
-//   float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size);
-//   float compactRatio = (float)(tsTsdbMetaCompactRatio)/100;
-
-//   if (delPercent < compactRatio && tombPercent < compactRatio) {
-//     return 0;
-//   }
-
-//   if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
-//     tsdbError("open meta file %s compact fail", pMFile->f.rname);
-//     return -1;
-//   }
-
-//   tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64
-//   ",size:%" PRId64,
-//     tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size);
-
-//   SMFile mf;
-//   SDiskID did;
-
-//   // first create tmp meta file
-//   did.level = TFS_PRIMARY_LEVEL;
-//   did.id = TFS_PRIMARY_ID;
-//   tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1);
-
-//   if (tsdbCreateMFile(&mf, true) < 0) {
-//     tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-
-//   tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
-
-//   // second iterator metaCache
-//   int code = -1;
-//   int64_t maxBufSize = 1024;
-//   SKVRecord *pRecord;
-//   void *pBuf = NULL;
-
-//   pBuf = taosMemoryMalloc((size_t)maxBufSize);
-//   if (pBuf == NULL) {
-//     goto _err;
-//   }
-
-//   // init Comp
-//   assert(pfs->metaCacheComp == NULL);
-//   pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-//   if (pfs->metaCacheComp == NULL) {
-//     goto _err;
-//   }
-
-//   pRecord = taosHashIterate(pfs->metaCache, NULL);
-//   while (pRecord) {
-//     if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
-//       tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                 tstrerror(terrno));
-//       goto _err;
-//     }
-//     if (pRecord->size > maxBufSize) {
-//       maxBufSize = pRecord->size;
-//       void* tmp = taosMemoryRealloc(pBuf, (size_t)maxBufSize);
-//       if (tmp == NULL) {
-//         goto _err;
-//       }
-//       pBuf = tmp;
-//     }
-//     int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
-//     if (nread < 0) {
-//       tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//         tstrerror(terrno));
-//       goto _err;
-//     }
-
-//     if (nread < pRecord->size) {
-//       tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
-//                 REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
-//       goto _err;
-//     }
-
-//     if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) {
-//       tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid,
-//                 tstrerror(terrno));
-//       goto _err;
-//     }
-
-//     pRecord = taosHashIterate(pfs->metaCache, pRecord);
-//   }
-//   code = 0;
-
-// _err:
-//   if (code == 0) TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbCloseMFile(pMFile);
-
-//   if (code == 0) {
-//     // rename meta.tmp -> meta
-//     tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf),
-//     TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname,
-//     TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN);
-//     // update current meta file info
-//     pfs->nstatus->pmf = NULL;
-//     tsdbUpdateMFile(pfs, &mf);
-
-//     taosHashCleanup(pfs->metaCache);
-//     pfs->metaCache = pfs->metaCacheComp;
-//     pfs->metaCacheComp = NULL;
-//   } else {
-//     // remove meta.tmp file
-//     taosRemoveFile(mf.f.aname);
-//     taosHashCleanup(pfs->metaCacheComp);
-//     pfs->metaCacheComp = NULL;
-//   }
-
-//   taosMemoryFreeClear(pBuf);
-
-//   ASSERT(mf.info.nDels == 0);
-//   ASSERT(mf.info.tombSize == 0);
-
-//   tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64,
-//     code,mf.info.nRecords,mf.info.size);
-//   return code;
-// }
-
-// // =================== Commit Time-Series Data
-// #if 0
-// static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
-//   for (int i = 0; i < nIters; i++) {
-//     TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
-//     if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true;
-//   }
-//   return false;
-// }
-// #endif
-
+// =================== Commit Time-Series Data
 static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
   SCommitIter *pIter = pCommith->iters + tid;
   TSKEY        nextKey = tsdbNextIterKey(pIter->pIter);

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 int tsdbBegin(STsdb *pTsdb) {
   STsdbMemTable *pMem;

source/dnode/vnode/src/tsdb/tsdbCompact.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#if 0
-#include "tsdbint.h"
-
-typedef struct {
-  STable *    pTable;
-  SBlockIdx * pBlkIdx;
-  SBlockIdx   bindex;
-  SBlockInfo *pInfo;
-} STableCompactH;
-
-typedef struct {
-  SRtn       rtn;
-  SFSIter    fsIter;
-  SArray *   tbArray;  // table array to cache table obj and block indexes
-  SReadH     readh;
-  SDFileSet  wSet;
-  SArray *   aBlkIdx;
-  SArray *   aSupBlk;
-  SDataCols *pDataCols;
-} SCompactH;
-
-#define TSDB_COMPACT_WSET(pComph) (&((pComph)->wSet))
-#define TSDB_COMPACT_REPO(pComph) TSDB_READ_REPO(&((pComph)->readh))
-#define TSDB_COMPACT_HEAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_HEAD)
-#define TSDB_COMPACT_DATA_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_DATA)
-#define TSDB_COMPACT_LAST_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_LAST)
-#define TSDB_COMPACT_SMAD_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAD)
-#define TSDB_COMPACT_SMAL_FILE(pComph) TSDB_DFILE_IN_SET(TSDB_COMPACT_WSET(pComph), TSDB_FILE_SMAL)
-#define TSDB_COMPACT_BUF(pComph) TSDB_READ_BUF(&((pComph)->readh))
-#define TSDB_COMPACT_COMP_BUF(pComph) TSDB_READ_COMP_BUF(&((pComph)->readh))
-
-static int  tsdbAsyncCompact(STsdbRepo *pRepo);
-static void tsdbStartCompact(STsdbRepo *pRepo);
-static void tsdbEndCompact(STsdbRepo *pRepo, int eno);
-static int  tsdbCompactMeta(STsdbRepo *pRepo);
-static int  tsdbCompactTSData(STsdbRepo *pRepo);
-static int  tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet);
-static bool tsdbShouldCompact(SCompactH *pComph);
-static int  tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo);
-static void tsdbDestroyCompactH(SCompactH *pComph);
-static int  tsdbInitCompTbArray(SCompactH *pComph);
-static void tsdbDestroyCompTbArray(SCompactH *pComph);
-static int  tsdbCacheFSetIndex(SCompactH *pComph);
-static int  tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet);
-static void tsdbCompactFSetEnd(SCompactH *pComph);
-static int  tsdbCompactFSetImpl(SCompactH *pComph);
-static int  tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
-                                      void **ppCBuf);
-
-enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
-int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); }
-
-void *tsdbCompactImpl(STsdbRepo *pRepo) {
-  // Check if there are files in TSDB FS to compact
-  if (REPO_FS(pRepo)->cstatus->pmf == NULL) {
-    tsdbInfo("vgId:%d no file to compact in FS", REPO_ID(pRepo));
-    return NULL;
-  }
-
-  tsdbStartCompact(pRepo);
-
-  if (tsdbCompactMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to compact META data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
-
-  if (tsdbCompactTSData(pRepo) < 0) {
-    tsdbError("vgId:%d failed to compact TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
-
-  tsdbEndCompact(pRepo, TSDB_CODE_SUCCESS);
-  return NULL;
-
-_err:
-  pRepo->code = terrno;
-  tsdbEndCompact(pRepo, terrno);
-  return NULL;
-}
-
-static int tsdbAsyncCompact(STsdbRepo *pRepo) {
-  if (pRepo->compactState != TSDB_NO_COMPACT) {
-    tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
-    return 0; 
-  } 
-  pRepo->compactState = TSDB_WAITING_COMPACT;   
-  tsem_wait(&(pRepo->readyToCommit));
-  return tsdbScheduleCommit(pRepo, COMPACT_REQ);
-}
-
-static void tsdbStartCompact(STsdbRepo *pRepo) {
-  assert(pRepo->compactState != TSDB_IN_COMPACT);
-  tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
-  tsdbStartFSTxn(pRepo, 0, 0);
-  pRepo->code = TSDB_CODE_SUCCESS;
-  pRepo->compactState = TSDB_IN_COMPACT;
-}
-
-static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
-  if (eno != TSDB_CODE_SUCCESS) {
-    tsdbEndFSTxnWithError(REPO_FS(pRepo));
-  } else {
-    tsdbEndFSTxn(pRepo);
-  }
-  pRepo->compactState = TSDB_NO_COMPACT;
-  tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
-  tsem_post(&(pRepo->readyToCommit));
-}
-
-static int tsdbCompactMeta(STsdbRepo *pRepo) {
-  STsdbFS *pfs = REPO_FS(pRepo);
-  tsdbUpdateMFile(pfs, pfs->cstatus->pmf);
-  return 0;
-}
-
-  static int tsdbCompactTSData(STsdbRepo *pRepo) {
-    SCompactH  compactH;
-    SDFileSet *pSet = NULL;
-
-    tsdbDebug("vgId:%d start to compact TS data", REPO_ID(pRepo));
-
-    // If no file, just return 0;
-    if (taosArrayGetSize(REPO_FS(pRepo)->cstatus->df) <= 0) {
-      tsdbDebug("vgId:%d no TS data file to compact, compact over", REPO_ID(pRepo));
-      return 0;
-    }
-
-    if (tsdbInitCompactH(&compactH, pRepo) < 0) {
-      return -1;
-    }
-
-    while ((pSet = tsdbFSIterNext(&(compactH.fsIter)))) {
-      // Remove those expired files
-      if (pSet->fid < compactH.rtn.minFid) {
-        tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid,
-                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
-        continue;
-      }
-
-      if (TSDB_FSET_LEVEL(pSet) == TFS_MAX_LEVEL) {
-        tsdbDebug("vgId:%d FSET %d on level %d, should not compact", REPO_ID(pRepo), pSet->fid, TFS_MAX_LEVEL);
-        tsdbUpdateDFileSet(REPO_FS(pRepo), pSet);
-        continue;
-      }
-
-      if (tsdbCompactFSet(&compactH, pSet) < 0) {
-        tsdbDestroyCompactH(&compactH);
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        return -1;
-      }
-    }
-
-    tsdbDestroyCompactH(&compactH);
-    tsdbDebug("vgId:%d compact TS data over", REPO_ID(pRepo));
-    return 0;
-  }
-
-  static int tsdbCompactFSet(SCompactH *pComph, SDFileSet *pSet) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    SDiskID    did;
-
-    tsdbDebug("vgId:%d start to compact FSET %d on level %d id %d", REPO_ID(pRepo), pSet->fid, TSDB_FSET_LEVEL(pSet),
-              TSDB_FSET_ID(pSet));
-
-    if (tsdbCompactFSetInit(pComph, pSet) < 0) {
-      return -1;
-    }
-
-    if (!tsdbShouldCompact(pComph)) {
-      tsdbDebug("vgId:%d no need to compact FSET %d", REPO_ID(pRepo), pSet->fid);
-      if (tsdbApplyRtnOnFSet(TSDB_COMPACT_REPO(pComph), pSet, &(pComph->rtn)) < 0) {
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-    } else {
-      // Create new fset as compacted fset
-      if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(pSet->fid, &(pComph->rtn)), &did) < 0) {
-        terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      tsdbInitDFileSet(TSDB_COMPACT_WSET(pComph), did, REPO_ID(pRepo), TSDB_FSET_FID(pSet),
-                      FS_TXN_VERSION(REPO_FS(pRepo)));
-      if (tsdbCreateDFileSet(TSDB_COMPACT_WSET(pComph), true) < 0) {
-        tsdbError("vgId:%d failed to compact FSET %d since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      if (tsdbCompactFSetImpl(pComph) < 0) {
-        tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-        tsdbRemoveDFileSet(TSDB_COMPACT_WSET(pComph));
-        tsdbCompactFSetEnd(pComph);
-        return -1;
-      }
-
-      tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-      tsdbUpdateDFileSet(REPO_FS(pRepo), TSDB_COMPACT_WSET(pComph));
-      tsdbDebug("vgId:%d FSET %d compact over", REPO_ID(pRepo), pSet->fid);
-    }
-
-    tsdbCompactFSetEnd(pComph);
-    return 0;
-  }
-
-  static bool tsdbShouldCompact(SCompactH *pComph) {
-    STsdbRepo *     pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg *      pCfg = REPO_CFG(pRepo);
-    SReadH *        pReadh = &(pComph->readh);
-    STableCompactH *pTh;
-    SBlock *        pBlock;
-    int             defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
-    SDFile *        pDataF = TSDB_READ_DATA_FILE(pReadh);
-    SDFile *        pLastF = TSDB_READ_LAST_FILE(pReadh);
-
-    int     tblocks = 0;       // total blocks
-    int     nSubBlocks = 0;    // # of blocks with sub-blocks
-    int     nSmallBlocks = 0;  // # of blocks with rows < defaultRows
-    int64_t tsize = 0;
-
-    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
-      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
-
-      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
-
-      for (size_t bidx = 0; bidx < pTh->pBlkIdx->numOfBlocks; bidx++) {
-        tblocks++;
-        pBlock = pTh->pInfo->blocks + bidx;
-
-        if (pBlock->numOfRows < defaultRows) {
-          nSmallBlocks++;
-        }
-
-        if (pBlock->numOfSubBlocks > 1) {
-          nSubBlocks++;
-          for (int k = 0; k < pBlock->numOfSubBlocks; k++) {
-            SBlock *iBlock = ((SBlock *)POINTER_SHIFT(pTh->pInfo, pBlock->offset)) + k;
-            tsize = tsize + iBlock->len;
-          }
-        } else if (pBlock->numOfSubBlocks == 1) {
-          tsize += pBlock->len;
-        } else {
-          ASSERT(0);
-        }
-      }
-    }
-
-    return (((nSubBlocks * 1.0 / tblocks) > 0.33) || ((nSmallBlocks * 1.0 / tblocks) > 0.33) ||
-            (tsize * 1.0 / (pDataF->info.size + pLastF->info.size - 2 * TSDB_FILE_HEAD_SIZE) < 0.85));
-  }
-
-  static int tsdbInitCompactH(SCompactH *pComph, STsdbRepo *pRepo) {
-    STsdbCfg *pCfg = REPO_CFG(pRepo);
-
-    memset(pComph, 0, sizeof(*pComph));
-
-    TSDB_FSET_SET_CLOSED(TSDB_COMPACT_WSET(pComph));
-
-    tsdbGetRtnSnap(pRepo, &(pComph->rtn));
-    tsdbFSIterInit(&(pComph->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
-
-    if (tsdbInitReadH(&(pComph->readh), pRepo) < 0) {
-      return -1;
-    }
-
-    if (tsdbInitCompTbArray(pComph) < 0) {
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
-    if (pComph->aBlkIdx == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
-    if (pComph->aSupBlk == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
-    if (pComph->pDataCols == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbDestroyCompactH(pComph);
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static void tsdbDestroyCompactH(SCompactH *pComph) {
-    pComph->pDataCols = tdFreeDataCols(pComph->pDataCols);
-    pComph->aSupBlk = taosArrayDestroy(pComph->aSupBlk);
-    pComph->aBlkIdx = taosArrayDestroy(pComph->aBlkIdx);
-    tsdbDestroyCompTbArray(pComph);
-    tsdbDestroyReadH(&(pComph->readh));
-    tsdbCloseDFileSet(TSDB_COMPACT_WSET(pComph));
-  }
-
-  static int tsdbInitCompTbArray(SCompactH *pComph) {  // Init pComp->tbArray
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-    if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
-
-    pComph->tbArray = taosArrayInit(pMeta->maxTables, sizeof(STableCompactH));
-    if (pComph->tbArray == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbUnlockRepoMeta(pRepo);
-      return -1;
-    }
-
-    // Note here must start from 0
-    for (int i = 0; i < pMeta->maxTables; i++) {
-      STableCompactH ch = {0};
-      if (pMeta->tables[i] != NULL) {
-        tsdbRefTable(pMeta->tables[i]);
-        ch.pTable = pMeta->tables[i];
-      }
-
-      if (taosArrayPush(pComph->tbArray, &ch) == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        tsdbUnlockRepoMeta(pRepo);
-        return -1;
-      }
-    }
-
-    if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
-    return 0;
-  }
-
-  static void tsdbDestroyCompTbArray(SCompactH *pComph) {
-    STableCompactH *pTh;
-
-    if (pComph->tbArray == NULL) return;
-
-    for (size_t i = 0; i < taosArrayGetSize(pComph->tbArray); i++) {
-      pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, i);
-      if (pTh->pTable) {
-        tsdbUnRefTable(pTh->pTable);
-      }
-
-      pTh->pInfo = taosTZfree(pTh->pInfo);
-    }
-
-    pComph->tbArray = taosArrayDestroy(pComph->tbArray);
-  }
-
-  static int tsdbCacheFSetIndex(SCompactH *pComph) {
-    SReadH *pReadH = &(pComph->readh);
-
-    if (tsdbLoadBlockIdx(pReadH) < 0) {
-      return -1;
-    }
-
-    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
-      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
-      pTh->pBlkIdx = NULL;
-
-      if (pTh->pTable == NULL) continue;
-      if (tsdbSetReadTable(pReadH, pTh->pTable) < 0) {
-        return -1;
-      }
-
-      if (pReadH->pBlkIdx == NULL) continue;
-      pTh->bindex = *(pReadH->pBlkIdx);
-      pTh->pBlkIdx = &(pTh->bindex);
-
-      if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) {
-        return -1;
-      }
-
-      if (tsdbLoadBlockInfo(pReadH, (void *)(pTh->pInfo)) < 0) {
-        return -1;
-      }
-    }
-
-    return 0;
-  }
-
-  static int tsdbCompactFSetInit(SCompactH *pComph, SDFileSet *pSet) {
-    taosArrayClear(pComph->aBlkIdx);
-    taosArrayClear(pComph->aSupBlk);
-
-    if (tsdbSetAndOpenReadFSet(&(pComph->readh), pSet) < 0) {
-      return -1;
-    }
-
-    if (tsdbCacheFSetIndex(pComph) < 0) {
-      tsdbCloseAndUnsetFSet(&(pComph->readh));
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static void tsdbCompactFSetEnd(SCompactH *pComph) { tsdbCloseAndUnsetFSet(&(pComph->readh)); }
-
-  static int tsdbCompactFSetImpl(SCompactH *pComph) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg * pCfg = REPO_CFG(pRepo);
-    SReadH *   pReadh = &(pComph->readh);
-    SBlockIdx  blkIdx;
-    void **    ppBuf = &(TSDB_COMPACT_BUF(pComph));
-    void **    ppCBuf = &(TSDB_COMPACT_COMP_BUF(pComph));
-    int        defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock);
-
-    taosArrayClear(pComph->aBlkIdx);
-
-    for (int tid = 1; tid < taosArrayGetSize(pComph->tbArray); tid++) {
-      STableCompactH *pTh = (STableCompactH *)taosArrayGet(pComph->tbArray, tid);
-      STSchema *      pSchema;
-
-      if (pTh->pTable == NULL || pTh->pBlkIdx == NULL) continue;
-
-      pSchema = tsdbGetTableSchemaImpl(pTh->pTable, true, true, -1);
-      taosArrayClear(pComph->aSupBlk);
-      if ((tdInitDataCols(pComph->pDataCols, pSchema) < 0) || (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) ||
-          (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0)) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-      tdFreeSchema(pSchema);
-
-      // Loop to compact each block data
-      for (int i = 0; i < pTh->pBlkIdx->numOfBlocks; i++) {
-        SBlock *pBlock = pTh->pInfo->blocks + i;
-
-        // Load the block data
-        if (tsdbLoadBlockData(pReadh, pBlock, pTh->pInfo) < 0) {
-          return -1;
-        }
-
-        // Merge pComph->pDataCols and pReadh->pDCols[0] and write data to file
-        if (pComph->pDataCols->numOfRows == 0 && pBlock->numOfRows >= defaultRows) {
-          if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pReadh->pDCols[0], ppBuf, ppCBuf) < 0) {
-            return -1;
-          }
-        } else {
-          int ridx = 0;
-
-          while (true) {
-            if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
-            int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
-
-            tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);
-
-            if (pComph->pDataCols->numOfRows < defaultRows) {
-              break;
-            }
-
-            if (tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
-              return -1;
-            }
-            tdResetDataCols(pComph->pDataCols);
-          }
-        }
-      }
-
-      if (pComph->pDataCols->numOfRows > 0 &&
-          tsdbWriteBlockToRightFile(pComph, pTh->pTable, pComph->pDataCols, ppBuf, ppCBuf) < 0) {
-        return -1;
-      }
-
-      if (tsdbWriteBlockInfoImpl(TSDB_COMPACT_HEAD_FILE(pComph), pTh->pTable, pComph->aSupBlk, NULL, ppBuf, &blkIdx) <
-          0) {
-        return -1;
-      }
-
-      if ((blkIdx.numOfBlocks > 0) && (taosArrayPush(pComph->aBlkIdx, (void *)(&blkIdx)) == NULL)) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-    }
-
-    if (tsdbWriteBlockIdx(TSDB_COMPACT_HEAD_FILE(pComph), pComph->aBlkIdx, ppBuf) < 0) {
-      return -1;
-    }
-
-    return 0;
-  }
-
-  static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
-                                      void **ppCBuf) {
-    STsdbRepo *pRepo = TSDB_COMPACT_REPO(pComph);
-    STsdbCfg * pCfg = REPO_CFG(pRepo);
-    SDFile *   pDFile;
-    bool       isLast;
-    SBlock     block;
-
-    ASSERT(pDataCols->numOfRows > 0);
-
-    if (pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
-      pDFile = TSDB_COMPACT_LAST_FILE(pComph);
-      isLast = true;
-    } else {
-      pDFile = TSDB_COMPACT_DATA_FILE(pComph);
-      isLast = false;
-    }
-
-    if (tsdbWriteBlockImpl(pRepo, pTable, pDFile, pDataCols, &block, isLast, true, ppBuf, ppCBuf) < 0) {
-      return -1;
-    }
-
-    if (taosArrayPush(pComph->aSupBlk, (void *)(&block)) == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      return -1;
-    }
-
-    return 0;
-}
-
-#endif
-

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
 static const char *tsdbTxnFname[] = {"current.t", "current"};
@@ -246,62 +246,6 @@ void *tsdbFreeFS(STsdbFS *pfs) {
   return NULL;
 }
 
-// static int tsdbProcessExpiredFS(STsdb *pRepo) {
-//   tsdbStartFSTxn(pRepo, 0, 0);
-//   // if (tsdbCreateMeta(pRepo) < 0) {
-//   //   tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//   //   return -1;
-//   // }
-
-//   if (tsdbApplyRtn(pRepo) < 0) {
-//     tsdbEndFSTxnWithError(REPO_FS(pRepo));
-//     tsdbError("vgId:%d failed to apply rtn since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-//   if (tsdbEndFSTxn(pRepo) < 0) {
-//     tsdbError("vgId:%d failed to end fs txn since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-//   return 0;
-// }
-
-// static int tsdbCreateMeta(STsdb *pRepo) {
-//   STsdbFS *pfs = REPO_FS(pRepo);
-//   SMFile * pOMFile = pfs->cstatus->pmf;
-//   SMFile   mf;
-//   SDiskID  did;
-
-//   if (pOMFile != NULL) {
-//     // keep the old meta file
-//     tsdbUpdateMFile(pfs, pOMFile);
-//     return 0;
-//   }
-
-//   // Create a new meta file
-//   did.level = TFS_PRIMARY_LEVEL;
-//   did.id = TFS_PRIMARY_ID;
-//   tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
-
-//   if (tsdbCreateMFile(&mf, true) < 0) {
-//     tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//     return -1;
-//   }
-
-//   tsdbInfo("vgId:%d meta file %s is created", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf));
-
-//   if (tsdbUpdateMFileHeader(&mf) < 0) {
-//     tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno));
-//     tsdbApplyMFileChange(&mf, pOMFile);
-//     return -1;
-//   }
-
-//   TSDB_FILE_FSYNC(&mf);
-//   tsdbCloseMFile(&mf);
-//   tsdbUpdateMFile(pfs, &mf);
-
-//   return 0;
-// }
-
 int tsdbOpenFS(STsdb *pRepo) {
   STsdbFS *pfs = REPO_FS(pRepo);
   char     current[TSDB_FILENAME_LEN] = "\0";
@@ -644,7 +588,7 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
 }
 
 static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
-  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), pRepo->vgId,
+  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
            tsdbTxnFname[ftype]);
 }
 
@@ -769,142 +713,6 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) {
   return 0;
 }
 
-// int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta) {
-//   char      tbuf[128];
-//   STsdbFS * pfs = REPO_FS(pRepo);
-//   SMFile    mf;
-//   SMFile *  pMFile = &mf;
-//   void *    pBuf = NULL;
-//   SKVRecord rInfo;
-//   int64_t   maxBufSize = 0;
-//   SMFInfo   minfo;
-
-//   taosHashClear(pfs->metaCache);
-
-//   // No meta file, just return
-//   if (pfs->cstatus->pmf == NULL) return 0;
-
-//   mf = pfs->cstatus->mf;
-//   // Load cache first
-//   if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
-//     return -1;
-//   }
-
-//   if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
-//     tsdbCloseMFile(pMFile);
-//     return -1;
-//   }
-
-//   while (true) {
-//     int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
-//     if (tsize == 0) break;
-
-//     if (tsize < 0) {
-//       tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno));
-//       return -1;
-//     }
-
-//     if (tsize < sizeof(SKVRecord)) {
-//       tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
-//                 TSDB_FILE_FULL_NAME(pMFile));
-//       terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//       tsdbCloseMFile(pMFile);
-//       return -1;
-//     }
-
-//     void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
-//     ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
-//     // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
-
-//     if (rInfo.offset < 0) {
-//       taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
-// #if 0
-//       pStore->info.size += sizeof(SKVRecord);
-//       pStore->info.nRecords--;
-//       pStore->info.nDels++;
-//       pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-// #endif
-//     } else {
-//       ASSERT(rInfo.offset > 0 && rInfo.size > 0);
-//       if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
-//         tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
-//                   TSDB_FILE_FULL_NAME(pMFile));
-//         terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       maxBufSize = TMAX(maxBufSize, rInfo.size);
-
-//       if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
-//         tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-// #if 0
-//       pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
-//       pStore->info.nRecords++;
-// #endif
-//     }
-//   }
-
-//   if (recoverMeta) {
-//     pBuf = taosMemoryMalloc((size_t)maxBufSize);
-//     if (pBuf == NULL) {
-//       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-//       tsdbCloseMFile(pMFile);
-//       return -1;
-//     }
-
-//     SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
-//     while (pRecord) {
-//       if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
-//         tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
-//       if (nread < 0) {
-//         tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       if (nread < pRecord->size) {
-//         tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
-//                   REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
-//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
-//         tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
-//                   tstrerror(terrno));
-//         taosMemoryFreeClear(pBuf);
-//         tsdbCloseMFile(pMFile);
-//         return -1;
-//       }
-
-//       pRecord = taosHashIterate(pfs->metaCache, pRecord);
-//     }
-
-//     tsdbOrgMeta(pRepo);
-//   }
-
-//   tsdbCloseMFile(pMFile);
-//   taosMemoryFreeClear(pBuf);
-//   return 0;
-// }
-
 static int tsdbScanRootDir(STsdb *pRepo) {
   char            rootDir[TSDB_FILENAME_LEN];
   char            bname[TSDB_FILENAME_LEN];
@@ -983,127 +791,6 @@ static bool tsdbIsTFileInFS(STsdbFS *pfs, const STfsFile *pf) {
   return false;
 }
 
-// static int tsdbRestoreMeta(STsdb *pRepo) {
-//   char         rootDir[TSDB_FILENAME_LEN];
-//   char         bname[TSDB_FILENAME_LEN];
-//   STfsDir *       tdir = NULL;
-//   const STfsFile *pf = NULL;
-//   const char * pattern = "^meta(-ver[0-9]+)?$";
-//   regex_t      regex;
-//   STsdbFS *    pfs = REPO_FS(pRepo);
-
-//   regcomp(&regex, pattern, REG_EXTENDED);
-
-//   tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
-
-//   tsdbGetRootDir(REPO_ID(pRepo), rootDir);
-
-//   tdir = tfsOpendir(rootDir);
-//   if (tdir == NULL) {
-//     tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
-//     regfree(&regex);
-//     return -1;
-//   }
-
-//   while ((pf = tfsReaddir(tdir))) {
-//     tfsBasename(pf, bname);
-
-//     if (strcmp(bname, "data") == 0) {
-//       // Skip the data/ directory
-//       continue;
-//     }
-
-//     if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
-//       // Skip current.t file
-//       tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), pf->aname);
-//       (void)tfsremove(pf);
-//       continue;
-//     }
-
-//     int code = regexec(&regex, bname, 0, NULL, 0);
-//     if (code == 0) {
-//       // Match
-//       if (pfs->cstatus->pmf != NULL) {
-//         tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
-//                   TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), pf->aname);
-//         terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-//         tfsClosedir(tdir);
-//         regfree(&regex);
-//         return -1;
-//       } else {
-//         uint32_t _version = 0;
-//         if (strcmp(bname, "meta") != 0) {
-//           sscanf(bname, "meta-ver%" PRIu32, &_version);
-//           pfs->cstatus->meta.version = _version;
-//         }
-
-//         pfs->cstatus->pmf = &(pfs->cstatus->mf);
-//         pfs->cstatus->pmf->f = *pf;
-//         TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
-
-//         if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
-//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//           tfsClosedir(tdir);
-//           regfree(&regex);
-//           return -1;
-//         }
-
-//         if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
-//           tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-//           tsdbCloseMFile(pfs->cstatus->pmf);
-//           tfsClosedir(tdir);
-//           regfree(&regex);
-//           return -1;
-//         }
-
-//         if (tsdbForceKeepFile) {
-//           struct stat tfstat;
-
-//           // Get real file size
-//           if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) {
-//             terrno = TAOS_SYSTEM_ERROR(errno);
-//             tsdbCloseMFile(pfs->cstatus->pmf);
-//             tfsClosedir(tdir);
-//             regfree(&regex);
-//             return -1;
-//           }
-
-//           if (pfs->cstatus->pmf->info.size != tfstat.st_size) {
-//             int64_t tfsize = pfs->cstatus->pmf->info.size;
-//             pfs->cstatus->pmf->info.size = tfstat.st_size;
-//             tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
-//                      TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size);
-//           }
-//         }
-
-//         tsdbCloseMFile(pfs->cstatus->pmf);
-//       }
-//     } else if (code == REG_NOMATCH) {
-//       // Not match
-//       tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname);
-//       tfsremove(pf);
-//       continue;
-//     } else {
-//       // Has other error
-//       tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
-//       terrno = TAOS_SYSTEM_ERROR(code);
-//       tfsClosedir(tdir);
-//       regfree(&regex);
-//       return -1;
-//     }
-//   }
-
-//   if (pfs->cstatus->pmf) {
-//     tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
-//   } else {
-//     tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
-//   }
-
-//   tfsClosedir(tdir);
-//   regfree(&regex);
-//   return 0;
-// }
-
 static int tsdbRestoreDFileSet(STsdb *pRepo) {
   char            dataDir[TSDB_FILENAME_LEN];
   char            bname[TSDB_FILENAME_LEN];

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 static const char *TSDB_FNAME_SUFFIX[] = {
     "head",  // TSDB_FILE_HEAD
@@ -33,271 +33,6 @@ static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
 static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
 static int   tsdbRollBackDFile(SDFile *pDFile);
 
-#if 0
-// ============== SMFile
-void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
-  char fname[TSDB_FILENAME_LEN];
-
-  TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK);
-
-  memset(&(pMFile->info), 0, sizeof(pMFile->info));
-  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
-
-  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
-  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
-}
-
-void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) {
-  *pMFile = *pOMFile;
-  TSDB_FILE_SET_CLOSED(pMFile);
-}
-
-int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
-  int tlen = 0;
-
-  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
-  tlen += tfsEncodeFile(buf, &(pMFile->f));
-
-  return tlen;
-}
-
-void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
-  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
-  buf = tfsDecodeFile(buf, &(pMFile->f));
-  TSDB_FILE_SET_CLOSED(pMFile);
-
-  return buf;
-}
-
-int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
-  int tlen = 0;
-
-  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
-  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));
-
-  return tlen;
-}
-
-void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
-  char *aname;
-  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
-  buf = taosDecodeString(buf, &aname);
-  strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
-  TSDB_FILE_SET_CLOSED(pMFile);
-
-  taosMemoryFreeClear(aname);
-
-  return buf;
-}
-
-int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
-  if (from == NULL && to == NULL) return 0;
-
-  if (from != NULL) {
-    if (to == NULL) {
-      return tsdbRemoveMFile(from);
-    } else {
-      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
-        if (from->info.size > to->info.size) {
-          tsdbRollBackMFile(to);
-        }
-      } else {
-        return tsdbRemoveMFile(from);
-      }
-    }
-  }
-
-  return 0;
-}
-
-int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
-  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);
-
-  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
-  if (pMFile->fd < 0) {
-    if (errno == ENOENT) {
-      // Try to create directory recursively
-      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
-      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
-        taosMemoryFreeClear(s);
-        return -1;
-      }
-      taosMemoryFreeClear(s);
-
-      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
-      if (pMFile->fd < 0) {
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-    } else {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-  }
-
-  if (!updateHeader) {
-    return 0;
-  }
-
-  pMFile->info.size += TSDB_FILE_HEAD_SIZE;
-
-  if (tsdbUpdateMFileHeader(pMFile) < 0) {
-    tsdbCloseMFile(pMFile);
-    tsdbRemoveMFile(pMFile);
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbUpdateMFileHeader(SMFile *pMFile) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
-    return -1;
-  }
-
-  void *ptr = buf;
-  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
-  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  ASSERT(TSDB_FILE_OPENED(pMFile));
-
-  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
-    return -1;
-  }
-
-  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  tsdbDecodeMFInfo(buf, pInfo);
-  return 0;
-}
-
-int tsdbScanAndTryFixMFile(STsdb *pRepo) {
-  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
-  struct stat mfstat;
-  SMFile      mf;
-
-  if (pMFile == NULL) {
-    // No meta file, no need to scan
-    return 0;
-  }
-
-  tsdbInitMFileEx(&mf, pMFile);
-
-  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
-    tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
-              TSDB_FILE_FULL_NAME(pMFile));
-    pRepo->state |= TSDB_STATE_BAD_META;
-    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
-    return 0;
-  }
-
-  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (pMFile->info.size < mfstat.st_size) {
-    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
-      return -1;
-    }
-
-    if (taosFtruncate(mf.fd, mf.info.size) < 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      tsdbCloseMFile(&mf);
-      return -1;
-    }
-
-    if (tsdbUpdateMFileHeader(&mf) < 0) {
-      tsdbCloseMFile(&mf);
-      return -1;
-    }
-
-    tsdbCloseMFile(&mf);
-    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
-             mfstat.st_size, pMFile->info.size);
-  } else if (pMFile->info.size > mfstat.st_size) {
-    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
-              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
-    pRepo->state |= TSDB_STATE_BAD_META;
-    TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return 0;
-  } else {
-    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
-  }
-
-  return 0;
-}
-
-int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
-  int tlen = 0;
-
-  tlen += taosEncodeVariantI64(buf, pInfo->size);
-  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
-  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
-  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
-  tlen += taosEncodeFixedU32(buf, pInfo->magic);
-
-  return tlen;
-}
-
-void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
-  buf = taosDecodeVariantI64(buf, &(pInfo->size));
-  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
-  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
-
-  return buf;
-}
-
-static int tsdbRollBackMFile(SMFile *pMFile) {
-  SMFile mf;
-
-  tsdbInitMFileEx(&mf, pMFile);
-
-  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
-    return -1;
-  }
-
-  if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    tsdbCloseMFile(&mf);
-    return -1;
-  }
-
-  if (tsdbUpdateMFileHeader(&mf) < 0) {
-    tsdbCloseMFile(&mf);
-    return -1;
-  }
-
-  TSDB_FILE_FSYNC(&mf);
-
-  tsdbCloseMFile(&mf);
-  return 0;
-}
-
-#endif
-
 // ============== Operations on SDFile
 void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype) {
   char fname[TSDB_FILENAME_LEN];
@@ -310,7 +45,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
   pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
   pDFile->info.fver = tsdbGetDFSVersion(ftype);
 
-  tsdbGetFilename(pRepo->vgId, fid, ver, ftype, fname);
+  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname);
   tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
 }
 

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "tsdb.h"
 
 #define TSDB_KEY_COL_OFFSET 0
 

source/dnode/vnode/src/tsdb/tsdbScan.c

@@ -13,9 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #if 0
-#include "tsdbint.h"
+#include "tsdb.h"
 #ifndef _TSDB_PLUGINS
 
 int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }