Merge branch '3.0' into feature/3.0_liaohj

CMakeLists.txt

@@ -35,6 +35,18 @@ cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 ## cJson
 cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${DEPS_TMP_FILE})
 
+## leveldb
+option(BUILD_WITH_LEVELDB "If build with leveldb" OFF)
+if(${BUILD_WITH_LEVELDB})
+    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_LEVELDB})
+
+## rocksdb
+option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON)
+if(${BUILD_WITH_ROCKSDB})
+    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE})
+endif(${BUILD_WITH_ROCKSDB})
+
 ## download dependencies
 configure_file(${DEPS_TMP_FILE} "${CMAKE_SOURCE_DIR}/deps/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .

cmake/EnableCMP0048.txt.in

+cmake_policy(SET CMP0048 NEW)
\ No newline at end of file

cmake/leveldb_CMakeLists.txt.in

+
+# leveldb
+ExternalProject_Add(leveldb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
+        GIT_TAG master
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/leveldb"
+        BINARY_DIR ""
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+)
\ No newline at end of file

cmake/rocksdb_CMakeLists.txt.in

+
+# rocksdb
+ExternalProject_Add(rocksdb
+        GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
+        GIT_TAG v6.23.3
+        SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/rocksdb"
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )
\ No newline at end of file

deps/CMakeLists.txt

+# googletest
 if(${BUILD_TEST})
     add_subdirectory(googletest)
 endif(${BUILD_TEST})
 
+# cJson
+# see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
+option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
+option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
+option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
 add_subdirectory(cJson)
 target_include_directories(
     cjson
     # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
     PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cJson>
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 
+# lz4
 add_subdirectory(lz4/build/cmake)
 target_include_directories(
     lz4_static
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
 )
 
+# zlib
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
 add_subdirectory(zlib)
 target_include_directories(
     zlib
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/zlib
 )
+unset(CMAKE_PROJECT_INCLUDE_BEFORE)
+
+# leveldb
+if(${BUILD_WITH_LEVELDB})
+option(LEVELDB_BUILD_TESTS "" OFF)
+    add_subdirectory(leveldb)
+    target_include_directories(
+        leveldb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
+    )
+endif(${BUILD_WITH_LEVELDB})
+
+# rocksdb
+if(${BUILD_WITH_ROCKSDB})
+    option(WITH_TESTS "" OFF)
+    option(WITH_BENCHMARK_TOOLS "" OFF)
+    option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
+    add_subdirectory(rocksdb)
+    target_include_directories(
+        rocksdb
+        PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
+    )
+endif(${BUILD_WITH_ROCKSDB})

include/common/taosmsg.h

@@ -990,6 +990,22 @@ typedef struct {
   /* data */
 } SAlterTableRsp;
 
+typedef struct {
+  /* data */
+} SDropStableReq;
+
+typedef struct {
+  /* data */
+} SDropStableRsp;
+
+typedef struct {
+  /* data */
+} SUpdateTagValReq;
+
+typedef struct {
+  /* data */
+} SUpdateTagValRsp;
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

src/vnode/inc/vnodeRead.h → include/common/tcompare.h

@@ -13,24 +13,24 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_READ_H
-#define TDENGINE_VNODE_READ_H
+#ifndef _TD_TCOMPARE_H_
+#define _TD_TCOMPARE_H_
+
+#include "compare.h"
+#include "ttypes.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "vnodeInt.h"
-
-int32_t vnodeInitRead(void);
-void    vnodeCleanupRead(void);
 
-int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam);
-void    vnodeFreeFromRQueue(void *pVnode, SVReadMsg *pRead);
-int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead);
-void    vnodeWaitReadCompleted(SVnodeObj *pVnode);
+int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
+int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
+int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_TCOMPARE_H_*/
\ No newline at end of file

include/common/tglobal.h

@@ -57,6 +57,7 @@ extern int32_t  tsCompressMsgSize;
 extern int32_t  tsCompressColData;
 extern int32_t  tsMaxNumOfDistinctResults;
 extern char     tsTempDir[];
+extern int64_t  tsMaxVnodeQueuedBytes;
 
 //query buffer management
 extern int32_t  tsQueryBufferSize;      // maximum allowed usage buffer size in MB for each data node during query processing

include/common/trow.h

@@ -16,10 +16,55 @@
 #ifndef _TD_COMMON_ROW_H_
 #define _TD_COMMON_ROW_H_
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+// types
+typedef void *                  SRow;
+typedef struct SRowBatch        SRowBatch;
+typedef struct SRowBuilder      SRowBuilder;
+typedef struct SRowBatchIter    SRowBatchIter;
+typedef struct SRowBatchBuilder SRowBatchBuilder;
+
+// SRow
+#define ROW_HEADER_SIZE (sizeof(uint8_t) + 2 * sizeof(uint16_t) + sizeof(uint64_t))
+#define rowType(r) (*(uint8_t *)(r))                                // row type
+#define rowLen(r) (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t)))  // row length
+#define rowSVer(r) \
+  (*(uint16_t *)POINTER_SHIFT(r, sizeof(uint8_t) + sizeof(uint16_t)))  // row schema version, only for SDataRow
+#define rowNCols(r) rowSVer(r)                                         // only for SKVRow
+#define rowVer(r) (*(uint64_t)POINTER_SHIFT(r, sizeof(uint8_t) + 2 * sizeof(uint16_t)))  // row version
+#define rowCopy(dest, r) memcpy((dest), r, rowLen(r))
+
+static FORCE_INLINE SRow rowDup(SRow row) {
+  SRow r = malloc(rowLen(row));
+  if (r == NULL) {
+    return NULL;
+  }
+
+  rowCopy(r, row);
+
+  return r;
+}
+
+// SRowBatch
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate();
+void         rowBuilderDestroy(SRowBuilder *);
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *);
+void           rowBatchIterDestroy(SRowBatchIter *);
+const SRow     rowBatchIterNext(SRowBatchIter *);
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);
+
 #ifdef __cplusplus
 }
 #endif

include/common/ttypes.h

@@ -6,10 +6,10 @@ extern "C" {
 #endif
 
 #include "taosdef.h"
+#include "types.h"
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
 typedef int32_t  VarDataOffsetT;
-typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
 typedef uint16_t TDRowLenT;    // not including overhead: 0 ~ 65535
 typedef uint32_t TDRowTLenT;   // total length, including overhead
 
@@ -30,11 +30,7 @@ typedef struct {
 } SNCharNullT;
 #pragma pack(pop)
 
-#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
-
-#define varDataLen(v)       ((VarDataLenT *)(v))[0]
 #define varDataTLen(v)      (sizeof(VarDataLenT) + varDataLen(v))
-#define varDataVal(v)       ((void *)((char *)v + VARSTR_HEADER_SIZE))
 #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
 #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
 #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))

include/libs/raft/raft.h

 /*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ * Copyright (c) 2019 TAOS Data, Inc. <cli@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
  * it under the terms of the GNU Affero General Public License, version 3
@@ -13,15 +13,149 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_RAFT_H_
-#define _TD_RAFT_H_
+#ifndef TD_RAFT_H
+#define TD_RAFT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <stdio.h>
+#include "taosdef.h"
+
+typedef unsigned int RaftId;
+typedef unsigned int RaftGroupId;
+
+// buffer holding data
+typedef struct RaftBuffer {
+  void* data;
+  size_t len;
+} RaftBuffer;
+
+// a single server information in a cluster
+typedef struct RaftServer {
+  RaftId    id;
+  char      fqdn[TSDB_FQDN_LEN];
+  uint16_t  port;
+} RaftServer;
+
+// all servers in a cluster
+typedef struct RaftConfiguration {
+  RaftServer *servers;
+  int nServer;
+} RaftConfiguration;
+
+// raft lib module
+struct Raft;
+typedef struct Raft Raft;
+
+struct RaftNode;
+typedef struct RaftNode RaftNode;
+
+// raft state machine
+struct RaftFSM;
+typedef struct RaftFSM {
+  // statemachine user data
+  void *data;
+
+  // apply buffer data, bufs will be free by raft module
+  int (*apply)(struct RaftFSM *fsm, const RaftBuffer *bufs[], int nBufs);
+
+  // configuration commit callback 
+  int (*onConfigurationCommit)(const RaftConfiguration* cluster);
+
+  // fsm return snapshot in ppBuf, bufs will be free by raft module
+  // TODO: getSnapshot SHOULD be async?
+  int (*getSnapshot)(struct RaftFSM *fsm, RaftBuffer **ppBuf);
+
+  // fsm restore with pBuf data
+  int (*restore)(struct RaftFSM *fsm, RaftBuffer *pBuf);
+
+  // fsm send data in buf to server,buf will be free by raft module
+  int (*send)(struct RaftFSM* fsm, const RaftServer* server, const RaftBuffer *buf);
+} RaftFSM;
+
+typedef struct RaftNodeOptions {
+  // user define state machine
+  RaftFSM* pFSM;
+
+  // election timeout(in ms)
+  // by default: 1000
+  int electionTimeoutMS;
+
+  // heart timeout(in ms)
+  // by default: 100
+  int heartbeatTimeoutMS;
+
+  // install snapshot timeout(in ms)
+  int installSnapshotTimeoutMS;
+
+  /** 
+   * number of log entries before starting a new snapshot. 
+   * by default: 1024
+   */
+  int snapshotThreshold;
+
+  /**
+   * Number of log entries to keep in the log after a snapshot has
+   * been taken. 
+   * by default: 128.
+   */
+  int snapshotTrailing;
+
+  /**
+   * Enable or disable pre-vote support.
+   * by default: false
+   */
+  bool preVote;
+
+} RaftNodeOptions;
+
+// create raft lib
+int RaftCreate(Raft** ppRaft);
+
+int RaftDestroy(Raft* pRaft);
+
+// start a raft node with options,node id,group id
+int RaftStart(Raft* pRaft, 
+              RaftId selfId,
+              RaftGroupId selfGroupId,
+              const RaftConfiguration* cluster,
+              const RaftNodeOptions* options,              
+              RaftNode **ppNode);
+
+// stop a raft node
+int RaftStop(RaftNode* pNode);
+
+// client apply a cmd in buf
+typedef void (*RaftApplyFp)(const RaftBuffer *pBuf, int result);
+
+int RaftApply(RaftNode *pNode,
+              const RaftBuffer *pBuf,
+              RaftApplyFp applyCb);
+
+// recv data from other servers in cluster,buf will be free in raft
+int RaftRecv(RaftNode *pNode, const RaftBuffer* pBuf);
+
+// change cluster servers API
+typedef void (*RaftChangeFp)(const RaftServer* pServer, int result);
+
+int RaftAddServer(RaftNode *pNode,
+                  const RaftServer* pServer,
+                  RaftChangeFp changeCb);
+
+int RaftRemoveServer(RaftNode *pNode, 
+                    const RaftServer* pServer,
+                    RaftChangeFp changeCb);
+
+// transfer leader to id
+typedef void (*RaftTransferFp)(RaftId id, int result);
+int RaftTransfer(RaftNode *pNode,
+                 RaftId id,
+                 RaftTransferFp transferCb);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_RAFT_H_*/
\ No newline at end of file
+#endif  /* TD_RAFT_H */ 
\ No newline at end of file

include/libs/tkv/tkv.h

@@ -20,52 +20,25 @@
 extern "C" {
 #endif
 
-typedef struct tkv_db_s tkv_db_t;
-
-typedef struct {
-  /* data */
-} tkv_key_t;
-
-typedef struct {
-  bool    pinned;
-  int64_t ref;  // TODO: use util library
-                // TODO: add a RW latch here
-  uint64_t offset;
-  void *   pObj;
-} tkv_obj_t;
-
-typedef int (*tkv_key_comp_fn_t)(const tkv_key_t *, const tkv_key_t *);
-typedef void (*tkv_get_key_fn_t)(const tkv_obj_t *, tkv_key_t *);
-typedef int (*tkv_obj_encode_fn_t)(void **buf, void *pObj);
-typedef void *(*tkv_obj_decode_fn_t)(void *buf, void **pObj);
-typedef int (*tkv_obj_comp_fn_t)(const tkv_obj_t *, const tkv_obj_t *);
-typedef void (*tkv_obj_destroy_fn_t)(void *);
-
-typedef struct {
-  uint64_t             memLimit;
-  tkv_get_key_fn_t     getKey;
-  tkv_obj_encode_fn_t  encode;
-  tkv_obj_decode_fn_t  decode;
-  tkv_obj_comp_fn_t    compare;
-  tkv_obj_destroy_fn_t destroy;
-} tkv_db_option_t;
-
-tkv_db_t *       tkvOpenDB(char *dir, tkv_db_option_t *);
-int              tkvCloseDB(tkv_db_t *);
-int              tkvPut(tkv_db_t *, tkv_obj_t *);
-int              tkvPutBatch(tkv_db_t *, tkv_obj_t **, int);  // TODO: use array here
-const tkv_obj_t *tkvGet(tkv_key_t *);
-int              tkvGetBatch(tkv_db_t *, tkv_key_t **, int, tkv_obj_t **);  // TODO: use array here
-int              tkvDrop(tkv_db_t *, tkv_key_t *);
-int              tkvDropBatch(tkv_db_t *, tkv_key_t **, int);  // TODO: use array here
-int              tkvCommit(tkv_db_t *, void * /*TODO*/);
-
-typedef struct {
-} tkv_db_iter_t;
-
-tkv_db_iter_t *  tkvIterNew(tkv_db_t *);
-void             tkvIterFree(tkv_db_iter_t *);
-const tkv_obj_t *tkvIterNext(tkv_db_iter_t *);
+// Types exported
+typedef struct STkvDb      STkvDb;
+typedef struct STkvOptions STkvOptions;
+typedef struct STkvCache   STkvCache;
+
+// DB operations
+STkvDb *tkvOpen(const STkvOptions *options, const char *path);
+void    tkvClose(STkvDb *db);
+void    tkvPut(STkvDb *db, void * /*TODO*/);
+
+// DB options
+STkvOptions *tkvOptionsCreate();
+void         tkvOptionsDestroy(STkvOptions *);
+void         tkvOptionsSetCache(STkvOptions *, STkvCache *);
+
+// DB cache
+typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType;
+STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type);
+void       tkvCacheDestroy(STkvCache *);
 
 #ifdef __cplusplus
 }

include/libs/wal/wal.h

@@ -60,8 +60,8 @@ void     walClose(twalh);
 //write
 int64_t  walWrite(twalh, int8_t msgType, void* body, uint32_t bodyLen);
 void     walFsync(twalh, bool forceHint);
-//int32_t  walCommit(twalh, uint64_t ver);
-//int32_t  walRollback(twalh, uint64_t ver);
+//int32_t  walCommit(twalh, int64_t ver);
+//int32_t  walRollback(twalh, int64_t ver);
 
 //read
 int32_t  walRead(twalh, SWalHead **, int64_t ver);

include/server/mnode/mnode.h

@@ -118,7 +118,7 @@ typedef struct {
 int32_t mnodeGetStatistics(SMnodeStat *stat);
 
 /**
- * Get the auth information.
+ * Get the auth information of Mnode.
  *
  * @param user, username.
  * @param spi,  security parameter index.

include/server/vnode/meta/meta.h

@@ -18,16 +18,42 @@
 
 #include "taosmsg.h"
 
+#include "os.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SMeta SMeta;
+typedef uint64_t tuid_t;
+
+// Types exported
+typedef struct SMeta             SMeta;
+typedef struct SMetaOptions      SMetaOptions;
+typedef struct SMetaQueryHandle  SMetaQueryHandle;
+typedef struct SMetaQueryOptions SMetaQueryOptions;
+
+// SMeta operations
+int    metaCreate(const char *path);
+int    metaDestroy(const char *path);
+SMeta *metaOpen(SMetaOptions *);
+void   metaClose(SMeta *);
+int    metaCreateTable(SMeta *, void *);
+int    metaDropTable(SMeta *, uint64_t tuid_t);
+int    metaAlterTable(SMeta *, void *);
+int    metaCommit(SMeta *);
+
+// Options
+SMetaOptions *metaOptionsCreate();
+void          metaOptionsDestroy(SMetaOptions *);
+void          metaOptionsSetCache(SMetaOptions *, size_t capacity);
+
+// SMetaQueryHandle
+SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *);
+void              metaQueryHandleDestroy(SMetaQueryHandle *);
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq);
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq);
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq);
-int metaCommit(SMeta *pMeta);
+// SMetaQueryOptions
+SMetaQueryOptions *metaQueryOptionsCreate();
+void               metaQueryOptionsDestroy(SMetaQueryOptions *);
 
 #ifdef __cplusplus
 }

include/server/vnode/tq/tq.h

@@ -25,23 +25,14 @@ extern "C" {
 typedef struct STQ STQ;
 
 STQ* tqInit(void* ref_func(void*), void* unref_func(void*));
-void tqCleanUp(STQ* pTq);
-
-//create persistent storage for meta info such as consuming offset
-//return value > 0: cgId
-//return value <= 0: error code
-int tqCreateGroup(STQ*);
-//create ring buffer in memory and load consuming offset
-int tqOpenGroup(STQ*, int cgId);
-//destroy ring buffer and persist consuming offset
-int tqCloseGroup(STQ*, int cgId);
-//delete persistent storage for meta info
-int tqDropGroup(STQ*, int cgId);
-
-int tqPushMsg(STQ*, void *, int64_t version);
+void tqCleanUp(STQ*);
+
+//void* will be replace by a msg type
+int tqPushMsg(STQ*, void* msg, int64_t version);
 int tqCommit(STQ*);
 
-int tqHandleMsg(STQ*, void *msg);
+//void* will be replace by a msg type
+int tqHandleMsg(STQ*, void* msg);
 
 #ifdef __cplusplus
 }

include/server/vnode/tsdb/tsdb.h

@@ -23,33 +23,60 @@
 extern "C" {
 #endif
 
-typedef struct STsdb STsdb;
-typedef struct {
-  int32_t id; // TODO: use a global definition
-  int32_t days;
-  int32_t keep;
-  int32_t keep1;
-  int32_t keep2;
-  int32_t minRows;
-  int32_t maxRows;
-  int8_t  precision;
-  int8_t  update;
-} STsdbCfg;
-
-// Module init and clear
-int tsdbInit();
-int tsdbClear();
-
-// Repository operations
-int    tsdbCreateRepo(int id);
-int    tsdbDropRepo(int id);
-STsdb *tsdbOpenRepo(STsdbCfg *pCfg);
-int    tsdbCloseRepo(STsdb *pTsdb);
-int    tsdbForceCloseRepo(STsdb *pTsdb);
-
-// Data commit
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg);
-int tsdbCommit(STsdb *pTsdb);
+// Types exported
+typedef struct STsdb             STsdb;
+typedef struct STsdbOptions      STsdbOptions;
+typedef struct STsdbSMAOptions   STsdbSMAOptions;  // SMA stands for Small Materialized Aggregation
+typedef struct STsdbReadOptions  STsdbReadOptions;
+typedef struct STsdbSnapshot     STsdbSnapshot;
+typedef struct STsdbQueryHandle  STsdbQueryHandle;
+
+// DB operations
+int    tsdbCreate(const char *path);
+int    tsdbDestroy(const char *path);
+STsdb *tsdbOpen(const STsdbOptions *options);
+void   tsdbClose(STsdb *);
+int    tsdbReset(STsdb *, const STsdbOptions *);
+int    tsdbInsert(STsdb *, SSubmitReq *, SSubmitRsp *);
+int    tsdbCommit(STsdb *);
+int    tsdbCompact(STsdb *);
+
+// Options
+STsdbOptions *tsdbOptionsCreate();
+void          tsdbOptionsDestroy(STsdbOptions *);
+void          tsdbOptionsSetId(STsdbOptions *, int id);
+void          tsdbOptionsSetHoursPerFile(STsdbOptions *, int hours);
+void          tsdbOptionsSetRetention(STsdbOptions *, int keep, int keep1, int keep2);
+void          tsdbOptionsSetMinAndMaxRows(STsdbOptions *, int minRows, int maxRows);
+void          tsdbOptionsSetPrecision(STsdbOptions *, int);
+void          tsdbOptionsSetCache(STsdbOptions *, int);
+typedef enum { TSDB_NO_UPDATE = 0, TSDB_WHOLE_ROW_UPDATE = 1, TSDB_PARTIAL_ROW_UPDATE = 2 } ETsdbUpdateType;
+void tsdbOptionsSetUpdate(STsdbOptions *, ETsdbUpdateType);
+void tsdbOptionsSetSMA(STsdbOptions *, STsdbSMAOptions *);
+
+// STsdbSMAOptions
+STsdbSMAOptions *tsdbSMAOptionsCreate();
+void             tsdbSMAOptionsDestroy(STsdbSMAOptions *);
+// void             tsdbSMAOptionsSetFuncs(STsdbSMAOptions *, SArray * /*Array of function to perform on each block*/);
+// void             tsdbSMAOptionsSetIntervals(STsdbSMAOptions *, SArray *);
+// void             tsdbSMAOptionsSetColTypes(STsdbSMAOptions *, SArray *);
+
+// STsdbQueryHandle
+STsdbQueryHandle *tsdbQueryHandleCreate(STsdb *, STsdbReadOptions *);
+void              tsdbQueryHandleDestroy(STsdbQueryHandle *);
+void              tsdbResetQueryHandle(STsdbQueryHandle *, STsdbReadOptions *);
+bool              tsdbNextDataBlock(STsdbQueryHandle *);
+// void              tsdbGetDataBlockInfo(STsdbQueryHandle *, SDataBlockInfo *);
+// void              tsdbGetDataBlockStatisInfo(STsdbQueryHandle *, SDataStatis **);
+
+// STsdbReadOptions
+STsdbReadOptions *tsdbReadOptionsCreate();
+void              tsdbReadOptionsDestroy(STsdbReadOptions *);
+void              tsdbReadOptionsSetSnapshot(STsdbReadOptions *, STsdbSnapshot *);
+
+// STsdbSnapshot
+STsdbSnapshot *tsdbSnapshotCreate(STsdb *);
+void           tsdbSnapshotDestroy(STsdbSnapshot *);
 
 #ifdef __cplusplus
 }

include/server/vnode/vnode.h

@@ -49,7 +49,7 @@ typedef struct {
 } SVnodeFp;
 
 typedef struct {
-  SVnodeFp      fp;
+  SVnodeFp fp;
 } SVnodePara;
 
 /**

include/util/tcompare.h → include/util/compare.h

@@ -46,11 +46,6 @@ int patternMatch(const char *pattern, const char *str, size_t size, const SPatte
 
 int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo);
 
-int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
-
-__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr);
 
 int32_t taosArrayCompareString(const void* a, const void* b);
 
@@ -82,12 +77,22 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
 
 int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight);
 int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight);
 int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight);
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight);
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight);
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight);
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -104,33 +104,6 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
-#define GET_INT8_VAL(x)    (*(int8_t *)(x))
-#define GET_INT16_VAL(x)   (*(int16_t *)(x))
-#define GET_INT32_VAL(x)   (*(int32_t *)(x))
-#define GET_INT64_VAL(x)   (*(int64_t *)(x))
-#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
-#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
-#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
-#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
-
-#ifdef _TD_ARM_32
-  float  taos_align_get_float(const char* pBuf);
-  double taos_align_get_double(const char* pBuf);
-
-  #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
-  #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
-  #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
-  #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
-  #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
-  #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
-#else
-  #define GET_FLOAT_VAL(x)       (*(float *)(x))
-  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
-  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
-  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
-  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
-  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
-#endif
 
 // TODO: check if below is necessary
 #define TSDB_RELATION_INVALID     0

include/util/tstep.h

@@ -20,14 +20,14 @@
 extern "C" {
 #endif
 
-typedef int32_t (*InitFp)(void **obj);
-typedef void (*CleanupFp)(void **obj);
+typedef int32_t (*InitFp)();
+typedef void (*CleanupFp)();
 typedef void (*ReportFp)(char *name, char *desc);
 
 struct SSteps *taosStepInit(int32_t maxsize, ReportFp fp);
 int32_t        taosStepExec(struct SSteps *steps);
 void           taosStepCleanup(struct SSteps *steps);
-int32_t        taosStepAdd(struct SSteps *steps, char *name, void **obj, InitFp initFp, CleanupFp cleanupFp);
+int32_t        taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp);
 
 #ifdef __cplusplus
 }

include/util/tworker.h

@@ -20,14 +20,16 @@
 extern "C" {
 #endif
 
-typedef int32_t (*ProcessReqFp)(void *ahandle, void *msg);
-typedef void (*SendRspFp)(void *ahandle, void *msg, int32_t qtype, int32_t code);
+typedef int32_t (*ProcessStartFp)(void *ahandle, void *pMsg, int32_t qtype);
+typedef void (*ProcessEndFp)(void *ahandle, void *pMsg, int32_t qtype, int32_t code);
 
-struct SWorkerPool;
+typedef bool (*ProcessWriteStartFp)(void *ahandle, void *pMsg, int32_t qtype);
+typedef void (*ProcessWriteSyncFp)(void *ahandle, int32_t code);
+typedef void (*ProcessWriteEndFp)(void *ahandle, void *pMsg, int32_t qtype);
 
-typedef struct {
-  pthread_t           thread;  // thread
+typedef struct SWorker {
   int32_t             id;      // worker ID
+  pthread_t           thread;  // thread
   struct SWorkerPool *pool;
 } SWorker;
 
@@ -35,18 +37,42 @@ typedef struct SWorkerPool {
   int32_t         max;  // max number of workers
   int32_t         min;  // min number of workers
   int32_t         num;  // current number of workers
-  void *          qset;
+  taos_qset       qset;
   const char *    name;
+  ProcessStartFp  startFp;
+  ProcessEndFp    endFp;
   SWorker *       workers;
-  ProcessReqFp    reqFp;
-  SendRspFp       rspFp;
   pthread_mutex_t mutex;
 } SWorkerPool;
 
-int32_t tWorkerInit(SWorkerPool *pPool);
-void    tWorkerCleanup(SWorkerPool *pPool);
-void *  tWorkerAllocQueue(SWorkerPool *pPool, void *ahandle);
-void    tWorkerFreeQueue(SWorkerPool *pPool, void *pQueue);
+typedef struct SWriteWorker {
+  int32_t                  id;      // worker id
+  pthread_t                thread;  // thread
+  taos_qall                qall;
+  taos_qset                qset;  // queue set
+  struct SWriteWorkerPool *pool;
+} SWriteWorker;
+
+typedef struct SWriteWorkerPool {
+  int32_t             max;     // max number of workers
+  int32_t             nextId;  // from 0 to max-1, cyclic
+  const char *        name;
+  ProcessWriteStartFp startFp;
+  ProcessWriteSyncFp  syncFp;
+  ProcessWriteEndFp   endFp;
+  SWriteWorker *      workers;
+  pthread_mutex_t     mutex;
+} SWriteWorkerPool;
+
+int32_t    tWorkerInit(SWorkerPool *pool);
+void       tWorkerCleanup(SWorkerPool *pool);
+taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle);
+void       tWorkerFreeQueue(SWorkerPool *pool, taos_queue queue);
+
+int32_t    tWriteWorkerInit(SWriteWorkerPool *pool);
+void       tWriteWorkerCleanup(SWriteWorkerPool *pool);
+taos_queue tWriteWorkerAllocQueue(SWriteWorkerPool *pool, void *ahandle);
+void       tWriteWorkerFreeQueue(SWriteWorkerPool *pool, taos_queue queue);
 
 #ifdef __cplusplus
 }

include/util/types.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TYPES_H_
+#define _TD_TYPES_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define GET_INT8_VAL(x)    (*(int8_t *)(x))
+#define GET_INT16_VAL(x)   (*(int16_t *)(x))
+#define GET_INT32_VAL(x)   (*(int32_t *)(x))
+#define GET_INT64_VAL(x)   (*(int64_t *)(x))
+#define GET_UINT8_VAL(x)   (*(uint8_t*) (x))
+#define GET_UINT16_VAL(x)  (*(uint16_t *)(x))
+#define GET_UINT32_VAL(x)  (*(uint32_t *)(x))
+#define GET_UINT64_VAL(x)  (*(uint64_t *)(x))
+
+static FORCE_INLINE float taos_align_get_float(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
+#else
+  assert(sizeof(float) == sizeof(uint32_t));
+#endif
+  float fv = 0;
+  memcpy(&fv, pBuf, sizeof(fv));  // in ARM, return *((const float*)(pBuf)) may cause problem
+  return fv;
+}
+
+static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
+#if __STDC_VERSION__ >= 201112L
+  static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)");
+#else
+  assert(sizeof(double) == sizeof(uint64_t));
+#endif
+  double dv = 0;
+  memcpy(&dv, pBuf, sizeof(dv));  // in ARM, return *((const double*)(pBuf)) may cause problem
+  return dv;
+}
+
+// #ifdef _TD_ARM_32
+//   float  taos_align_get_float(const char* pBuf);
+//   double taos_align_get_double(const char* pBuf);
+
+//   #define GET_FLOAT_VAL(x)       taos_align_get_float(x)
+//   #define GET_DOUBLE_VAL(x)      taos_align_get_double(x)
+//   #define SET_FLOAT_VAL(x, y)  { float z = (float)(y);   (*(int32_t*) x = *(int32_t*)(&z)); }
+//   #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); }
+//   #define SET_FLOAT_PTR(x, y)  { (*(int32_t*) x = *(int32_t*)y); }
+//   #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); }
+// #else
+  #define GET_FLOAT_VAL(x)       (*(float *)(x))
+  #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
+  #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
+  #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }
+  #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
+// #endif
+
+typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
+#define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
+
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
+
+typedef int32_t VarDataOffsetT;
+typedef int16_t VarDataLenT;  // maxVarDataLen: 32767
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_TYPES_H_*/
\ No newline at end of file

source/common/src/tcompare.c

@@ -12,334 +12,8 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _BSD_SOURCE
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _DEFAULT_SOURCE
 
-#include "os.h"
 #include "tcompare.h"
-#include "ulog.h"
-#include "thash.h"
-#include "regex.h"
-#include "ttypes.h"
-
-int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
-}
-
-int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
-}
-
-int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
-}
-
-int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
-  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
-}
-
-int32_t compareInt8Val(const void *pLeft, const void *pRight) {
-  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
-  return compareInt8Val(pRight, pLeft);
-}
-
-int32_t compareInt16Val(const void *pLeft, const void *pRight) {
-  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt16Val(pRight, pLeft);
-}
-
-int32_t compareInt32Val(const void *pLeft, const void *pRight) {
-  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt32Val(pRight, pLeft);
-}
-
-int32_t compareInt64Val(const void *pLeft, const void *pRight) {
-  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
-  return compareInt64Val(pRight, pLeft);
-}
-
-int32_t compareUint32Val(const void *pLeft, const void *pRight) {
-  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint32Val(pRight, pLeft);
-}
-
-int32_t compareUint64Val(const void *pLeft, const void *pRight) {
-  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint64Val(pRight, pLeft);
-}
-
-int32_t compareUint16Val(const void *pLeft, const void *pRight) {
-  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint16Val(pRight, pLeft);
-}
-
-int32_t compareUint8Val(const void* pLeft, const void* pRight) {
-  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
-  if (left > right) return 1;
-  if (left < right) return -1;
-  return 0;
-}
-
-int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
-  return compareUint8Val(pRight, pLeft);
-}
-
-int32_t compareFloatVal(const void *pLeft, const void *pRight) {
-  float p1 = GET_FLOAT_VAL(pLeft);
-  float p2 = GET_FLOAT_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
-  return compareFloatVal(pRight, pLeft);
-}
-
-int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
-  double p1 = GET_DOUBLE_VAL(pLeft);
-  double p2 = GET_DOUBLE_VAL(pRight);
-
-  if (isnan(p1) && isnan(p2)) {
-    return 0;
-  }
-
-  if (isnan(p1)) {
-    return -1;
-  }
-
-  if (isnan(p2)) {
-    return 1;
-  }
-  if (FLT_EQUAL(p1, p2)) {
-    return 0;
-  }
-  return FLT_GREATER(p1, p2) ? 1: -1;
-}
-
-int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
-  return compareDoubleVal(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1:-1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedStr(pRight, pLeft);
-}
-
-int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
-  int32_t len1 = varDataLen(pLeft);
-  int32_t len2 = varDataLen(pRight);
-
-  if (len1 != len2) {
-    return len1 > len2? 1:-1;
-  } else {
-    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
-    if (ret == 0) {
-      return 0;
-    } else {
-      return ret > 0 ? 1 : -1;
-    }
-  }
-}
-
-int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
-  return compareLenPrefixedWStr(pRight, pLeft);
-}
-
-/*
- * Compare two strings
- *    TSDB_MATCH:            Match
- *    TSDB_NOMATCH:          No match
- *    TSDB_NOWILDCARDMATCH:  No match in spite of having * or % wildcards.
- * Like matching rules:
- *      '%': Matches zero or more characters
- *      '_': Matches one character
- *
- */
-int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
-  char c, c1;
-
-  int32_t i = 0;
-  int32_t j = 0;
-  int32_t o = 0;
-  int32_t m = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == pInfo->matchAll) { /* Match "*" */
-
-      while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
-        if (c == pInfo->matchOne) {
-          if (j > size || str[j++] == 0) {
-            // empty string, return not match
-            return TSDB_PATTERN_NOWILDCARDMATCH;
-          } else {
-            ++o;
-          }
-        }
-      }
-
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
-      }
-
-      char next[3] = {toupper(c), tolower(c), 0};
-      m = o;
-      while (1) {
-        size_t n = strcspn(str + m, next);
-        str += m + n;
-
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-        m = 0;
-      }
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-    ++o; 
-    
-    if (j <= size) {
-      if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; }
-      if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
-
-int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) {
-  wchar_t c, c1;
-  wchar_t matchOne = L'_';  // "_"
-  wchar_t matchAll = L'%';  // "%"
-
-  int32_t i = 0;
-  int32_t j = 0;
-
-  while ((c = patterStr[i++]) != 0) {
-    if (c == matchAll) { /* Match "%" */
-
-      while ((c = patterStr[i++]) == matchAll || c == matchOne) {
-        if (c == matchOne && (j >= size || str[j++] == 0)) {
-          return TSDB_PATTERN_NOWILDCARDMATCH;
-        }
-      }
-      if (c == 0) {
-        return TSDB_PATTERN_MATCH;
-      }
-
-      wchar_t accept[3] = {towupper(c), towlower(c), 0};
-      while (1) {
-        size_t n = wcscspn(str, accept);
-
-        str += n;
-        if (str[0] == 0 || (n >= size)) {
-          break;
-        }
-
-        int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
-        if (ret != TSDB_PATTERN_NOMATCH) {
-          return ret;
-        }
-      }
-
-      return TSDB_PATTERN_NOWILDCARDMATCH;
-    }
-
-    c1 = str[j++];
-
-    if (j <= size) {
-      if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
-        continue;
-      }
-    }
-
-    return TSDB_PATTERN_NOMATCH;
-  }
-  
-  return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
-}
 
 int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
@@ -359,62 +33,6 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
-int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight);
-}
-
-int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) {
-  return compareStrRegexComp(pLeft, pRight) ? 0 : 1;
-}
-
-int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
-  size_t sz = varDataLen(pRight);
-  char *pattern = malloc(sz + 1);
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-  pattern[sz] = 0;
-
-  sz = varDataLen(pLeft);
-  char *str = malloc(sz + 1);
-  memcpy(str, varDataVal(pLeft), sz);
-  str[sz] = 0;
-
-  int errCode = 0;
-  regex_t regex;
-  char    msgbuf[256] = {0};
-
-  int cflags = REG_EXTENDED;
-  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
-    regfree(&regex);
-    free(str);
-    free(pattern);
-    return 1;
-  }
-
-  errCode = regexec(&regex, str, 0, NULL, 0);
-  if (errCode != 0 && errCode != REG_NOMATCH) {
-    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
-    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
-  }
-  int32_t result = (errCode == 0) ? 0 : 1;
-  regfree(&regex);
-  free(str);
-  free(pattern);
-  return result;
-}
-
-int32_t taosArrayCompareString(const void* a, const void* b) {
-  const char* x = *(const char**)a;
-  const char* y = *(const char**)b;
-
-  return compareLenPrefixedStr(x, y);
-}
-
-int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
-}
-
 int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
@@ -600,4 +218,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       }
     }
   }
-}
+}
\ No newline at end of file

source/common/src/tglobal.c

@@ -60,6 +60,7 @@ float   tsRatioOfQueryCores = 1.0f;
 int8_t  tsDaylight       = 0;
 int8_t  tsEnableCoreFile = 0;
 int32_t tsMaxBinaryDisplayWidth = 30;
+int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB
 
 /*
  * denote if the server needs to compress response message at the application layer to client, including query rsp,

source/common/src/trow.c

@@ -11,4 +11,79 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
+ */
+
+#include "trow.h"
+
+/* ------------ Structures ---------- */
+struct SRowBatch {
+  int32_t compress : 1;  // if batch row is compressed
+  int32_t nrows : 31;    // number of rows
+  int32_t tlen;          // total length (including `nrows` and `tlen`)
+  char    rows[];
+};
+
+struct SRowBuilder {
+  // TODO
+};
+
+struct SRowBatchIter {
+  int32_t    counter;  // row counter
+  SRowBatch *rb;       // row batch to iter
+  SRow       nrow;     // next row
+};
+
+struct SRowBatchBuilder {
+  // TODO
+};
+
+/* ------------ Methods ---------- */
+
+// SRowBuilder
+SRowBuilder *rowBuilderCreate() {
+  SRowBuilder *pRowBuilder = NULL;
+  // TODO
+
+  return pRowBuilder;
+}
+
+void rowBuilderDestroy(SRowBuilder *pRowBuilder) {
+  if (pRowBuilder) {
+    free(pRowBuilder);
+  }
+}
+
+// SRowBatchIter
+SRowBatchIter *rowBatchIterCreate(SRowBatch *pRowBatch) {
+  SRowBatchIter *pRowBatchIter = (SRowBatchIter *)malloc(sizeof(*pRowBatchIter));
+  if (pRowBatchIter == NULL) {
+    return NULL;
+  }
+
+  pRowBatchIter->counter = 0;
+  pRowBatchIter->rb = pRowBatch;
+  pRowBatchIter->nrow = pRowBatch->rows;
+
+  return pRowBatchIter;
+};
+
+void rowBatchIterDestroy(SRowBatchIter *pRowBatchIter) {
+  if (pRowBatchIter) {
+    free(pRowBatchIter);
+  }
+}
+
+const SRow rowBatchIterNext(SRowBatchIter *pRowBatchIter) {
+  SRow r = NULL;
+  if (pRowBatchIter->counter < pRowBatchIter->rb->nrows) {
+    r = pRowBatchIter->nrow;
+    pRowBatchIter->counter += 1;
+    pRowBatchIter->nrow = (SRow)POINTER_SHIFT(r, rowLen(r));
+  }
+
+  return r;
+}
+
+// SRowBatchBuilder
+SRowBatchBuilder *rowBatchBuilderCreate();
+void              rowBatchBuilderDestroy(SRowBatchBuilder *);
\ No newline at end of file

source/server/CMakeLists.txt

@@ -8,4 +8,5 @@ add_executable(taosd ${TAOSD_SRC})
 target_link_libraries(
     taosd
     PUBLIC dnode
+    PUBLIC util
 )
\ No newline at end of file

source/server/dnode/inc/dnodeCfg.h

@@ -21,21 +21,14 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnCfg {
-  int32_t         dnodeId;
-  int32_t         dropped;
-  char            clusterId[TSDB_CLUSTER_ID_LEN];
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnCfg;
 
-int32_t dnodeInitCfg(SDnCfg **cfg);
-void    dnodeCleanupCfg(SDnCfg **cfg);
-void    dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data);
-int32_t dnodeGetDnodeId(SDnCfg *cfg);
-void    dnodeGetClusterId(SDnCfg *cfg, char *clusterId);
-void    dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId);
-void    dnodeSetDropped(SDnCfg *cfg);
+int32_t dnodeInitCfg();
+void    dnodeCleanupCfg();
+void    dnodeUpdateCfg(SDnodeCfg *data);
+int32_t dnodeGetDnodeId();
+void    dnodeGetClusterId(char *clusterId);
+void    dnodeGetCfg(int32_t *dnodeId, char *clusterId);
+void    dnodeSetDropped();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeCheck.h

@@ -21,11 +21,9 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnCheck {
-} SDnCheck;
 
-int32_t dnodeInitCheck(SDnCheck **check);
-void    dnodeCleanupCheck(SDnCheck **check);
+int32_t dnodeInitCheck();
+void    dnodeCleanupCheck();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeEps.h

@@ -19,22 +19,12 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "thash.h"
 #include "dnodeInt.h"
 
-typedef struct SDnEps {
-  int32_t         dnodeId;
-  int32_t         dnodeNum;
-  SDnodeEp *      dnodeList;
-  SHashObj *      dnodeHash;
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnEps;
-
-int32_t dnodeInitEps(SDnEps **eps);
-void    dnodeCleanupEps(SDnEps **eps);
-void    dnodeUpdateEps(SDnEps *eps, SDnodeEps *data);
-bool    dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr);
+int32_t dnodeInitEps();
+void    dnodeCleanupEps();
+void    dnodeUpdateEps(SDnodeEps *data);
+bool    dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr);
 void    dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeInt.h

@@ -19,36 +19,12 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "taoserror.h"
 #include "taosmsg.h"
-#include "tglobal.h"
 #include "tlog.h"
 #include "trpc.h"
-#include "tstep.h"
 #include "dnode.h"
 
-struct SDnCfg;
-struct SDnCheck;
-struct SDnEps;
-struct SDnMnEps;
-struct SDnStatus;
-struct SDnTelem;
-struct SDnTrans;
-struct SDnMain;
-
-typedef struct SDnode {
-  struct SSteps*    steps;
-  struct SDnCfg*    cfg;
-  struct SDnCheck*  check;
-  struct SDnEps*    eps;
-  struct SDnMnEps*  meps;
-  struct SDnStatus* status;
-  struct SDnTelem*  telem;
-  struct SDnTrans*  trans;
-  struct SDnMain*   main;
-} SDnode;
-
-SDnode* dnodeInst();
+extern int32_t dDebugFlag;
 
 #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", 255, __VA_ARGS__); }}
 #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", 255, __VA_ARGS__); }}

source/server/dnode/inc/dnodeMain.h

@@ -27,14 +27,8 @@ typedef enum {
   TD_RUN_STAT_STOPPED
 } RunStat;
 
-typedef struct SDnMain {
-  RunStat      runStatus;
-  void *       dnodeTimer;
-  SStartupStep startup;
-} SDnMain;
-
-int32_t dnodeInitMain(SDnMain **main);
-void    dnodeCleanupMain(SDnMain **main);
+int32_t dnodeInitMain();
+void    dnodeCleanupMain();
 int32_t dnodeInitStorage();
 void    dnodeCleanupStorage();
 void    dnodeReportStartup(char *name, char *desc);
@@ -42,6 +36,9 @@ void    dnodeReportStartupFinished(char *name, char *desc);
 void    dnodeProcessStartupReq(SRpcMsg *pMsg);
 void    dnodeProcessCreateMnodeReq(SRpcMsg *pMsg);
 void    dnodeProcessConfigDnodeReq(SRpcMsg *pMsg);
+RunStat dnodeGetRunStat();
+void    dnodeSetRunStat();
+void*   dnodeGetTimer();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeMnodeEps.h

@@ -21,19 +21,12 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnMnEps {
-  SRpcEpSet       mnodeEpSet;
-  SMInfos         mnodeInfos;
-  char            file[PATH_MAX + 20];
-  pthread_mutex_t mutex;
-} SDnMnEps;
-
-int32_t dnodeInitMnodeEps(SDnMnEps **meps);
-void    dnodeCleanupMnodeEps(SDnMnEps **meps);
-void    dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *pMinfos);
-void    dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *pEpSet);
-void    dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet);
+int32_t dnodeInitMnodeEps();
+void    dnodeCleanupMnodeEps();
+void    dnodeUpdateMnodeFromStatus(SMInfos *pMinfos);
+void    dnodeUpdateMnodeFromPeer(SRpcEpSet *pEpSet);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
 void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeStatus.h

@@ -21,14 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef struct SDnStatus {
-  void *   dnodeTimer;
-  void *   statusTimer;
-  uint32_t rebootTime;
-} SDnStatus;
-
-int32_t dnodeInitStatus(SDnStatus **status);
-void    dnodeCleanupStatus(SDnStatus **status);
+int32_t dnodeInitStatus();
+void    dnodeCleanupStatus();
 void    dnodeProcessStatusRsp(SRpcMsg *pMsg);
 
 #ifdef __cplusplus

source/server/dnode/inc/dnodeTelem.h

@@ -21,21 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-/*
- * sem_timedwait is NOT implemented on MacOSX
- * thus we use pthread_mutex_t/pthread_cond_t to simulate
- */
-typedef struct SDnTelem {
-  bool             enable;
-  pthread_mutex_t  lock;
-  pthread_cond_t   cond;
-  volatile int32_t exit;
-  pthread_t        thread;
-  char             email[TSDB_FQDN_LEN];
-} SDnTelem;
-
-int32_t dnodeInitTelem(SDnTelem **telem);
-void    dnodeCleanupTelem(SDnTelem **telem);
+int32_t dnodeInitTelem();
+void    dnodeCleanupTelem();
 
 #ifdef __cplusplus
 }

source/server/dnode/inc/dnodeTrans.h

@@ -21,20 +21,8 @@ extern "C" {
 #endif
 #include "dnodeInt.h"
 
-typedef void (*RpcMsgFp)( SRpcMsg *pMsg);
-
-typedef struct SDnTrans {
-  void *   serverRpc;
-  void *   clientRpc;
-  void *   shellRpc;
-  int32_t  queryReqNum;
-  int32_t  submitReqNum;
-  RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX];
-  RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX];
-} SDnTrans;
-
-int32_t dnodeInitTrans(SDnTrans **rans);
-void    dnodeCleanupTrans(SDnTrans **trans);
+int32_t dnodeInitTrans();
+void    dnodeCleanupTrans();
 void    dnodeSendMsgToMnode(SRpcMsg *rpcMsg);
 void    dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);
 void    dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet);

source/server/dnode/src/dnodeCfg.c

@@ -16,56 +16,65 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
+#include "tglobal.h"
 #include "dnodeCfg.h"
 
-static int32_t dnodeReadCfg(SDnCfg *cfg) {
+static struct DnCfg {
+  int32_t         dnodeId;
+  int32_t         dropped;
+  char            clusterId[TSDB_CLUSTER_ID_LEN];
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDcfg;
+
+static int32_t dnodeReadCfg() {
   int32_t len = 0;
   int32_t maxLen = 200;
   char *  content = calloc(1, maxLen + 1);
   cJSON * root = NULL;
   FILE *  fp = NULL;
 
-  fp = fopen(cfg->file, "r");
+  fp = fopen(tsDcfg.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", cfg->file);
+    dDebug("file %s not exist", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", cfg->file);
+    dError("failed to read %s since content is null", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", cfg->file);
+    dError("failed to read %s since invalid json format", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
 
   cJSON *dnodeId = cJSON_GetObjectItem(root, "dnodeId");
   if (!dnodeId || dnodeId->type != cJSON_Number) {
-    dError("failed to read %s since dnodeId not found", cfg->file);
+    dError("failed to read %s since dnodeId not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  cfg->dnodeId = (int32_t)dnodeId->valueint;
+  tsDcfg.dnodeId = (int32_t)dnodeId->valueint;
 
   cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
   if (!dropped || dropped->type != cJSON_Number) {
-    dError("failed to read %s since dropped not found", cfg->file);
+    dError("failed to read %s since dropped not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  cfg->dropped = (int32_t)dropped->valueint;
+  tsDcfg.dropped = (int32_t)dropped->valueint;
 
   cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
   if (!clusterId || clusterId->type != cJSON_String) {
-    dError("failed to read %s since clusterId not found", cfg->file);
+    dError("failed to read %s since clusterId not found", tsDcfg.file);
     goto PARSE_CFG_OVER;
   }
-  tstrncpy(cfg->clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
+  tstrncpy(tsDcfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
 
-  dInfo("successed to read %s", cfg->file);
+  dInfo("successed to read %s", tsDcfg.file);
 
 PARSE_CFG_OVER:
   if (content != NULL) free(content);
@@ -76,10 +85,10 @@ PARSE_CFG_OVER:
   return 0;
 }
 
-static int32_t dnodeWriteCfg(SDnCfg *cfg) {
-  FILE *fp = fopen(cfg->file, "w");
+static int32_t dnodeWriteCfg() {
+  FILE *fp = fopen(tsDcfg.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", cfg->file, strerror(errno));
+    dError("failed to write %s since %s", tsDcfg.file, strerror(errno));
     return -1;
   }
 
@@ -88,9 +97,9 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", cfg->dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", cfg->dropped);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", cfg->clusterId);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDcfg.dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", tsDcfg.dropped);
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsDcfg.clusterId);
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
@@ -99,27 +108,23 @@ static int32_t dnodeWriteCfg(SDnCfg *cfg) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", cfg->file);
+  dInfo("successed to write %s", tsDcfg.file);
   return 0;
 }
 
-int32_t dnodeInitCfg(SDnCfg **out) {
-  SDnCfg* cfg = calloc(1, sizeof(SDnCfg));
-  if (cfg == NULL) return -1;
-
-  cfg->dnodeId = 0;
-  cfg->dropped = 0;
-  cfg->clusterId[0] = 0;
-  snprintf(cfg->file, sizeof(cfg->file), "%s/dnodeCfg.json", tsDnodeDir);
-  pthread_mutex_init(&cfg->mutex, NULL);
-  *out = cfg;
-
-  int32_t ret = dnodeReadCfg(cfg);
+int32_t dnodeInitCfg() {
+  tsDcfg.dnodeId = 0;
+  tsDcfg.dropped = 0;
+  tsDcfg.clusterId[0] = 0;
+  snprintf(tsDcfg.file, sizeof(tsDcfg.file), "%s/dnodeCfg.json", tsDnodeDir);
+  pthread_mutex_init(&tsDcfg.mutex, NULL);
+  
+  int32_t ret = dnodeReadCfg();
   if (ret == 0) {
     dInfo("dnode cfg is initialized");
   }
 
-  if (cfg->dropped) {
+  if (tsDcfg.dropped) {
     dInfo("dnode is dropped and start to exit");
     return -1;
   }
@@ -127,51 +132,47 @@ int32_t dnodeInitCfg(SDnCfg **out) {
   return ret;
 }
 
-void dnodeCleanupCfg(SDnCfg **out) {
-  SDnCfg* cfg = *out;
-  *out = NULL;
-
-  pthread_mutex_destroy(&cfg->mutex);
-  free(cfg);
+void dnodeCleanupCfg() {
+  pthread_mutex_destroy(&tsDcfg.mutex);
 }
 
-void dnodeUpdateCfg(SDnCfg *cfg, SDnodeCfg *data) {
-  if (cfg == NULL || cfg->dnodeId == 0) return;
+void dnodeUpdateCfg(SDnodeCfg *data) {
+  if (tsDcfg.dnodeId != 0) return;
 
-  pthread_mutex_lock(&cfg->mutex);
+  pthread_mutex_lock(&tsDcfg.mutex);
 
-  cfg->dnodeId = data->dnodeId;
-  tstrncpy(cfg->clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN);
-  dInfo("dnodeId is set to %d, clusterId is set to %s", cfg->dnodeId, cfg->clusterId);
+  tsDcfg.dnodeId = data->dnodeId;
+  tstrncpy(tsDcfg.clusterId, data->clusterId, TSDB_CLUSTER_ID_LEN);
+  dInfo("dnodeId is set to %d, clusterId is set to %s", data->dnodeId, data->clusterId);
 
-  dnodeWriteCfg(cfg);
-  pthread_mutex_unlock(&cfg->mutex);
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-void dnodeSetDropped(SDnCfg *cfg) {
-  pthread_mutex_lock(&cfg->mutex);
-  cfg->dropped = 1;
-  dnodeWriteCfg(cfg);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeSetDropped() {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  tsDcfg.dropped = 1;
+  dnodeWriteCfg();
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-int32_t dnodeGetDnodeId(SDnCfg *cfg) {
+int32_t dnodeGetDnodeId() {
   int32_t dnodeId = 0;
-  pthread_mutex_lock(&cfg->mutex);
-  dnodeId = cfg->dnodeId;
-  pthread_mutex_unlock(&cfg->mutex);
+  pthread_mutex_lock(&tsDcfg.mutex);
+  dnodeId = tsDcfg.dnodeId;
+  pthread_mutex_unlock(&tsDcfg.mutex);
   return dnodeId;
 }
 
-void dnodeGetClusterId(SDnCfg *cfg, char *clusterId) {
-  pthread_mutex_lock(&cfg->mutex);
-  tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeGetClusterId(char *clusterId) {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }
 
-void dnodeGetCfg(SDnCfg *cfg, int32_t *dnodeId, char *clusterId) {
-  pthread_mutex_lock(&cfg->mutex);
-  *dnodeId = cfg->dnodeId;
-  tstrncpy(clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN);
-  pthread_mutex_unlock(&cfg->mutex);
+void dnodeGetCfg(int32_t *dnodeId, char *clusterId) {
+  pthread_mutex_lock(&tsDcfg.mutex);
+  *dnodeId = tsDcfg.dnodeId;
+  tstrncpy(clusterId, tsDcfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  pthread_mutex_unlock(&tsDcfg.mutex);
 }

source/server/dnode/src/dnodeCheck.c

@@ -118,7 +118,7 @@ static int32_t dnodeCheckMem() {
 }
 
 static int32_t dnodeCheckDisk() {
-#if 0  
+#if 0
   taosGetDisk();
 
   if (tsAvailDataDirGB < tsMinimalDataDirGB) {
@@ -145,12 +145,7 @@ static int32_t dnodeCheckAccess() { return 0; }
 static int32_t dnodeCheckVersion() { return 0; }
 static int32_t dnodeCheckDatafile() { return 0; }
 
-int32_t dnodeInitCheck(SDnCheck **out) {
-  SDnCheck *check = calloc(1, sizeof(SDnCheck));
-  if (check == NULL) return -1;
-
-  *out = check;
-
+int32_t dnodeInitCheck() {
   if (dnodeCheckNetwork() != 0) {
     dError("failed to check network");
     return -1;
@@ -195,9 +190,4 @@ int32_t dnodeInitCheck(SDnCheck **out) {
   return 0;
 }
 
-void dnodeCleanupCheck(SDnCheck **out) {
-  SDnCheck *check = *out;
-  *out = NULL;
-
-  free(check);
-}
\ No newline at end of file
+void dnodeCleanupCheck() {}
\ No newline at end of file

source/server/dnode/src/dnodeEps.c

@@ -16,86 +16,96 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
+#include "thash.h"
 #include "tglobal.h"
 #include "dnodeEps.h"
 #include "dnodeCfg.h"
 
-static void dnodePrintEps(SDnEps *eps) {
-  dDebug("print dnodeEp, dnodeNum:%d", eps->dnodeNum);
-  for (int32_t i = 0; i < eps->dnodeNum; i++) {
-    SDnodeEp *ep = &eps->dnodeList[i];
+static struct {
+  int32_t         dnodeId;
+  int32_t         dnodeNum;
+  SDnodeEp *      dnodeList;
+  SHashObj *      dnodeHash;
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDeps;
+
+static void dnodePrintEps() {
+  dDebug("print dnodeEp, dnodeNum:%d", tsDeps.dnodeNum);
+  for (int32_t i = 0; i < tsDeps.dnodeNum; i++) {
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
     dDebug("dnode:%d, dnodeFqdn:%s dnodePort:%u", ep->dnodeId, ep->dnodeFqdn, ep->dnodePort);
   }
 }
 
-static void dnodeResetEps(SDnEps *eps, SDnodeEps *data) {
+static void dnodeResetEps(SDnodeEps *data) {
   assert(data != NULL);
 
-  if (data->dnodeNum > eps->dnodeNum) {
+  if (data->dnodeNum > tsDeps.dnodeNum) {
     SDnodeEp *tmp = calloc(data->dnodeNum, sizeof(SDnodeEp));
     if (tmp == NULL) return;
 
-    tfree(eps->dnodeList);
-    eps->dnodeList = tmp;
-    eps->dnodeNum = data->dnodeNum;
-    memcpy(eps->dnodeList, data->dnodeEps, eps->dnodeNum * sizeof(SDnodeEp));
-    dnodePrintEps(eps);
+    tfree(tsDeps.dnodeList);
+    tsDeps.dnodeList = tmp;
+    tsDeps.dnodeNum = data->dnodeNum;
+    memcpy(tsDeps.dnodeList, data->dnodeEps, tsDeps.dnodeNum * sizeof(SDnodeEp));
+    dnodePrintEps();
 
-    for (int32_t i = 0; i < eps->dnodeNum; ++i) {
-      SDnodeEp *ep = &eps->dnodeList[i];
-      taosHashPut(eps->dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
+    for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) {
+      SDnodeEp *ep = &tsDeps.dnodeList[i];
+      taosHashPut(tsDeps.dnodeHash, &ep->dnodeId, sizeof(int32_t), ep, sizeof(SDnodeEp));
     }
   }
 }
 
-static int32_t dnodeReadEps(SDnEps *eps) {
+static int32_t dnodeReadEps() {
   int32_t len = 0;
   int32_t maxLen = 30000;
   char *  content = calloc(1, maxLen + 1);
   cJSON * root = NULL;
   FILE *  fp = NULL;
 
-  fp = fopen(eps->file, "r");
+  fp = fopen(tsDeps.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", eps->file);
+    dDebug("file %s not exist", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", eps->file);
+    dError("failed to read %s since content is null", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", eps->file);
+    dError("failed to read %s since invalid json format", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   cJSON *dnodeNum = cJSON_GetObjectItem(root, "dnodeNum");
   if (!dnodeNum || dnodeNum->type != cJSON_Number) {
-    dError("failed to read %s since dnodeNum not found", eps->file);
+    dError("failed to read %s since dnodeNum not found", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   cJSON *dnodeInfos = cJSON_GetObjectItem(root, "dnodeInfos");
   if (!dnodeInfos || dnodeInfos->type != cJSON_Array) {
-    dError("failed to read %s since dnodeInfos not found", eps->file);
+    dError("failed to read %s since dnodeInfos not found", tsDeps.file);
     goto PRASE_EPS_OVER;
   }
 
   int32_t dnodeInfosSize = cJSON_GetArraySize(dnodeInfos);
   if (dnodeInfosSize != dnodeNum->valueint) {
-    dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", eps->file, dnodeInfosSize,
+    dError("failed to read %s since dnodeInfos size:%d not matched dnodeNum:%d", tsDeps.file, dnodeInfosSize,
            (int32_t)dnodeNum->valueint);
     goto PRASE_EPS_OVER;
   }
 
-  eps->dnodeNum = dnodeInfosSize;
-  eps->dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp));
-  if (eps->dnodeList == NULL) {
+  tsDeps.dnodeNum = dnodeInfosSize;
+  tsDeps.dnodeList = calloc(dnodeInfosSize, sizeof(SDnodeEp));
+  if (tsDeps.dnodeList == NULL) {
     dError("failed to calloc dnodeEpList since %s", strerror(errno));
     goto PRASE_EPS_OVER;
   }
@@ -104,40 +114,40 @@ static int32_t dnodeReadEps(SDnEps *eps) {
     cJSON *dnodeInfo = cJSON_GetArrayItem(dnodeInfos, i);
     if (dnodeInfo == NULL) break;
 
-    SDnodeEp *ep = &eps->dnodeList[i];
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
 
     cJSON *dnodeId = cJSON_GetObjectItem(dnodeInfo, "dnodeId");
     if (!dnodeId || dnodeId->type != cJSON_Number) {
-      dError("failed to read %s, dnodeId not found", eps->file);
+      dError("failed to read %s, dnodeId not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     ep->dnodeId = (int32_t)dnodeId->valueint;
 
     cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
     if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
-      dError("failed to read %s, dnodeFqdn not found", eps->file);
+      dError("failed to read %s, dnodeFqdn not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     tstrncpy(ep->dnodeFqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
 
     cJSON *dnodePort = cJSON_GetObjectItem(dnodeInfo, "dnodePort");
     if (!dnodePort || dnodePort->type != cJSON_Number) {
-      dError("failed to read %s, dnodePort not found", eps->file);
+      dError("failed to read %s, dnodePort not found", tsDeps.file);
       goto PRASE_EPS_OVER;
     }
     ep->dnodePort = (uint16_t)dnodePort->valueint;
   }
 
-  dInfo("succcessed to read file %s", eps->file);
-  dnodePrintEps(eps);
+  dInfo("succcessed to read file %s", tsDeps.file);
+  dnodePrintEps();
 
 PRASE_EPS_OVER:
   if (content != NULL) free(content);
   if (root != NULL) cJSON_Delete(root);
   if (fp != NULL) fclose(fp);
 
-  if (dnodeIsDnodeEpChanged(eps, eps->dnodeId, tsLocalEp)) {
-    dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", eps->dnodeId, tsLocalEp);
+  if (dnodeIsDnodeEpChanged(tsDeps.dnodeId, tsLocalEp)) {
+    dError("dnode:%d, localEp different from %s dnodeEps.json and need reconfigured", tsDeps.dnodeId, tsLocalEp);
     return -1;
   }
 
@@ -145,10 +155,10 @@ PRASE_EPS_OVER:
   return 0;
 }
 
-static int32_t dnodeWriteEps(SDnEps *eps) {
-  FILE *fp = fopen(eps->file, "w");
+static int32_t dnodeWriteEps() {
+  FILE *fp = fopen(tsDeps.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", eps->file, strerror(errno));
+    dError("failed to write %s since %s", tsDeps.file, strerror(errno));
     return -1;
   }
 
@@ -157,14 +167,14 @@ static int32_t dnodeWriteEps(SDnEps *eps) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", eps->dnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeNum\": %d,\n", tsDeps.dnodeNum);
   len += snprintf(content + len, maxLen - len, "  \"dnodeInfos\": [{\n");
-  for (int32_t i = 0; i < eps->dnodeNum; ++i) {
-    SDnodeEp *ep = &eps->dnodeList[i];
+  for (int32_t i = 0; i < tsDeps.dnodeNum; ++i) {
+    SDnodeEp *ep = &tsDeps.dnodeList[i];
     len += snprintf(content + len, maxLen - len, "    \"dnodeId\": %d,\n", ep->dnodeId);
     len += snprintf(content + len, maxLen - len, "    \"dnodeFqdn\": \"%s\",\n", ep->dnodeFqdn);
     len += snprintf(content + len, maxLen - len, "    \"dnodePort\": %u\n", ep->dnodePort);
-    if (i < eps->dnodeNum - 1) {
+    if (i < tsDeps.dnodeNum - 1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");
@@ -178,24 +188,20 @@ static int32_t dnodeWriteEps(SDnEps *eps) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", eps->file);
+  dInfo("successed to write %s", tsDeps.file);
   return 0;
 }
 
-int32_t dnodeInitEps(SDnEps **out) {
-  SDnEps *eps = calloc(1, sizeof(SDnEps));
-  if (eps == NULL) return -1;
+int32_t dnodeInitEps() {
+  tsDeps.dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
+  if (tsDeps.dnodeHash == NULL) return -1;
 
-  eps->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-  if (eps->dnodeHash == NULL) return -1;
+  tsDeps.dnodeId = dnodeGetDnodeId();
+  tsDeps.dnodeNum = 0;
+  snprintf(tsDeps.file, sizeof(tsDeps.file), "%s/dnodeEps.json", tsDnodeDir);
+  pthread_mutex_init(&tsDeps.mutex, NULL);
 
-  eps->dnodeId = dnodeInst()->cfg->dnodeId;
-  eps->dnodeNum = 0;
-  snprintf(eps->file, sizeof(eps->file), "%s/dnodeEps.json", tsDnodeDir);
-  pthread_mutex_init(&eps->mutex, NULL);
-  *out = eps;
-
-  int32_t ret = dnodeReadEps(eps);
+  int32_t ret = dnodeReadEps();
   if (ret == 0) {
     dInfo("dnode eps is initialized");
   }
@@ -203,29 +209,25 @@ int32_t dnodeInitEps(SDnEps **out) {
   return ret;
 }
 
-void dnodeCleanupEps(SDnEps **out) {
-  SDnEps *eps = *out;
-  *out = NULL;
-
-  pthread_mutex_lock(&eps->mutex);
+void dnodeCleanupEps() {
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  if (eps->dnodeList != NULL) {
-    free(eps->dnodeList);
-    eps->dnodeList = NULL;
+  if (tsDeps.dnodeList != NULL) {
+    free(tsDeps.dnodeList);
+    tsDeps.dnodeList = NULL;
   }
 
-  if (eps->dnodeHash) {
-    taosHashCleanup(eps->dnodeHash);
-    eps->dnodeHash = NULL;
+  if (tsDeps.dnodeHash) {
+    taosHashCleanup(tsDeps.dnodeHash);
+    tsDeps.dnodeHash = NULL;
   }
 
-  eps->dnodeNum = 0;
-  pthread_mutex_unlock(&eps->mutex);
-  pthread_mutex_destroy(&eps->mutex);
-  free(eps);
+  tsDeps.dnodeNum = 0;
+  pthread_mutex_unlock(&tsDeps.mutex);
+  pthread_mutex_destroy(&tsDeps.mutex);
 }
 
-void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) {
+void dnodeUpdateEps(SDnodeEps *data) {
   if (data == NULL || data->dnodeNum <= 0) return;
 
   data->dnodeNum = htonl(data->dnodeNum);
@@ -234,28 +236,28 @@ void dnodeUpdateEps(SDnEps *eps, SDnodeEps *data) {
     data->dnodeEps[i].dnodePort = htons(data->dnodeEps[i].dnodePort);
   }
 
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  if (data->dnodeNum != eps->dnodeNum) {
-    dnodeResetEps(eps, data);
-    dnodeWriteEps(eps);
+  if (data->dnodeNum != tsDeps.dnodeNum) {
+    dnodeResetEps(data);
+    dnodeWriteEps();
   } else {
     int32_t size = data->dnodeNum * sizeof(SDnodeEp);
-    if (memcmp(eps->dnodeList, data->dnodeEps, size) != 0) {
-      dnodeResetEps(eps, data);
-      dnodeWriteEps(eps);
+    if (memcmp(tsDeps.dnodeList, data->dnodeEps, size) != 0) {
+      dnodeResetEps(data);
+      dnodeWriteEps();
     }
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 }
 
-bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) {
+bool dnodeIsDnodeEpChanged(int32_t dnodeId, char *epstr) {
   bool changed = false;
 
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t));
+  SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t));
   if (ep != NULL) {
     char epSaved[TSDB_EP_LEN + 1];
     snprintf(epSaved, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
@@ -263,21 +265,20 @@ bool dnodeIsDnodeEpChanged(SDnEps *eps, int32_t dnodeId, char *epstr) {
     tstrncpy(epstr, epSaved, TSDB_EP_LEN);
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 
   return changed;
 }
 
 void dnodeGetDnodeEp(int32_t dnodeId, char *epstr, char *fqdn, uint16_t *port) {
-  SDnEps *eps = dnodeInst()->eps;
-  pthread_mutex_lock(&eps->mutex);
+  pthread_mutex_lock(&tsDeps.mutex);
 
-  SDnodeEp *ep = taosHashGet(eps->dnodeHash, &dnodeId, sizeof(int32_t));
+  SDnodeEp *ep = taosHashGet(tsDeps.dnodeHash, &dnodeId, sizeof(int32_t));
   if (ep != NULL) {
     if (port) *port = ep->dnodePort;
     if (fqdn) tstrncpy(fqdn, ep->dnodeFqdn, TSDB_FQDN_LEN);
     if (epstr) snprintf(epstr, TSDB_EP_LEN, "%s:%u", ep->dnodeFqdn, ep->dnodePort);
   }
 
-  pthread_mutex_unlock(&eps->mutex);
+  pthread_mutex_unlock(&tsDeps.mutex);
 }

source/server/dnode/src/dnodeInt.c

@@ -33,10 +33,7 @@
 #include "mnode.h"
 #include "vnode.h"
 
-SDnode *dnodeInst() {
-  static SDnode inst = {0};
-  return &inst;
-}
+static struct SSteps *tsSteps;
 
 static int32_t dnodeInitVnodeModule(void **unused) {
   SVnodePara para;
@@ -48,58 +45,50 @@ static int32_t dnodeInitVnodeModule(void **unused) {
 }
 
 static int32_t dnodeInitMnodeModule(void **unused) {
-  SDnode *dnode = dnodeInst();
-
   SMnodePara para;
   para.fp.GetDnodeEp = dnodeGetDnodeEp;
   para.fp.SendMsgToDnode = dnodeSendMsgToDnode;
   para.fp.SendMsgToMnode = dnodeSendMsgToMnode;
   para.fp.SendRedirectMsg = dnodeSendRedirectMsg;
-  para.dnodeId = dnode->cfg->dnodeId;
-  strncpy(para.clusterId, dnode->cfg->clusterId, sizeof(para.clusterId));
+  dnodeGetCfg(&para.dnodeId, para.clusterId);
 
   return mnodeInit(para);
 }
 
 int32_t dnodeInit() {
-  struct SSteps *steps = taosStepInit(24, dnodeReportStartup);
-  if (steps == NULL) return -1;
+  tsSteps = taosStepInit(24, dnodeReportStartup);
+  if (tsSteps == NULL) return -1;
 
-  SDnode *dnode = dnodeInst();
+  taosStepAdd(tsSteps, "dnode-main", dnodeInitMain, dnodeCleanupMain);
+  taosStepAdd(tsSteps, "dnode-storage", dnodeInitStorage, dnodeCleanupStorage);
+  //taosStepAdd(tsSteps, "dnode-tfs", tfInit, tfCleanup);
+  taosStepAdd(tsSteps, "dnode-rpc", rpcInit, rpcCleanup);
+  taosStepAdd(tsSteps, "dnode-check", dnodeInitCheck, dnodeCleanupCheck);
+  taosStepAdd(tsSteps, "dnode-cfg", dnodeInitCfg, dnodeCleanupCfg);
+  taosStepAdd(tsSteps, "dnode-deps", dnodeInitEps, dnodeCleanupEps);
+  taosStepAdd(tsSteps, "dnode-meps", dnodeInitMnodeEps, dnodeCleanupMnodeEps);
+  //taosStepAdd(tsSteps, "dnode-wal", walInit, walCleanUp);
+  //taosStepAdd(tsSteps, "dnode-sync", syncInit, syncCleanUp);
+  taosStepAdd(tsSteps, "dnode-vnode", dnodeInitVnodeModule, vnodeCleanup);
+  taosStepAdd(tsSteps, "dnode-mnode", dnodeInitMnodeModule, mnodeCleanup);
+  taosStepAdd(tsSteps, "dnode-trans", dnodeInitTrans, dnodeCleanupTrans);
+  taosStepAdd(tsSteps, "dnode-status", dnodeInitStatus, dnodeCleanupStatus);
+  taosStepAdd(tsSteps, "dnode-telem", dnodeInitTelem, dnodeCleanupTelem);
+  //taosStepAdd(tsSteps, "dnode-script",scriptEnvPoolInit, scriptEnvPoolCleanup);
 
-  taosStepAdd(steps, "dnode-main", (void **)&dnode->main, (InitFp)dnodeInitMain, (CleanupFp)dnodeCleanupMain);
-  taosStepAdd(steps, "dnode-storage", NULL, (InitFp)dnodeInitStorage, (CleanupFp)dnodeCleanupStorage);
-  //taosStepAdd(steps, "dnode-tfs", NULL, (InitFp)tfInit, (CleanupFp)tfCleanup);
-  taosStepAdd(steps, "dnode-rpc", NULL, (InitFp)rpcInit, (CleanupFp)rpcCleanup);
-  taosStepAdd(steps, "dnode-check", (void **)&dnode->check, (InitFp)dnodeInitCheck, (CleanupFp)dnodeCleanupCheck);
-  taosStepAdd(steps, "dnode-cfg", (void **)&dnode->cfg, (InitFp)dnodeInitCfg, (CleanupFp)dnodeCleanupCfg);
-  taosStepAdd(steps, "dnode-deps", (void **)&dnode->eps, (InitFp)dnodeInitEps, (CleanupFp)dnodeCleanupEps);
-  taosStepAdd(steps, "dnode-meps", (void **)&dnode->meps, (InitFp)dnodeInitMnodeEps, (CleanupFp)dnodeCleanupMnodeEps);
-  //taosStepAdd(steps, "dnode-wal", NULL, (InitFp)walInit, (CleanupFp)walCleanUp);
-  //taosStepAdd(steps, "dnode-sync", NULL, (InitFp)syncInit, (CleanupFp)syncCleanUp);
-  taosStepAdd(steps, "dnode-vnode", NULL, (InitFp)dnodeInitVnodeModule, (CleanupFp)vnodeCleanup);
-  taosStepAdd(steps, "dnode-mnode", NULL, (InitFp)dnodeInitMnodeModule, (CleanupFp)mnodeCleanup);
-  taosStepAdd(steps, "dnode-trans", (void **)&dnode->trans, (InitFp)dnodeInitTrans, (CleanupFp)dnodeCleanupTrans);
-  taosStepAdd(steps, "dnode-status", (void **)&dnode->status, (InitFp)dnodeInitStatus, (CleanupFp)dnodeCleanupStatus);
-  taosStepAdd(steps, "dnode-telem", (void **)&dnode->telem, (InitFp)dnodeInitTelem, (CleanupFp)dnodeCleanupTelem);
-  //taosStepAdd(steps, "dnode-script", NULL, (InitFp)scriptEnvPoolInit, (CleanupFp)scriptEnvPoolCleanup);
+  taosStepExec(tsSteps);
 
-  dnode->steps = steps;
-  taosStepExec(dnode->steps);
-
-  if (dnode->main) {
-    dnode->main->runStatus = TD_RUN_STAT_RUNNING;
-    dnodeReportStartupFinished("TDengine", "initialized successfully");
-    dInfo("TDengine is initialized successfully");
-  }
+  dnodeSetRunStat(TD_RUN_STAT_RUNNING);
+  dnodeReportStartupFinished("TDengine", "initialized successfully");
+  dInfo("TDengine is initialized successfully");
 
   return 0;
 }
 
 void dnodeCleanup() {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main->runStatus != TD_RUN_STAT_STOPPED) {
-    dnode->main->runStatus = TD_RUN_STAT_STOPPED;
-    taosStepCleanup(dnode->steps);
+  if (dnodeGetRunStat() != TD_RUN_STAT_STOPPED) {
+    dnodeSetRunStat(TD_RUN_STAT_STOPPED);
+    taosStepCleanup(tsSteps);
+    tsSteps = NULL;
   }
 }

source/server/dnode/src/dnodeMain.c

@@ -17,15 +17,22 @@
 #include "os.h"
 #include "tcache.h"
 #include "tconfig.h"
+#include "tglobal.h"
 #if 0
 #include "tfs.h"
 #endif
+#include "tnote.h"
+#include "tcompression.h"
+#include "ttimer.h"
 #include "dnodeCfg.h"
 #include "dnodeMain.h"
 #include "mnode.h"
-#include "tcompression.h"
-#include "tnote.h"
-#include "ttimer.h"
+
+static struct {
+  RunStat      runStatus;
+  void *       dnodeTimer;
+  SStartupStep startup;
+} tsDmain;
 
 static void dnodeCheckDataDirOpenned(char *dir) {
 #if 0
@@ -47,27 +54,14 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 #endif
 }
 
-void dnodePrintDiskInfo() {
-  dInfo("==================================");
-  dInfo(" os totalDisk:           %f(GB)", tsTotalDataDirGB);
-  dInfo(" os usedDisk:            %f(GB)", tsUsedDataDirGB);
-  dInfo(" os availDisk:           %f(GB)", tsAvailDataDirGB);
-  dInfo("==================================");
-}
-
-int32_t dnodeInitMain(SDnMain **out) {
-  SDnMain* main = calloc(1, sizeof(SDnMain));
-  if (main == NULL) return -1;
-
-  main->runStatus = TD_RUN_STAT_STOPPED;
-  main->dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR");
-  if (main->dnodeTimer == NULL) {
+int32_t dnodeInitMain() {
+  tsDmain.runStatus = TD_RUN_STAT_STOPPED;
+  tsDmain.dnodeTimer = taosTmrInit(100, 200, 60000, "DND-TMR");
+  if (tsDmain.dnodeTimer == NULL) {
     dError("failed to init dnode timer");
     return -1;
   }
 
-  *out = main;
-
   tscEmbedded = 1;
   taosIgnSIGPIPE();
   taosBlockSIGPIPE();
@@ -76,7 +70,6 @@ int32_t dnodeInitMain(SDnMain **out) {
   taosReadGlobalLogCfg();
   taosSetCoreDump(tsEnableCoreFile);
 
-
   if (!taosMkDir(tsLogDir)) {
    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
    return -1;
@@ -101,13 +94,10 @@ int32_t dnodeInitMain(SDnMain **out) {
   return taosCheckGlobalCfg();
 }
 
-void dnodeCleanupMain(SDnMain **out) {
-  SDnMain *main = *out;
-  *out = NULL;
-
-  if (main->dnodeTimer != NULL) {
-    taosTmrCleanUp(main->dnodeTimer);
-    main->dnodeTimer = NULL;
+void dnodeCleanupMain() {
+  if (tsDmain.dnodeTimer != NULL) {
+    taosTmrCleanUp(tsDmain.dnodeTimer);
+    tsDmain.dnodeTimer = NULL;
   }
 
 #if 0
@@ -115,8 +105,6 @@ void dnodeCleanupMain(SDnMain **out) {
 #endif
   taosCloseLog();
   taosStopCacheRefreshWorker();
-
-  free(main);
 }
 
 int32_t dnodeInitStorage() {
@@ -138,7 +126,7 @@ int32_t dnodeInitStorage() {
   }
 
   strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
-#endif    
+#endif
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
@@ -164,7 +152,6 @@ int32_t dnodeInitStorage() {
     return -1;
   }
 
-
   TDIR *tdir = tfsOpendir("vnode_bak/.staging");
   bool  stagingNotEmpty = tfsReaddir(tdir) != NULL;
   tfsClosedir(tdir);
@@ -190,7 +177,7 @@ int32_t dnodeInitStorage() {
 }
 
 void dnodeCleanupStorage() {
-#if 0  
+#if 0
   // storage destroy
   tfsDestroy();
 
@@ -202,18 +189,14 @@ void dnodeCleanupStorage() {
 }
 
 void dnodeReportStartup(char *name, char *desc) {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main != NULL) {
-    SStartupStep *startup = &dnode->main->startup;
-    tstrncpy(startup->name, name, strlen(startup->name));
-    tstrncpy(startup->desc, desc, strlen(startup->desc));
-    startup->finished = 0;
-  }
+  SStartupStep *startup = &tsDmain.startup;
+  tstrncpy(startup->name, name, strlen(startup->name));
+  tstrncpy(startup->desc, desc, strlen(startup->desc));
+  startup->finished = 0;
 }
 
 void dnodeReportStartupFinished(char *name, char *desc) {
-  SDnode *dnode = dnodeInst();
-  SStartupStep *startup = &dnode->main->startup;
+  SStartupStep *startup = &tsDmain.startup;
   tstrncpy(startup->name, name, strlen(startup->name));
   tstrncpy(startup->desc, desc, strlen(startup->desc));
   startup->finished = 1;
@@ -222,9 +205,8 @@ void dnodeReportStartupFinished(char *name, char *desc) {
 void dnodeProcessStartupReq(SRpcMsg *pMsg) {
   dInfo("startup msg is received, cont:%s", (char *)pMsg->pCont);
 
-  SDnode *dnode = dnodeInst();
   SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep));
-  memcpy(pStep, &dnode->main->startup, sizeof(SStartupStep));
+  memcpy(pStep, &tsDmain.startup, sizeof(SStartupStep));
 
   dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished);
 
@@ -234,12 +216,11 @@ void dnodeProcessStartupReq(SRpcMsg *pMsg) {
 }
 
 static int32_t dnodeStartMnode(SRpcMsg *pMsg) {
-  SDnode *dnode = dnodeInst();
   SCreateMnodeMsg *pCfg = pMsg->pCont;
   pCfg->dnodeId = htonl(pCfg->dnodeId);
-  if (pCfg->dnodeId != dnode->cfg->dnodeId) {
+  if (pCfg->dnodeId != dnodeGetDnodeId()) {
     dDebug("dnode:%d, in create meps msg is not equal with saved dnodeId:%d", pCfg->dnodeId,
-           dnodeGetDnodeId(dnode->cfg));
+           dnodeGetDnodeId());
     return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED;
   }
 
@@ -277,4 +258,10 @@ void dnodeProcessConfigDnodeReq(SRpcMsg *pMsg) {
 
   rpcSendResponse(&rspMsg);
   rpcFreeCont(pMsg->pCont);
-}
\ No newline at end of file
+}
+
+RunStat dnodeGetRunStat() { return tsDmain.runStatus; }
+
+void dnodeSetRunStat(RunStat stat) { tsDmain.runStatus = stat; }
+
+void* dnodeGetTimer() { return tsDmain.dnodeTimer; }
\ No newline at end of file

source/server/dnode/src/dnodeMnodeEps.c

@@ -22,43 +22,51 @@
 #include "dnodeMnodeEps.h"
 #include "mnode.h"
 
-static void dnodePrintMnodeEps(SDnMnEps *meps) {
-  SRpcEpSet *epset = &meps->mnodeEpSet;
+static struct {
+  SRpcEpSet       mnodeEpSet;
+  SMInfos         mnodeInfos;
+  char            file[PATH_MAX + 20];
+  pthread_mutex_t mutex;
+} tsDmeps;
+
+
+static void dnodePrintMnodeEps() {
+  SRpcEpSet *epset = &tsDmeps.mnodeEpSet;
   dInfo("print mnode eps, num:%d inuse:%d", epset->numOfEps, epset->inUse);
   for (int32_t i = 0; i < epset->numOfEps; i++) {
     dInfo("ep index:%d, %s:%u", i, epset->fqdn[i], epset->port[i]);
   }
 }
 
-static void dnodeResetMnodeEps(SDnMnEps *meps, SMInfos *mInfos) {
+static void dnodeResetMnodeEps(SMInfos *mInfos) {
   if (mInfos == NULL || mInfos->mnodeNum == 0) {
-    meps->mnodeEpSet.numOfEps = 1;
-    taosGetFqdnPortFromEp(tsFirst, meps->mnodeEpSet.fqdn[0], &meps->mnodeEpSet.port[0]);
+    tsDmeps.mnodeEpSet.numOfEps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsDmeps.mnodeEpSet.fqdn[0], &tsDmeps.mnodeEpSet.port[0]);
 
     if (strcmp(tsSecond, tsFirst) != 0) {
-      meps->mnodeEpSet.numOfEps = 2;
-      taosGetFqdnPortFromEp(tsSecond, meps->mnodeEpSet.fqdn[1], &meps->mnodeEpSet.port[1]);
+      tsDmeps.mnodeEpSet.numOfEps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsDmeps.mnodeEpSet.fqdn[1], &tsDmeps.mnodeEpSet.port[1]);
     }
-    dnodePrintMnodeEps(meps);
+    dnodePrintMnodeEps();
     return;
   }
 
-    int32_t size = sizeof(SMInfos);
-  memcpy(&meps->mnodeInfos, mInfos, size);
+  int32_t size = sizeof(SMInfos);
+  memcpy(&tsDmeps.mnodeInfos, mInfos, size);
 
-  meps->mnodeEpSet.inUse = meps->mnodeInfos.inUse;
-  meps->mnodeEpSet.numOfEps = meps->mnodeInfos.mnodeNum;
-  for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) {
-    taosGetFqdnPortFromEp(meps->mnodeInfos.mnodeInfos[i].mnodeEp, meps->mnodeEpSet.fqdn[i], &meps->mnodeEpSet.port[i]);
+  tsDmeps.mnodeEpSet.inUse = tsDmeps.mnodeInfos.inUse;
+  tsDmeps.mnodeEpSet.numOfEps = tsDmeps.mnodeInfos.mnodeNum;
+  for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) {
+    taosGetFqdnPortFromEp(tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp, tsDmeps.mnodeEpSet.fqdn[i], &tsDmeps.mnodeEpSet.port[i]);
   }
 
-  dnodePrintMnodeEps(meps);
+  dnodePrintMnodeEps();
 }
 
-static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
-  FILE *fp = fopen(meps->file, "w");
+static int32_t dnodeWriteMnodeEps() {
+  FILE *fp = fopen(tsDmeps.file, "w");
   if (!fp) {
-    dError("failed to write %s since %s", meps->file, strerror(errno));
+    dError("failed to write %s since %s", tsDmeps.file, strerror(errno));
     return -1;
   }
 
@@ -67,13 +75,13 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", meps->mnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", meps->mnodeInfos.mnodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsDmeps.mnodeInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsDmeps.mnodeInfos.mnodeNum);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < meps->mnodeInfos.mnodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", meps->mnodeInfos.mnodeInfos[i].mnodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", meps->mnodeInfos.mnodeInfos[i].mnodeEp);
-    if (i < meps->mnodeInfos.mnodeNum - 1) {
+  for (int32_t i = 0; i < tsDmeps.mnodeInfos.mnodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsDmeps.mnodeInfos.mnodeInfos[i].mnodeEp);
+    if (i < tsDmeps.mnodeInfos.mnodeNum - 1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");
@@ -87,11 +95,11 @@ static int32_t dnodeWriteMnodeEps(SDnMnEps *meps) {
   free(content);
   terrno = 0;
 
-  dInfo("successed to write %s", meps->file);
+  dInfo("successed to write %s", tsDmeps.file);
   return 0;
 }
 
-static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
+static int32_t dnodeReadMnodeEps() {
   int32_t len = 0;
   int32_t maxLen = 2000;
   char *  content = calloc(1, maxLen + 1);
@@ -100,22 +108,22 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
   SMInfos mInfos = {0};
   bool    nodeChanged = false;
 
-  fp = fopen(meps->file, "r");
+  fp = fopen(tsDmeps.file, "r");
   if (!fp) {
-    dDebug("file %s not exist", meps->file);
+    dDebug("file %s not exist", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
   len = (int32_t)fread(content, 1, maxLen, fp);
   if (len <= 0) {
-    dError("failed to read %s since content is null", meps->file);
+    dError("failed to read %s since content is null", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
   content[len] = 0;
   root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read %s since invalid json format", meps->file);
+    dError("failed to read %s since invalid json format", tsDmeps.file);
     goto PARSE_MINFOS_OVER;
   }
 
@@ -124,7 +132,7 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
     dError("failed to read mnodeEpSet.json since inUse not found");
     goto PARSE_MINFOS_OVER;
   }
-  meps->mnodeInfos.inUse = (int8_t)inUse->valueint;
+  tsDmeps.mnodeInfos.inUse = (int8_t)inUse->valueint;
 
   cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
   if (!nodeNum || nodeNum->type != cJSON_Number) {
@@ -165,11 +173,11 @@ static int32_t dnodeReadMnodeEps(SDnMnEps *meps, SDnEps *deps) {
     mInfo->mnodeId = (int32_t)nodeId->valueint;
     tstrncpy(mInfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
 
-    bool changed = dnodeIsDnodeEpChanged(deps, mInfo->mnodeId, mInfo->mnodeEp);
+    bool changed = dnodeIsDnodeEpChanged(mInfo->mnodeId, mInfo->mnodeEp);
     if (changed) nodeChanged = changed;
   }
 
-  dInfo("successed to read file %s", meps->file);
+  dInfo("successed to read file %s", tsDmeps.file);
 
 PARSE_MINFOS_OVER:
   if (content != NULL) free(content);
@@ -182,25 +190,24 @@ PARSE_MINFOS_OVER:
     dnodeGetDnodeEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL);
   }
 
-  dnodeResetMnodeEps(meps, &mInfos);
+  dnodeResetMnodeEps(&mInfos);
 
   if (nodeChanged) {
-    dnodeWriteMnodeEps(meps);
+    dnodeWriteMnodeEps();
   }
 
   return 0;
 }
 
 void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
-  SDnMnEps *meps = dnodeInst()->meps;
   SRpcConnInfo connInfo = {0};
   rpcGetConnInfo(rpcMsg->handle, &connInfo);
 
   SRpcEpSet epSet = {0};
   if (forShell) {
-    dnodeGetEpSetForShell(meps, &epSet);
+    dnodeGetEpSetForShell(&epSet);
   } else {
-    dnodeGetEpSetForPeer(meps, &epSet);
+    dnodeGetEpSetForPeer(&epSet);
   }
 
   dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
@@ -222,16 +229,12 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
   rpcSendRedirectRsp(rpcMsg->handle, &epSet);
 }
 
-int32_t dnodeInitMnodeEps(SDnMnEps **out) {
-  SDnMnEps *meps = calloc(1, sizeof(SDnMnEps));
-  if (meps == NULL) return -1;
-
-  snprintf(meps->file, sizeof(meps->file), "%s/mnodeEpSet.json", tsDnodeDir);
-  pthread_mutex_init(&meps->mutex, NULL);
-  *out = meps;
+int32_t dnodeInitMnodeEps() {
+  snprintf(tsDmeps.file, sizeof(tsDmeps.file), "%s/mnodeEpSet.json", tsDnodeDir);
+  pthread_mutex_init(&tsDmeps.mutex, NULL);
 
-  dnodeResetMnodeEps(meps, NULL);
-  int32_t ret = dnodeReadMnodeEps(meps, dnodeInst()->eps);
+  dnodeResetMnodeEps(NULL);
+  int32_t ret = dnodeReadMnodeEps();
   if (ret == 0) {
     dInfo("dnode mInfos is initialized");
   }
@@ -239,17 +242,11 @@ int32_t dnodeInitMnodeEps(SDnMnEps **out) {
   return ret;
 }
 
-void dnodeCleanupMnodeEps(SDnMnEps **out) {
-  SDnMnEps *meps = *out;
-  *out = NULL;
-
-  if (meps != NULL) {
-    pthread_mutex_destroy(&meps->mutex);
-    free(meps);
-  }
+void dnodeCleanupMnodeEps() {
+  pthread_mutex_destroy(&tsDmeps.mutex);
 }
 
-void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) {
+void dnodeUpdateMnodeFromStatus(SMInfos *mInfos) {
   if (mInfos->mnodeNum <= 0 || mInfos->mnodeNum > TSDB_MAX_REPLICA) {
     dError("invalid mInfos since num:%d invalid", mInfos->mnodeNum);
     return;
@@ -264,53 +261,51 @@ void dnodeUpdateMnodeFromStatus(SDnMnEps *meps, SMInfos *mInfos) {
     }
   }
 
-  pthread_mutex_lock(&meps->mutex);
-  if (mInfos->mnodeNum != meps->mnodeInfos.mnodeNum) {
-    dnodeResetMnodeEps(meps, mInfos);
-    dnodeWriteMnodeEps(meps);
+  pthread_mutex_lock(&tsDmeps.mutex);
+  if (mInfos->mnodeNum != tsDmeps.mnodeInfos.mnodeNum) {
+    dnodeResetMnodeEps(mInfos);
+    dnodeWriteMnodeEps();
   } else {
     int32_t size = sizeof(SMInfos);
-    if (memcmp(mInfos, &meps->mnodeInfos, size) != 0) {
-      dnodeResetMnodeEps(meps, mInfos);
-      dnodeWriteMnodeEps(meps);
+    if (memcmp(mInfos, &tsDmeps.mnodeInfos, size) != 0) {
+      dnodeResetMnodeEps(mInfos);
+      dnodeWriteMnodeEps();
     }
   }
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeUpdateMnodeFromPeer(SDnMnEps *meps, SRpcEpSet *ep) {
+void dnodeUpdateMnodeFromPeer(SRpcEpSet *ep) {
   if (ep->numOfEps <= 0) {
     dError("mInfos is changed, but content is invalid, discard it");
     return;
   }
 
-  pthread_mutex_lock(&meps->mutex);
+  pthread_mutex_lock(&tsDmeps.mutex);
 
   dInfo("mInfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
   for (int32_t i = 0; i < ep->numOfEps; ++i) {
     ep->port[i] -= TSDB_PORT_DNODEDNODE;
     dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
   }
-  meps->mnodeEpSet = *ep;
+  tsDmeps.mnodeEpSet = *ep;
 
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeGetEpSetForPeer(SDnMnEps *meps, SRpcEpSet *epSet) {
-  pthread_mutex_lock(&meps->mutex);
+void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsDmeps.mutex);
 
-  *epSet = meps->mnodeEpSet;
+  *epSet = tsDmeps.mnodeEpSet;
   for (int32_t i = 0; i < epSet->numOfEps; ++i) {
     epSet->port[i] += TSDB_PORT_DNODEDNODE;
   }
 
-  pthread_mutex_unlock(&meps->mutex);
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }
 
-void dnodeGetEpSetForShell(SDnMnEps *meps, SRpcEpSet *epSet) {
-  pthread_mutex_lock(&meps->mutex);
-
-  *epSet = meps->mnodeEpSet;
-
-  pthread_mutex_unlock(&meps->mutex);
+void dnodeGetEpSetForShell(SRpcEpSet *epSet) {
+  pthread_mutex_lock(&tsDmeps.mutex);
+  *epSet = tsDmeps.mnodeEpSet;
+  pthread_mutex_unlock(&tsDmeps.mutex);
 }

source/server/dnode/src/dnodeStatus.c

@@ -25,15 +25,15 @@
 #include "dnodeMain.h"
 #include "vnode.h"
 
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  SDnStatus *status = handle;
-  if (status->dnodeTimer == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
+static struct {
+  void *   dnodeTimer;
+  void *   statusTimer;
+  uint32_t rebootTime;
+} tsStatus;
 
-  if (status->statusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+static void dnodeSendStatusMsg(void *handle, void *tmrId) {
+  if (tsStatus.statusTimer == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     dError("failed to start status timer");
     return;
   }
@@ -41,16 +41,15 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   int32_t     contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
   SStatusMsg *pStatus = rpcMallocCont(contLen);
   if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     dError("failed to malloc status message");
     return;
   }
 
-  SDnode *dnode = dnodeInst();
-  dnodeGetCfg(dnode->cfg, &pStatus->dnodeId, pStatus->clusterId);
-  pStatus->dnodeId = htonl(dnodeGetDnodeId(dnode->cfg));
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId = htonl(dnodeGetDnodeId());
   pStatus->version = htonl(tsVersion);
-  pStatus->lastReboot = htonl(status->rebootTime);
+  pStatus->lastReboot = htonl(tsStatus.rebootTime);
   pStatus->numOfCores = htons((uint16_t)tsNumOfCores);
   pStatus->diskAvailable = tsAvailDataDirGB;
   pStatus->alternativeRole = tsAlternativeRole;
@@ -80,69 +79,58 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
   pStatus->openVnodes = htons(pStatus->openVnodes);
 
-  SRpcMsg rpcMsg = {.ahandle = status, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS};
+  SRpcMsg rpcMsg = {.ahandle = NULL, .pCont = pStatus, .contLen = contLen, .msgType = TSDB_MSG_TYPE_DM_STATUS};
 
   dnodeSendMsgToMnode(&rpcMsg);
 }
 
 void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  SDnode *dnode = dnodeInst();
-  SDnStatus *status = pMsg->ahandle;
-
   if (pMsg->code != TSDB_CODE_SUCCESS) {
     dError("status rsp is received, error:%s", tstrerror(pMsg->code));
     if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) {
       char clusterId[TSDB_CLUSTER_ID_LEN];
-      dnodeGetClusterId(dnode->cfg, clusterId);
+      dnodeGetClusterId(clusterId);
       if (clusterId[0] != '\0') {
-        dnodeSetDropped(dnode->cfg);
+        dnodeSetDropped();
         dError("exit zombie dropped dnode");
         exit(EXIT_FAILURE);
       }
     }
 
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
     return;
   }
 
   SStatusRsp *pStatusRsp = pMsg->pCont;
   SMInfos *   minfos = &pStatusRsp->mnodes;
-  dnodeUpdateMnodeFromStatus(dnode->meps, minfos);
+  dnodeUpdateMnodeFromStatus(minfos);
 
   SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
   pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
   pCfg->moduleStatus = htonl(pCfg->moduleStatus);
   pCfg->dnodeId = htonl(pCfg->dnodeId);
-  dnodeUpdateCfg(dnode->cfg, pCfg);
+  dnodeUpdateCfg(pCfg);
 
   vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
 
   SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
-  dnodeUpdateEps(dnode->eps, pEps);
+  dnodeUpdateEps(pEps);
 
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, status, status->dnodeTimer, &status->statusTimer);
+  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
 }
 
-int32_t dnodeInitStatus(SDnStatus **out) {
-  SDnStatus *status = calloc(1, sizeof(SDnStatus));
-  if (status == NULL) return -1;
-  status->statusTimer = NULL;
-  status->dnodeTimer = dnodeInst()->main->dnodeTimer;
-  status->rebootTime = taosGetTimestampSec();
-  taosTmrReset(dnodeSendStatusMsg, 500, status, status->dnodeTimer, &status->statusTimer);
-  *out = status;
+int32_t dnodeInitStatus() {
+  tsStatus.statusTimer = NULL;
+  tsStatus.dnodeTimer = dnodeGetTimer();
+  tsStatus.rebootTime = taosGetTimestampSec();
+  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsStatus.dnodeTimer, &tsStatus.statusTimer);
   dInfo("dnode status timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeCleanupStatus(SDnStatus **out) {
-  SDnStatus *status = *out;
-  *out = NULL;
-
-  if (status->statusTimer != NULL) {
-    taosTmrStopA(&status->statusTimer);
-    status->statusTimer = NULL;
+void dnodeCleanupStatus() {
+  if (tsStatus.statusTimer != NULL) {
+    taosTmrStopA(&tsStatus.statusTimer);
+    tsStatus.statusTimer = NULL;
   }
-
-  free(status);
 }

source/server/dnode/src/dnodeTelem.c

@@ -25,6 +25,19 @@
 #define TELEMETRY_PORT 80
 #define REPORT_INTERVAL 86400
 
+/*
+ * sem_timedwait is NOT implemented on MacOSX
+ * thus we use pthread_mutex_t/pthread_cond_t to simulate
+ */
+static struct {
+  bool             enable;
+  pthread_mutex_t  lock;
+  pthread_cond_t   cond;
+  volatile int32_t exit;
+  pthread_t        thread;
+  char             email[TSDB_FQDN_LEN];
+} tsTelem;
+
 static void dnodeBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); }
 
 static void dnodeCloseObject(SBufferWriter* bw) {
@@ -154,14 +167,14 @@ static void dnodeAddMemoryInfo(SBufferWriter* bw) {
   fclose(fp);
 }
 
-static void dnodeAddVersionInfo(SDnTelem* telem, SBufferWriter* bw) {
+static void dnodeAddVersionInfo(SBufferWriter* bw) {
   dnodeAddStringField(bw, "version", version);
   dnodeAddStringField(bw, "buildInfo", buildinfo);
   dnodeAddStringField(bw, "gitInfo", gitinfo);
-  dnodeAddStringField(bw, "email", telem->email);
+  dnodeAddStringField(bw, "email", tsTelem.email);
 }
 
-static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) {
+static void dnodeAddRuntimeInfo(SBufferWriter* bw) {
   SMnodeStat stat = {0};
   if (mnodeGetStatistics(&stat) != 0) {
     return;
@@ -179,7 +192,7 @@ static void dnodeAddRuntimeInfo(SDnTelem* telem, SBufferWriter* bw) {
   dnodeAddIntField(bw, "compStorage", stat.compStorage);
 }
 
-static void dnodeSendTelemetryReport(SDnTelem* telem) {
+static void dnodeSendTelemetryReport() {
   char     buf[128] = {0};
   uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER);
   if (ip == 0xffffffff) {
@@ -192,16 +205,18 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) {
     return;
   }
 
-  SDnode *dnode = dnodeInst();
+  char clusterId[TSDB_CLUSTER_ID_LEN] = {0};
+  dnodeGetClusterId(clusterId);
+
   SBufferWriter bw = tbufInitWriter(NULL, false);
   dnodeBeginObject(&bw);
-  dnodeAddStringField(&bw, "instanceId", dnode->cfg->clusterId);
+  dnodeAddStringField(&bw, "instanceId", clusterId);
   dnodeAddIntField(&bw, "reportVersion", 1);
   dnodeAddOsInfo(&bw);
   dnodeAddCpuInfo(&bw);
   dnodeAddMemoryInfo(&bw);
-  dnodeAddVersionInfo(telem, &bw);
-  dnodeAddRuntimeInfo(telem, &bw);
+  dnodeAddVersionInfo(&bw);
+  dnodeAddRuntimeInfo(&bw);
   dnodeCloseObject(&bw);
 
   const char* header =
@@ -227,25 +242,23 @@ static void dnodeSendTelemetryReport(SDnTelem* telem) {
 }
 
 static void* dnodeTelemThreadFp(void* param) {
-  SDnTelem* telem = param;
-
   struct timespec end = {0};
   clock_gettime(CLOCK_REALTIME, &end);
   end.tv_sec += 300;  // wait 5 minutes before send first report
 
   setThreadName("dnode-telem");
 
-  while (!telem->exit) {
+  while (!tsTelem.exit) {
     int32_t         r = 0;
     struct timespec ts = end;
-    pthread_mutex_lock(&telem->lock);
-    r = pthread_cond_timedwait(&telem->cond, &telem->lock, &ts);
-    pthread_mutex_unlock(&telem->lock);
+    pthread_mutex_lock(&tsTelem.lock);
+    r = pthread_cond_timedwait(&tsTelem.cond, &tsTelem.lock, &ts);
+    pthread_mutex_unlock(&tsTelem.lock);
     if (r == 0) break;
     if (r != ETIMEDOUT) continue;
 
     if (mnodeIsServing()) {
-      dnodeSendTelemetryReport(telem);
+      dnodeSendTelemetryReport();
     }
     end.tv_sec += REPORT_INTERVAL;
   }
@@ -253,40 +266,35 @@ static void* dnodeTelemThreadFp(void* param) {
   return NULL;
 }
 
-static void dnodeGetEmail(SDnTelem* telem, char* filepath) {
+static void dnodeGetEmail(char* filepath) {
   int32_t fd = taosOpenFileRead(filepath);
   if (fd < 0) {
     return;
   }
 
-  if (taosReadFile(fd, (void*)telem->email, TSDB_FQDN_LEN) < 0) {
+  if (taosReadFile(fd, (void*)tsTelem.email, TSDB_FQDN_LEN) < 0) {
     dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
   }
 
   taosCloseFile(fd);
 }
 
-int32_t dnodeInitTelem(SDnTelem** out) {
-  SDnTelem* telem = calloc(1, sizeof(SDnTelem));
-  if (telem == NULL) return -1;
-
-  telem->enable = tsEnableTelemetryReporting;
-  *out = telem;
+int32_t dnodeInitTelem() {
+  tsTelem.enable = tsEnableTelemetryReporting;
+  if (!tsTelem.enable) return 0;
 
-  if (!telem->enable) return 0;
+  tsTelem.exit = 0;
+  pthread_mutex_init(&tsTelem.lock, NULL);
+  pthread_cond_init(&tsTelem.cond, NULL);
+  tsTelem.email[0] = 0;
 
-  telem->exit = 0;
-  pthread_mutex_init(&telem->lock, NULL);
-  pthread_cond_init(&telem->cond, NULL);
-  telem->email[0] = 0;
-
-  dnodeGetEmail(telem, "/usr/local/taos/email");
+  dnodeGetEmail("/usr/local/taos/email");
 
   pthread_attr_t attr;
   pthread_attr_init(&attr);
   pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
 
-  int32_t code = pthread_create(&telem->thread, &attr, dnodeTelemThreadFp, telem);
+  int32_t code = pthread_create(&tsTelem.thread, &attr, dnodeTelemThreadFp, NULL);
   pthread_attr_destroy(&attr);
   if (code != 0) {
     dTrace("failed to create telemetry thread since :%s", strerror(code));
@@ -296,26 +304,18 @@ int32_t dnodeInitTelem(SDnTelem** out) {
   return 0;
 }
 
-void dnodeCleanupTelem(SDnTelem** out) {
-  SDnTelem* telem = *out;
-  *out = NULL;
+void dnodeCleanupTelem() {
+  if (!tsTelem.enable) return;
 
-  if (!telem->enable) {
-    free(telem);
-    return;
-  }
+  if (taosCheckPthreadValid(tsTelem.thread)) {
+    pthread_mutex_lock(&tsTelem.lock);
+    tsTelem.exit = 1;
+    pthread_cond_signal(&tsTelem.cond);
+    pthread_mutex_unlock(&tsTelem.lock);
 
-  if (taosCheckPthreadValid(telem->thread)) {
-    pthread_mutex_lock(&telem->lock);
-    telem->exit = 1;
-    pthread_cond_signal(&telem->cond);
-    pthread_mutex_unlock(&telem->lock);
-
-    pthread_join(telem->thread, NULL);
+    pthread_join(tsTelem.thread, NULL);
   }
 
-  pthread_mutex_destroy(&telem->lock);
-  pthread_cond_destroy(&telem->cond);
-
-  free(telem);
+  pthread_mutex_destroy(&tsTelem.lock);
+  pthread_cond_destroy(&tsTelem.cond);
 }

source/server/dnode/src/dnodeTrans.c

@@ -14,7 +14,7 @@
  */
 
 /* this file is mainly responsible for the communication between DNODEs. Each 
- * dnode works as both server and client. SDnode may send status, grant, config
+ * dnode works as both server and client. Dnode may send status, grant, config
  * messages to mnode, mnode may send create/alter/drop table/vnode messages 
  * to dnode. All theses messages are handled from here
  */
@@ -29,8 +29,19 @@
 #include "vnode.h"
 #include "mnode.h"
 
+typedef void (*RpcMsgFp)( SRpcMsg *pMsg);
+
+static struct {
+  void *   serverRpc;
+  void *   clientRpc;
+  void *   shellRpc;
+  int32_t  queryReqNum;
+  int32_t  submitReqNum;
+  RpcMsgFp peerMsgFp[TSDB_MSG_TYPE_MAX];
+  RpcMsgFp shellMsgFp[TSDB_MSG_TYPE_MAX];
+} tsTrans;
+
 static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode * dnode = dnodeInst();
   SRpcMsg rspMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0};
 
   if (pMsg->pCont == NULL) return;
@@ -39,7 +50,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) {
+  if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);
@@ -53,7 +64,7 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -64,27 +75,27 @@ static void dnodeProcessPeerReq(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 }
 
-int32_t dnodeInitServer(SDnTrans *trans) {
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE]  = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]   = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]   = vnodeProcessMsg;
+int32_t dnodeInitServer() {
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]   = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]   = vnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE]  = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]   = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]    = vnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE]  = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]   = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE] = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]    = vnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM]  = vnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE]  = dnodeProcessConfigDnodeReq;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE]  = dnodeProcessCreateMnodeReq;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE]  = dnodeProcessConfigDnodeReq;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE]  = dnodeProcessCreateMnodeReq;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH]          = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT]         = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS]        = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH]          = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT]         = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS]        = mnodeProcessMsg;
 
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
@@ -96,8 +107,8 @@ int32_t dnodeInitServer(SDnTrans *trans) {
   rpcInit.connType = TAOS_CONN_SERVER;
   rpcInit.idleTime = tsShellActivityTimer * 1000;
 
-  trans->serverRpc = rpcOpen(&rpcInit);
-  if (trans->serverRpc == NULL) {
+  tsTrans.serverRpc = rpcOpen(&rpcInit);
+  if (tsTrans.serverRpc == NULL) {
     dError("failed to init peer rpc server");
     return -1;
   }
@@ -106,17 +117,16 @@ int32_t dnodeInitServer(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupServer(SDnTrans *trans) {
-  if (trans->serverRpc) {
-    rpcClose(trans->serverRpc);
-    trans->serverRpc = NULL;
+void dnodeCleanupServer() {
+  if (tsTrans.serverRpc) {
+    rpcClose(tsTrans.serverRpc);
+    tsTrans.serverRpc = NULL;
     dInfo("dnode peer server is closed");
   }
 }
 
 static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode *dnode = dnodeInst();
-  if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) {
+  if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
     dTrace("msg:%p is ignored since dnode is stopping", pMsg);
     rpcFreeCont(pMsg->pCont);
@@ -124,10 +134,10 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 
   if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) {
-    dnodeUpdateMnodeFromPeer(dnode->meps, pEpSet);
+    dnodeUpdateMnodeFromPeer(pEpSet);
   }
 
-  RpcMsgFp fp = dnode->trans->peerMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.peerMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -141,27 +151,27 @@ static void dnodeProcessRspFromPeer(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   rpcFreeCont(pMsg->pCont);
 }
 
-int32_t dnodeInitClient(SDnTrans *trans) {
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP]   = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP]   = mnodeProcessMsg;
+int32_t dnodeInitClient() {
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_TABLE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE_RSP]   = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE_RSP]   = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP]   = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP]    = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP]   = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE_RSP] = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE_RSP]    = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM_RSP]  = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE_RSP]  = mnodeProcessMsg;
 
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP]  = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP]          = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP]         = mnodeProcessMsg;
-  trans->peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP]        = dnodeProcessStatusRsp;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE_RSP]  = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_AUTH_RSP]          = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_GRANT_RSP]         = mnodeProcessMsg;
+  tsTrans.peerMsgFp[TSDB_MSG_TYPE_DM_STATUS_RSP]        = dnodeProcessStatusRsp;
 
   char secret[TSDB_KEY_LEN] = "secret";
   SRpcInit rpcInit;
@@ -176,8 +186,8 @@ int32_t dnodeInitClient(SDnTrans *trans) {
   rpcInit.ckey         = "key";
   rpcInit.secret       = secret;
 
-  trans->clientRpc = rpcOpen(&rpcInit);
-  if (trans->clientRpc == NULL) {
+  tsTrans.clientRpc = rpcOpen(&rpcInit);
+  if (tsTrans.clientRpc == NULL) {
     dError("failed to init peer rpc client");
     return -1;
   }
@@ -186,26 +196,25 @@ int32_t dnodeInitClient(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupClient(SDnTrans *trans) {
-  if (trans->clientRpc) {
-    rpcClose(trans->clientRpc);
-    trans->clientRpc = NULL;
+void dnodeCleanupClient() {
+  if (tsTrans.clientRpc) {
+    rpcClose(tsTrans.clientRpc);
+    tsTrans.clientRpc = NULL;
     dInfo("dnode peer rpc client is closed");
   }
 }
 
 static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  SDnode * dnode = dnodeInst();
   SRpcMsg rpcMsg = {.handle = pMsg->handle, .pCont = NULL, .contLen = 0};
 
   if (pMsg->pCont == NULL) return;
-  if (dnode->main->runStatus == TD_RUN_STAT_STOPPED) {
+  if (dnodeGetRunStat() == TD_RUN_STAT_STOPPED) {
     dError("RPC %p, shell msg:%s is ignored since dnode exiting", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_DND_EXITING;
     rpcSendResponse(&rpcMsg);
     rpcFreeCont(pMsg->pCont);
     return;
-  } else if (dnode->main->runStatus != TD_RUN_STAT_RUNNING) {
+  } else if (dnodeGetRunStat() != TD_RUN_STAT_RUNNING) {
     dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rpcMsg);
@@ -213,14 +222,13 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
     return;
   }
 
-  SDnTrans *trans = dnode->trans;
   if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
-    atomic_fetch_add_32(&trans->queryReqNum, 1);
+    atomic_fetch_add_32(&tsTrans.queryReqNum, 1);
   } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
-    atomic_fetch_add_32(&trans->submitReqNum, 1);
+    atomic_fetch_add_32(&tsTrans.submitReqNum, 1);
   } else {}
 
-  RpcMsgFp fp = trans->shellMsgFp[pMsg->msgType];
+  RpcMsgFp fp = tsTrans.shellMsgFp[pMsg->msgType];
   if (fp != NULL) {
     (*fp)(pMsg);
   } else {
@@ -247,27 +255,23 @@ static int32_t dnodeAuthNetTest(char *user, char *spi, char *encrypt, char *secr
 }
 
 void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
-  SDnode *dnode = dnodeInst();
-  rpcSendRequest(dnode->trans->clientRpc, epSet, rpcMsg, NULL);
+  rpcSendRequest(tsTrans.clientRpc, epSet, rpcMsg, NULL);
 }
 
 void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
-  SDnode *   dnode = dnodeInst();
   SRpcEpSet epSet = {0};
-  dnodeGetEpSetForPeer(dnode->meps, &epSet);
+  dnodeGetEpSetForPeer(&epSet);
   dnodeSendMsgToDnode(&epSet, rpcMsg);
 }
 
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
-  SDnode *   dnode = dnodeInst();
   SRpcEpSet epSet = {0};
-  dnodeGetEpSetForPeer(dnode->meps, &epSet);
-  rpcSendRecv(dnode->trans->clientRpc, &epSet, rpcMsg, rpcRsp);
+  dnodeGetEpSetForPeer(&epSet);
+  rpcSendRecv(tsTrans.clientRpc, &epSet, rpcMsg, rpcRsp);
 }
 
 void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
-  SDnode *dnode = dnodeInst();
-  rpcSendRecv(dnode->trans->clientRpc, epSet, rpcMsg, rpcRsp);
+  rpcSendRecv(tsTrans.clientRpc, epSet, rpcMsg, rpcRsp);
 }
 
 static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
@@ -303,52 +307,52 @@ static int32_t dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, c
   return rpcRsp.code;
 }
 
-int32_t dnodeInitShell(SDnTrans *trans) {
-  trans->shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_QUERY]  = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_FETCH]  = vnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg;
+int32_t dnodeInitShell() {
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_QUERY]  = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_FETCH]  = vnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessMsg;
 
   // the following message shall be treated as mnode write
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]         = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION]   = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP]         = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_COMPACT_VNODE]   = mnodeProcessMsg;
 
   // the following message shall be treated as mnode query
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]     = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]       = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]        = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]    = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META]   = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_SHOW]          = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]      = mnodeProcessMsg;
-  trans->shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg;
-
-  trans->shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]     = dnodeProcessStartupReq;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]     = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]       = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]        = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]    = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP] = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META]   = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_SHOW]          = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]      = mnodeProcessMsg;
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = mnodeProcessMsg;
+
+  tsTrans.shellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST]     = dnodeProcessStartupReq;
 
   int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
   if (numOfThreads < 1) {
@@ -366,8 +370,8 @@ int32_t dnodeInitShell(SDnTrans *trans) {
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.afp          = dnodeRetrieveUserAuthInfo;
 
-  trans->shellRpc = rpcOpen(&rpcInit);
-  if (trans->shellRpc == NULL) {
+  tsTrans.shellRpc = rpcOpen(&rpcInit);
+  if (tsTrans.shellRpc == NULL) {
     dError("failed to init shell rpc server");
     return -1;
   }
@@ -376,41 +380,31 @@ int32_t dnodeInitShell(SDnTrans *trans) {
   return 0;
 }
 
-void dnodeCleanupShell(SDnTrans *trans) {
-  if (trans->shellRpc) {
-    rpcClose(trans->shellRpc);
-    trans->shellRpc = NULL;
+void dnodeCleanupShell() {
+  if (tsTrans.shellRpc) {
+    rpcClose(tsTrans.shellRpc);
+    tsTrans.shellRpc = NULL;
   }
 }
 
-int32_t dnodeInitTrans(SDnTrans **out) {
-  SDnTrans *trans = calloc(1, sizeof(SDnTrans));
-  if (trans == NULL) return -1;
-
-  *out = trans;
-
-  if (dnodeInitClient(trans) != 0) {
+int32_t dnodeInitTrans() {
+  if (dnodeInitClient() != 0) {
     return -1;
   }
 
-  if (dnodeInitServer(trans) != 0) {
+  if (dnodeInitServer() != 0) {
     return -1;
   }
 
-  if (dnodeInitShell(trans) != 0) {
+  if (dnodeInitShell() != 0) {
     return -1;
   }
 
   return 0;
 }
 
-void dnodeCleanupTrans(SDnTrans **out) {
-  SDnTrans* trans = *out;
-  *out = NULL;
-
-  dnodeCleanupShell(trans);
-  dnodeCleanupServer(trans);
-  dnodeCleanupClient(trans);
-
-  free(trans);
+void dnodeCleanupTrans() {
+  dnodeCleanupShell();
+  dnodeCleanupServer();
+  dnodeCleanupClient();
 }

source/server/vnode/CMakeLists.txt

@@ -3,7 +3,7 @@ add_subdirectory(tq)
 add_subdirectory(tsdb)
 
 aux_source_directory(src VNODE_SRC)
-add_library(vnode ${VNODE_SRC})
+add_library(vnode STATIC ${VNODE_SRC})
 target_include_directories(
     vnode
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode"
@@ -15,4 +15,11 @@ target_link_libraries(
     PUBLIC meta
     PUBLIC tq
     PUBLIC tsdb
-)
\ No newline at end of file
+    PUBLIC wal
+    PUBLIC cjson
+)
+
+# test
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})
\ No newline at end of file

src/vnode/inc/vnodeCfg.h → source/server/vnode/inc/vnodeCfg.h

@@ -13,19 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_CFG_H
-#define TDENGINE_VNODE_CFG_H
+#ifndef _TD_VNODE_CFG_H_
+#define _TD_VNODE_CFG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeReadCfg(SVnodeObj *pVnode);
+int32_t vnodeReadCfg(SVnode *pVnode);
 int32_t vnodeWriteCfg(SCreateVnodeMsg *pVnodeCfg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_CFG_H_*/

source/server/vnode/inc/vnodeInt.h

@@ -16,7 +16,6 @@
 #ifndef _TD_VNODE_INT_H_
 #define _TD_VNODE_INT_H_
 
-
 #include "os.h"
 #include "amalloc.h"
 #include "meta.h"
@@ -25,20 +24,83 @@
 #include "trpc.h"
 #include "tsdb.h"
 #include "vnode.h"
+#include "tlog.h"
+#include "tqueue.h"
+#include "wal.h"
+#include "tworker.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct SVnode {
+extern int32_t vDebugFlag;
+
+#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }}
+#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }}
+#define vWarn(...)  { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }}
+#define vInfo(...)  { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("VND ", 255, __VA_ARGS__); }}
+#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
+#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
+
+typedef struct {
   SMeta *        pMeta;
   STsdb *        pTsdb;
   STQ *          pTQ;
   SMemAllocator *allocator;
+
+  int32_t  vgId;      // global vnode group ID
+  int32_t  refCount;  // reference count
+  int64_t  queuedWMsgSize;
+  int32_t  queuedWMsg;
+  int32_t  queuedRMsg;
+  int32_t  numOfExistQHandle;  // current initialized and existed query handle in current dnode
+  int32_t  flowctrlLevel;
+  int8_t   preClose;  // drop and close switch
+  int8_t   reserved[3];
+  int64_t  sequence;  // for topic
+  int8_t   status;
+  int8_t   role;
+  int8_t   accessState;
+  int8_t   isFull;
+  int8_t   isCommiting;
+  int8_t   dbReplica;
+  int8_t   dropped;
+  int8_t   dbType;
+  uint64_t version;   // current version
+  uint64_t cversion;  // version while commit start
+  uint64_t fversion;  // version on saved data file
+  void *   wqueue;    // write queue
+  void *   qqueue;    // read query queue
+  void *   fqueue;    // read fetch/cancel queue
+  void *   wal;
+  void *   tsdb;
+  int64_t  sync;
+  void *   events;
+  void *   cq;  // continuous query
+  int32_t  dbCfgVersion;
+  int32_t  vgCfgVersion;
+  // STsdbCfg tsdbCfg;
+#if 0  
+  SSyncCfg syncCfg;
+#endif  
+  SWalCfg  walCfg;
+  void *   qMgmt;
+  char *   rootDir;
+  tsem_t   sem;
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
+  pthread_mutex_t statusMutex;
 } SVnode;
 
+typedef struct {
+  int32_t len;
+  void *  rsp;
+  void *  qhandle;  // used by query and retrieve msg
+} SVnRsp;
+
+void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_VNODE_INT_H_*/
\ No newline at end of file
+#endif /*_TD_VNODE_INT_H_*/

source/server/vnode/inc/vnodeMain.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_MAIN_H_
+#define _TD_VNODE_MAIN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitMain();
+void    vnodeCleanupMain();
+
+int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeDrop(int32_t vgId);
+int32_t vnodeOpen(int32_t vgId);
+int32_t vnodeAlter(SVnode *pVnode, SCreateVnodeMsg *pVnodeCfg);
+int32_t vnodeSync(int32_t vgId);
+int32_t vnodeClose(int32_t vgId);
+void    vnodeCleanUp(SVnode *pVnode);
+void    vnodeDestroy(SVnode *pVnode);
+int32_t vnodeCompact(int32_t vgId);
+void    vnodeBackup(int32_t vgId);
+void    vnodeGetStatus(struct SStatusMsg *status);
+
+SVnode *vnodeAcquire(int32_t vgId);
+SVnode *vnodeAcquireNotClose(int32_t vgId);
+void    vnodeRelease(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_MAIN_H_*/

source/server/vnode/inc/vnodeMemAllocator.h

@@ -16,12 +16,38 @@
 #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
 #define _TD_VNODE_MEM_ALLOCATOR_H_
 
-#include "amalloc.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SVnodeMemAllocator SVnodeMemAllocator;
+
+SVnodeMemAllocator *VMACreate(size_t size /* base size */, size_t ssize /* step size */,
+                              size_t threshold /* threshold size when full*/);
+void                VMADestroy(SVnodeMemAllocator *pvma);
+void                VMAReset(SVnodeMemAllocator *pvma);
+void *              VMAMalloc(SVnodeMemAllocator *pvma, size_t size);
+void                VMAFree(SVnodeMemAllocator *pvma, void *ptr);
+bool                VMAIsFull(SVnodeMemAllocator *pvma);
+
+// ------------------ FOR TEST ONLY ------------------
+typedef struct SVMANode {
+  struct SVMANode *prev;
+  size_t           tsize;
+  size_t           used;
+  char             data[];
+} SVMANode;
+
+struct SVnodeMemAllocator {
+  bool      full;       // if allocator is full
+  size_t    threshold;  // threshold;
+  size_t    ssize;      // step size to allocate
+  SVMANode *inuse;      // inuse node to allocate
+  SVMANode  node;       // basic node to use
+};
+
 #ifdef __cplusplus
 }
 #endif

src/vnode/inc/vnodeBackup.h → source/server/vnode/inc/vnodeMgmt.h

@@ -13,20 +13,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_BACKUP_H
-#define TDENGINE_VNODE_BACKUP_H
+#ifndef _TD_VNODE_MGMT_H_
+#define _TD_VNODE_MGMT_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitBackup();
-void    vnodeCleanupBackup();
-int32_t vnodeBackup(int32_t vgId);
+int32_t vnodeInitMgmt();
+void    vnodeCleanupMgmt();
+void    vnodeProcessMgmtMsg(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_MGMT_H_*/

src/vnode/inc/vnodeMain.h → source/server/vnode/inc/vnodeMgmtMsg.h

@@ -13,25 +13,23 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_MAIN_H
-#define TDENGINE_VNODE_MAIN_H
+#ifndef _TD_VNODE_MGMT_MSG_H_
+#define _TD_VNODE_MGMT_MSG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg);
-int32_t vnodeDrop(int32_t vgId);
-int32_t vnodeOpen(int32_t vgId);
-int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
-int32_t vnodeSync(int32_t vgId);
-int32_t vnodeClose(int32_t vgId);
-void    vnodeCleanUp(SVnodeObj *pVnode);
-void    vnodeDestroy(SVnodeObj *pVnode);
+int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg);
+int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_MGMT_H_*/

src/vnode/inc/vnodeMgmt.h → source/server/vnode/inc/vnodeRead.h

@@ -13,30 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_MGMT_H
-#define TDENGINE_VNODE_MGMT_H
+#ifndef _TD_VNODE_READ_H_
+#define _TD_VNODE_READ_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitMgmt();
-void    vnodeCleanupMgmt();
+int32_t    vnodeInitRead();
+void       vnodeCleanupRead();
+taos_queue vnodeAllocQueryQueue(SVnode *pVnode);
+taos_queue vnodeAllocFetchQueue(SVnode *pVnode);
+void       vnodeFreeQueryQueue(taos_queue pQueue);
+void       vnodeFreeFetchQueue(taos_queue pQueue);
 
-void*   vnodeAcquire(int32_t vgId);
-void    vnodeRelease(void *pVnode);
-void*   vnodeGetWal(void *pVnode);
+void vnodeProcessReadMsg(SRpcMsg *pRpcMsg);
+int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle);
 
-int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
-void    vnodeBuildStatusMsg(void *pStatus);
-void    vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes);
-
-void    vnodeAddIntoHash(SVnodeObj* pVnode);
-void    vnodeRemoveFromHash(SVnodeObj * pVnode);
+void vnodeStartRead(SVnode *pVnode);
+void vnodeStopRead(SVnode *pVnode);
+void vnodeWaitReadCompleted(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_READ_H_*/

src/vnode/inc/vnodeWrite.h → source/server/vnode/inc/vnodeReadMsg.h

@@ -13,24 +13,32 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_WRITE_H
-#define TDENGINE_VNODE_WRITE_H
+#ifndef _TD_VNODE_READ_MSG_H_
+#define _TD_VNODE_READ_MSG_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitWrite(void);
-void    vnodeCleanupWrite(void);
+typedef struct SReadMsg {
+  int32_t code;
+  int32_t contLen;
+  int8_t  qtype;
+  int8_t  msgType;
+  SVnode *pVnode;
+  SVnRsp rspRet;
+  void *  rpcHandle;
+  void *  rpcAhandle;
+  void *  qhandle;
+  char    pCont[];
+} SReadMsg;
 
-int32_t vnodeWriteToWQueue(void *pVnode, void *pHead, int32_t qtype, void *pRpcMsg);
-void    vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite);
-int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet);
-void    vnodeWaitWriteCompleted(SVnodeObj *pVnode);
+int32_t vnodeProcessQueryMsg(SVnode *pVnode, SReadMsg *pRead);
+int32_t vnodeProcessFetchMsg(SVnode *pVnode, SReadMsg *pRead);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
\ No newline at end of file
+#endif /*_TD_VNODE_READ_MSG_H_*/

src/vnode/inc/vnodeStatus.h → source/server/vnode/inc/vnodeStatus.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_STATUS_H
-#define TDENGINE_VNODE_STATUS_H
+#ifndef _TD_VNODE_STATUS_H_
+#define _TD_VNODE_STATUS_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -25,24 +25,23 @@ typedef enum _VN_STATUS {
   TAOS_VN_STATUS_INIT = 0,
   TAOS_VN_STATUS_READY = 1,
   TAOS_VN_STATUS_CLOSING = 2,
-  TAOS_VN_STATUS_UPDATING = 3,
-  TAOS_VN_STATUS_RESET = 4,
+  TAOS_VN_STATUS_UPDATING = 3
 } EVnodeStatus;
 
-bool vnodeSetInitStatus(SVnodeObj* pVnode);
-bool vnodeSetReadyStatus(SVnodeObj* pVnode);
-bool vnodeSetClosingStatus(SVnodeObj* pVnode);
-bool vnodeSetUpdatingStatus(SVnodeObj* pVnode);
-bool vnodeSetResetStatus(SVnodeObj* pVnode);
+// vnodeStatus
+extern char* vnodeStatus[];
 
-bool vnodeInInitStatus(SVnodeObj* pVnode);
-bool vnodeInReadyStatus(SVnodeObj* pVnode);
-bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode);
-bool vnodeInClosingStatus(SVnodeObj* pVnode);
-bool vnodeInResetStatus(SVnodeObj* pVnode);
+bool vnodeSetInitStatus(SVnode* pVnode);
+bool vnodeSetReadyStatus(SVnode* pVnode);
+bool vnodeSetClosingStatus(SVnode* pVnode);
+bool vnodeSetUpdatingStatus(SVnode* pVnode);
+
+bool vnodeInInitStatus(SVnode* pVnode);
+bool vnodeInReadyStatus(SVnode* pVnode);
+bool vnodeInClosingStatus(SVnode* pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
\ No newline at end of file
+#endif /*_TD_VNODE_STATUS_H_*/
\ No newline at end of file

src/vnode/inc/vnodeVersion.h → source/server/vnode/inc/vnodeVersion.h

@@ -13,19 +13,19 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_VERSION_H
-#define TDENGINE_VNODE_VERSION_H
+#ifndef _TD_VNODE_VERSION_H_
+#define _TD_VNODE_VERSION_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeReadVersion(SVnodeObj *pVnode);
-int32_t vnodeSaveVersion(SVnodeObj *pVnode);
+int32_t vnodeReadVersion(SVnode *pVnode);
+int32_t vnodeSaveVersion(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif /*_TD_VNODE_VERSION_H_*/

src/vnode/inc/vnodeWorker.h → source/server/vnode/inc/vnodeWorker.h

@@ -13,21 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODE_WORKER_H
-#define TDENGINE_VNODE_WORKER_H
+#ifndef _TD_VNODE_WORKER_H_
+#define _TD_VNODE_WORKER_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 #include "vnodeInt.h"
 
-int32_t vnodeInitMWorker();
-void    vnodeCleanupMWorker();
-int32_t vnodeCleanupInMWorker(SVnodeObj *pVnode);
-int32_t vnodeDestroyInMWorker(SVnodeObj *pVnode);
+int32_t vnodeInitWorker();
+void    vnodeCleanupWorker();
+void    vnodeProcessCleanupTask(SVnode *pVnode);
+void    vnodeProcessDestroyTask(SVnode *pVnode);
+void    vnodeProcessBackupTask(SVnode *pVnode);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif
\ No newline at end of file
+#endif /*_TD_VNODE_WORKER_H_*/
\ No newline at end of file

source/server/vnode/inc/vnodeWrite.h

@@ -16,6 +16,25 @@
 #ifndef _TD_VNODE_WRITE_H_
 #define _TD_VNODE_WRITE_H_
 
-int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp);
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t    vnodeInitWrite();
+void       vnodeCleanupWrite();
+taos_queue vnodeAllocWriteQueue(SVnode *pVnode);
+void       vnodeFreeWriteQueue(taos_queue pQueue);
+
+void    vnodeProcessWriteMsg(SRpcMsg *pRpcMsg);
+int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead);
+
+void vnodeStartWrite(SVnode *pVnode);
+void vnodeStopWrite(SVnode *pVnode);
+void vnodeWaitWriteCompleted(SVnode *pVnode);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /*_TD_VNODE_WRITE_H_*/
\ No newline at end of file

source/server/vnode/inc/vnodeWriteMsg.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_WRITE_MSG_H_
+#define _TD_VNODE_WRITE_MSG_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+ 
+int32_t vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pReq, SSubmitRsp *pRsp);
+int32_t vnodeProcessCreateTableReq(SVnode *pVnode, SCreateTableReq *pReq, SCreateTableRsp *pRsp);
+int32_t vnodeProcessDropTableReq(SVnode *pVnode, SDropTableReq *pReq, SDropTableRsp *pRsp);
+int32_t vnodeProcessAlterTableReq(SVnode *pVnode, SAlterTableReq *pReq, SAlterTableRsp *pRsp);
+int32_t vnodeProcessDropStableReq(SVnode *pVnode, SDropStableReq *pReq, SDropStableRsp *pRsp);
+int32_t vnodeProcessUpdateTagValReq(SVnode *pVnode, SUpdateTagValReq *pReq, SUpdateTagValRsp *pRsp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_WRITE_MSG_H_*/
\ No newline at end of file

source/server/vnode/meta/CMakeLists.txt

 aux_source_directory(src META_SRC)
-add_library(meta ${META_SRC})
+add_library(meta STATIC ${META_SRC})
 target_include_directories(
     meta
     PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
@@ -8,4 +8,9 @@ target_include_directories(
 target_link_libraries(
     meta
     PUBLIC common
-)
\ No newline at end of file
+)
+target_link_libraries(meta PUBLIC rocksdb)
+
+if(${BUILD_TEST})
+    add_subdirectory(test)
+endif(${BUILD_TEST})

source/server/vnode/meta/src/meta.c

@@ -13,24 +13,72 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <rocksdb/c.h>
+
+#include "thash.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "ttypes.h"
+
 #include "meta.h"
 
-int metaCreateTable(SMeta *pMeta, SCreateTableReq *pReq) {
-  // TODO
-  return 0;
-}
+typedef struct STable {
+  uint64_t uid;
+  tstr *   name;
+  uint64_t suid;
+  SArray * schema;
+} STable;
 
-int metaDropTable(SMeta *pMeta, SDropTableReq *pReq) {
-  // TODO
-  return 0;
+typedef struct STableObj {
+  bool     pin;
+  uint64_t ref;
+  SRWLatch latch;
+  uint64_t offset;
+  SList *  ctbList;  //  child table list
+  STable * pTable;
+} STableObj;
+
+struct SMeta {
+  pthread_rwlock_t rwLock;
+
+  SHashObj * pTableObjHash;  // uid --> STableObj
+  SList *    stbList;        // super table list
+  rocksdb_t *tbnameDb;       // tbname --> uid
+  rocksdb_t *tagDb;          // uid --> tag
+  rocksdb_t *schemaDb;
+  size_t     totalUsed;
+};
+
+SMeta *metaOpen(SMetaOptions *options) {
+  SMeta *pMeta = NULL;
+  char * err = NULL;
+
+  pMeta = (SMeta *)calloc(1, sizeof(*pMeta));
+  if (pMeta == NULL) {
+    return NULL;
+  }
+
+  pthread_rwlock_init(&(pMeta->rwLock), NULL);
+
+  pMeta->pTableObjHash = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+
+  pMeta->stbList = tdListNew(sizeof(STableObj *));
+
+  // Open tbname DB
+  rocksdb_options_t *tbnameDbOptions = rocksdb_options_create();
+  pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err);
+
+  // Open tag DB
+  pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err);
+
+  // Open schema DB
+  pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err);
+
+  return pMeta;
 }
 
-int metaAlterTable(SMeta *pMeta, SAlterTableReq *pReq) {
+void metaClose(SMeta *pMeta) {
   // TODO
-  return 0;
 }
 
-int metaCommit(SMeta *pMeta) {
-  // TODO
-  return 0;
-}
\ No newline at end of file
+int metaCommit(SMeta *meta) { return 0; }
\ No newline at end of file

source/server/vnode/meta/test/CMakeLists.txt

+add_executable(metaTest "")
+target_sources(metaTest
+    PRIVATE
+    "../src/meta.c"
+    "metaTests.cpp"
+)
+target_include_directories(metaTest 
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_link_libraries(metaTest
+    os
+    util
+    common
+    rocksdb
+    gtest_main
+)
\ No newline at end of file

source/server/vnode/meta/test/metaTests.cpp

+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "meta.h"
+
+TEST(MetaTest, meta_open_test) {
+    metaOpen(NULL);
+    std::cout << "Hello META!" << std::endl;
+}
\ No newline at end of file

src/vnode/src/vnodeCfg.c → source/server/vnode/src/vnodeCfg.c

@@ -17,10 +17,10 @@
 #include "os.h"
 #include "cJSON.h"
 #include "tglobal.h"
-#include "dnode.h"
 #include "vnodeCfg.h"
 
-static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
+static void vnodeLoadCfg(SVnode *pVnode, SCreateVnodeMsg *vnodeMsg) {
+#if 0
   tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
   pVnode->dbCfgVersion = vnodeMsg->cfg.dbCfgVersion;
   pVnode->vgCfgVersion = vnodeMsg->cfg.vgCfgVersion;
@@ -56,9 +56,11 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
     SNodeInfo *node = &pVnode->syncCfg.nodeInfo[i];
     vInfo("vgId:%d, dnode:%d, %s:%u", pVnode->vgId, node->nodeId, node->nodeFqdn, node->nodePort);
   }
+#endif  
 }
 
-int32_t vnodeReadCfg(SVnodeObj *pVnode) {
+int32_t vnodeReadCfg(SVnode *pVnode) {
+#if 0
   int32_t ret = TSDB_CODE_VND_APP_ERROR;
   int32_t len = 0;
   int     maxLen = 1000;
@@ -66,6 +68,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
   cJSON * root = NULL;
   FILE *  fp = NULL;
   bool    nodeChanged = false;
+
   SCreateVnodeMsg vnodeMsg;
 
   char file[TSDB_FILENAME_LEN + 30] = {0};
@@ -286,8 +289,13 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
     }
     tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
 
-    bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp);
-    if (changed) nodeChanged = changed;
+    char nodeEpStr[TSDB_EP_LEN];
+    vnodeGetDnodeEp(node->nodeId, nodeEpStr, NULL, NULL);
+    bool changed = (strcmp(node->nodeEp, nodeEpStr) != 0);
+    if (changed) {
+      tstrncpy(node->nodeEp, nodeEpStr, TSDB_EP_LEN);
+      nodeChanged = changed;
+    }
   }
 
   ret = TSDB_CODE_SUCCESS;
@@ -350,7 +358,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
   for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) {
     SVnodeDesc *node = &pMsg->nodes[i];
-    dnodeUpdateEp(node->nodeId, node->nodeEp, NULL, NULL);
+    vnodeGetDnodeEp(node->nodeId, node->nodeEp, NULL, NULL);
     len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", node->nodeId);
     len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", node->nodeEp);
     if (i < pMsg->cfg.vgReplica - 1) {
@@ -368,5 +376,6 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) {
   terrno = 0;
 
   vInfo("vgId:%d, successed to write %s", pMsg->cfg.vgId, file);
+#endif  
   return TSDB_CODE_SUCCESS;
 }

source/server/vnode/src/vnodeInt.c

@@ -13,16 +13,39 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tstep.h"
+#include "vnodeMain.h"
+#include "vnodeMgmt.h"
+#include "vnodeRead.h"
+#include "vnodeWorker.h"
+#include "vnodeWrite.h"
 
-int32_t vnodeInit(SVnodePara para) { return 0; }
+static struct {
+  struct SSteps *steps;
+  SVnodeFp       fp;
+} tsVint;
 
-void vnodeCleanup() {}
+int32_t vnodeInit(SVnodePara para) {
+  tsVint.fp = para.fp;
 
-int32_t vnodeGetStatistics(SVnodeStat *stat) { return 0; }
+  struct SSteps *steps = taosStepInit(8, NULL);
+  if (steps == NULL) return -1;
 
-void vnodeGetStatus(struct SStatusMsg *status) {}
+  taosStepAdd(steps, "vnode-main", vnodeInitMain, vnodeCleanupMain);
+  taosStepAdd(steps, "vnode-worker",vnodeInitWorker, vnodeCleanupWorker);
+  taosStepAdd(steps, "vnode-read", vnodeInitRead, vnodeCleanupRead);
+  taosStepAdd(steps, "vnode-mgmt", vnodeInitMgmt, vnodeCleanupMgmt);
+  taosStepAdd(steps, "vnode-write", vnodeInitWrite, vnodeCleanupWrite);
+  // taosStepAdd(steps, "vnode-queue", tsdbInitCommitQueue, tsdbDestroyCommitQueue);
 
-void vnodeSetAccess(struct SVgroupAccess *access, int32_t numOfVnodes) {}
+  tsVint.steps = steps;
+  return taosStepExec(tsVint.steps);
+}
 
-void vnodeProcessMsg(SRpcMsg *msg) {}
+void vnodeCleanup() { taosStepCleanup(tsVint.steps); }
+
+void vnodeGetDnodeEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port) {
+  return (*tsVint.fp.GetDnodeEp)(dnodeId, ep, fqdn, port);
+}
\ No newline at end of file

source/server/vnode/src/vnodeMemAllocator.c

@@ -11,4 +11,114 @@
  *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
+ */
+
+#include "vnodeMemAllocator.h"
+
+#define VMA_IS_FULL(pvma) \
+  (((pvma)->inuse != &((pvma)->node)) || ((pvma)->inuse->tsize - (pvma)->inuse->used < (pvma)->threshold))
+
+static SVMANode *VMANodeNew(size_t size);
+static void      VMANodeFree(SVMANode *node);
+
+SVnodeMemAllocator *VMACreate(size_t size, size_t ssize, size_t threshold) {
+  SVnodeMemAllocator *pvma = NULL;
+
+  if (size < threshold) {
+    return NULL;
+  }
+
+  pvma = (SVnodeMemAllocator *)malloc(sizeof(*pvma) + size);
+  if (pvma) {
+    pvma->full = false;
+    pvma->threshold = threshold;
+    pvma->ssize = ssize;
+    pvma->inuse = &(pvma->node);
+
+    pvma->inuse->prev = NULL;
+    pvma->inuse->tsize = size;
+    pvma->inuse->used = 0;
+  }
+
+  return pvma;
+}
+
+void VMADestroy(SVnodeMemAllocator *pvma) {
+  if (pvma) {
+    VMAReset(pvma);
+    free(pvma);
+  }
+}
+
+void VMAReset(SVnodeMemAllocator *pvma) {
+  while (pvma->inuse != &(pvma->node)) {
+    SVMANode *node = pvma->inuse;
+    pvma->inuse = node->prev;
+    VMANodeFree(node);
+  }
+
+  pvma->inuse->used = 0;
+  pvma->full = false;
+}
+
+void *VMAMalloc(SVnodeMemAllocator *pvma, size_t size) {
+  void * ptr = NULL;
+  size_t tsize = size + sizeof(size_t);
+
+  if (pvma->inuse->tsize - pvma->inuse->used < tsize) {
+    SVMANode *pNode = VMANodeNew(MAX(pvma->ssize, tsize));
+    if (pNode == NULL) {
+      return NULL;
+    }
+
+    pNode->prev = pvma->inuse;
+    pvma->inuse = pNode;
+  }
+
+  ptr = pvma->inuse->data + pvma->inuse->used;
+  pvma->inuse->used += tsize;
+  *(size_t *)ptr = size;
+  ptr = POINTER_SHIFT(ptr, sizeof(size_t));
+
+  pvma->full = VMA_IS_FULL(pvma);
+
+  return ptr;
+}
+
+void VMAFree(SVnodeMemAllocator *pvma, void *ptr) {
+  if (ptr) {
+    size_t size = *(size_t *)POINTER_SHIFT(ptr, -sizeof(size_t));
+    if (POINTER_SHIFT(ptr, size) == pvma->inuse->data + pvma->inuse->used) {
+      pvma->inuse->used -= (size + sizeof(size_t));
+
+      if ((pvma->inuse->used == 0) && (pvma->inuse != &(pvma->node))) {
+        SVMANode *node = pvma->inuse;
+        pvma->inuse = node->prev;
+        VMANodeFree(node);
+      }
+
+      pvma->full = VMA_IS_FULL(pvma);
+    }
+  }
+}
+
+bool VMAIsFull(SVnodeMemAllocator *pvma) { return pvma->full; }
+
+static SVMANode *VMANodeNew(size_t size) {
+  SVMANode *node = NULL;
+
+  node = (SVMANode *)malloc(sizeof(*node) + size);
+  if (node) {
+    node->prev = NULL;
+    node->tsize = size;
+    node->used = 0;
+  }
+
+  return node;
+}
+
+static void VMANodeFree(SVMANode *node) {
+  if (node) {
+    free(node);
+  }
+}
\ No newline at end of file

source/server/vnode/src/vnodeMgmt.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeMain.h"
+#include "vnodeMgmt.h"
+#include "vnodeMgmtMsg.h"
+
+typedef struct {
+  SRpcMsg rpcMsg;
+  char    pCont[];
+} SVnMgmtMsg;
+
+static struct {
+  SWorkerPool pool;
+  taos_queue  pQueue;
+  int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
+} tsVmgmt = {0};
+
+static int32_t vnodeProcessMgmtStart(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype) {
+  SRpcMsg *pMsg = &pMgmt->rpcMsg;
+  int32_t  msgType = pMsg->msgType;
+
+  if (tsVmgmt.msgFp[msgType]) {
+    vTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[msgType]);
+    return (*tsVmgmt.msgFp[msgType])(pMsg);
+  } else {
+    vError("msg:%p, ahandle:%p type:%s not processed since no handle", pMgmt, pMsg->ahandle, taosMsg[msgType]);
+    return TSDB_CODE_DND_MSG_NOT_PROCESSED;
+  }
+}
+
+static void vnodeSendMgmtEnd(void *unused, SVnMgmtMsg *pMgmt, int32_t qtype, int32_t code) {
+  SRpcMsg *pMsg = &pMgmt->rpcMsg;
+  SRpcMsg  rsp = {0};
+
+  rsp.code = code;
+  vTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code);
+  if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) {
+    rsp.handle = pMsg->handle;
+    rsp.pCont = NULL;
+    rpcSendResponse(&rsp);
+  }
+
+  taosFreeQitem(pMsg);
+}
+
+static void vnodeInitMgmtReqFp() {
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = vnodeProcessCreateVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = vnodeProcessAlterVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE]   = vnodeProcessSyncVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_COMPACT_VNODE]= vnodeProcessCompactVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = vnodeProcessDropVnodeMsg;
+  tsVmgmt.msgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = vnodeProcessAlterStreamReq;
+}
+
+static int32_t vnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t     size = sizeof(SVnMgmtMsg) + pMsg->contLen;
+  SVnMgmtMsg *pMgmt = taosAllocateQitem(size);
+  if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
+
+  pMgmt->rpcMsg = *pMsg;
+  pMgmt->rpcMsg.pCont = pMgmt->pCont;
+  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+  taosWriteQitem(tsVmgmt.pQueue, TAOS_QTYPE_RPC, pMgmt);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeProcessMgmtMsg(SRpcMsg *pMsg) {
+  int32_t code = vnodeWriteToMgmtQueue(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+int32_t vnodeInitMgmt() {
+  vnodeInitMgmtReqFp();
+
+  SWorkerPool *pPool = &tsVmgmt.pool;
+  pPool->name = "vmgmt";
+  pPool->startFp = (ProcessStartFp)vnodeProcessMgmtStart;
+  pPool->endFp = (ProcessEndFp)vnodeSendMgmtEnd;
+  pPool->min = 1;
+  pPool->max = 1;
+  if (tWorkerInit(pPool) != 0) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  tsVmgmt.pQueue = tWorkerAllocQueue(pPool, NULL);
+
+  vInfo("vmgmt is initialized, max worker %d", pPool->max);
+  return TSDB_CODE_SUCCESS;
+}
+
+void vnodeCleanupMgmt() {
+  tWorkerFreeQueue(&tsVmgmt.pool, tsVmgmt.pQueue);
+  tWorkerCleanup(&tsVmgmt.pool);
+  tsVmgmt.pQueue = NULL;
+  vInfo("vmgmt is closed");
+}

source/server/vnode/src/vnodeMgmtMsg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "vnodeMain.h"
+#include "vnodeMgmtMsg.h"
+
+static SCreateVnodeMsg* vnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = rpcMsg->pCont;
+  pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
+  pCreate->cfg.dbCfgVersion        = htonl(pCreate->cfg.dbCfgVersion);
+  pCreate->cfg.vgCfgVersion        = htonl(pCreate->cfg.vgCfgVersion);
+  pCreate->cfg.maxTables           = htonl(pCreate->cfg.maxTables);
+  pCreate->cfg.cacheBlockSize      = htonl(pCreate->cfg.cacheBlockSize);
+  pCreate->cfg.totalBlocks         = htonl(pCreate->cfg.totalBlocks);
+  pCreate->cfg.daysPerFile         = htonl(pCreate->cfg.daysPerFile);
+  pCreate->cfg.daysToKeep1         = htonl(pCreate->cfg.daysToKeep1);
+  pCreate->cfg.daysToKeep2         = htonl(pCreate->cfg.daysToKeep2);
+  pCreate->cfg.daysToKeep          = htonl(pCreate->cfg.daysToKeep);
+  pCreate->cfg.minRowsPerFileBlock = htonl(pCreate->cfg.minRowsPerFileBlock);
+  pCreate->cfg.maxRowsPerFileBlock = htonl(pCreate->cfg.maxRowsPerFileBlock);
+  pCreate->cfg.fsyncPeriod         = htonl(pCreate->cfg.fsyncPeriod);
+  pCreate->cfg.commitTime          = htonl(pCreate->cfg.commitTime);
+
+  for (int32_t j = 0; j < pCreate->cfg.vgReplica; ++j) {
+    pCreate->nodes[j].nodeId = htonl(pCreate->nodes[j].nodeId);
+  }
+
+  return pCreate;
+}
+
+int32_t vnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
+  SCreateVnodeMsg *pCreate = vnodeParseVnodeMsg(rpcMsg);
+  SVnode *pVnode = vnodeAcquire(pCreate->cfg.vgId);
+  if (pVnode != NULL) {
+    vDebug("vgId:%d, already exist, return success", pCreate->cfg.vgId);
+    vnodeRelease(pVnode);
+    return TSDB_CODE_SUCCESS;
+  } else {
+    vDebug("vgId:%d, create vnode msg is received", pCreate->cfg.vgId);
+    return vnodeCreate(pCreate);
+  }
+}
+
+int32_t vnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
+  SAlterVnodeMsg *pAlter = vnodeParseVnodeMsg(rpcMsg);
+
+  void *pVnode = vnodeAcquireNotClose(pAlter->cfg.vgId);
+  if (pVnode != NULL) {
+    vDebug("vgId:%d, alter vnode msg is received", pAlter->cfg.vgId);
+    int32_t code = vnodeAlter(pVnode, pAlter);
+    vnodeRelease(pVnode);
+    return code;
+  } else {
+    vInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    return TSDB_CODE_VND_INVALID_VGROUP_ID;
+  }
+}
+
+int32_t vnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) {
+  SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont;
+  pSyncVnode->vgId = htonl(pSyncVnode->vgId);
+
+  return vnodeSync(pSyncVnode->vgId);
+}
+
+int32_t vnodeProcessCompactVnodeMsg(SRpcMsg *rpcMsg) {
+  SCompactVnodeMsg *pCompactVnode = rpcMsg->pCont;
+  pCompactVnode->vgId = htonl(pCompactVnode->vgId);
+  return vnodeCompact(pCompactVnode->vgId);
+}
+
+int32_t vnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
+  SDropVnodeMsg *pDrop = rpcMsg->pCont;
+  pDrop->vgId = htonl(pDrop->vgId);
+
+  return vnodeDrop(pDrop->vgId);
+}
+
+int32_t vnodeProcessAlterStreamReq(SRpcMsg *pMsg) { return 0; }

source/server/vnode/src/vnodeRead.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "tglobal.h"
+// #include "query.h"
+#include "vnodeMain.h"
+#include "vnodeRead.h"
+#include "vnodeReadMsg.h"
+#include "vnodeStatus.h"
+
+static struct {
+  SWorkerPool query;
+  SWorkerPool fetch;
+  int32_t (*msgFp[TSDB_MSG_TYPE_MAX])(SVnode *, struct SReadMsg *);
+} tsVread = {0};
+
+void vnodeStartRead(SVnode *pVnode) {}
+void vnodeStopRead(SVnode *pVnode) {}
+
+void vnodeWaitReadCompleted(SVnode *pVnode) {
+  while (pVnode->queuedRMsg > 0) {
+    vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg);
+    taosMsleep(10);
+  }
+}
+
+static int32_t vnodeWriteToRQueue(SVnode *pVnode, void *pCont, int32_t contLen, int8_t qtype, SRpcMsg *pRpcMsg) {
+  if (pVnode->dropped) {
+    return TSDB_CODE_APP_NOT_READY;
+  }
+
+#if 0
+  if (!((pVnode->role == TAOS_SYNC_ROLE_MASTER) || (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE))) {
+    return TSDB_CODE_APP_NOT_READY;
+  }
+#endif  
+
+  if (!vnodeInReadyStatus(pVnode)) {
+    vDebug("vgId:%d, failed to write into vread queue, vnode status is %s", pVnode->vgId, vnodeStatus[pVnode->status]);
+    return TSDB_CODE_APP_NOT_READY;
+  }
+
+  int32_t   size = sizeof(SReadMsg) + contLen;
+  SReadMsg *pRead = taosAllocateQitem(size);
+  if (pRead == NULL) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  if (pRpcMsg != NULL) {
+    pRead->rpcHandle = pRpcMsg->handle;
+    pRead->rpcAhandle = pRpcMsg->ahandle;
+    pRead->msgType = pRpcMsg->msgType;
+    pRead->code = pRpcMsg->code;
+  }
+
+  if (contLen != 0) {
+    pRead->contLen = contLen;
+    memcpy(pRead->pCont, pCont, contLen);
+  } else {
+    pRead->qhandle = pCont;
+  }
+
+  pRead->qtype = qtype;
+
+  atomic_add_fetch_32(&pVnode->refCount, 1);
+  atomic_add_fetch_32(&pVnode->queuedRMsg, 1);
+
+  if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pRead->msgType == TSDB_MSG_TYPE_FETCH) {
+    return taosWriteQitem(pVnode->fqueue, qtype, pRead);
+  } else {
+    return taosWriteQitem(pVnode->qqueue, qtype, pRead);
+  }
+}
+
+static void vnodeFreeFromRQueue(SVnode *pVnode, SReadMsg *pRead) {
+  atomic_sub_fetch_32(&pVnode->queuedRMsg, 1);
+
+  taosFreeQitem(pRead);
+  vnodeRelease(pVnode);
+}
+
+int32_t vnodeReputPutToRQueue(SVnode *pVnode, void **qhandle, void *ahandle) {
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
+  rpcMsg.ahandle = ahandle;
+
+  int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg);
+  if (code == TSDB_CODE_SUCCESS) {
+    vTrace("QInfo:%p add to vread queue for exec query", *qhandle);
+  }
+
+  return code;
+}
+
+void vnodeProcessReadMsg(SRpcMsg *pMsg) {
+  int32_t queuedMsgNum = 0;
+  int32_t leftLen = pMsg->contLen;
+  int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID;
+  char *  pCont = pMsg->pCont;
+
+  while (leftLen > 0) {
+    SMsgHead *pHead = (SMsgHead *)pCont;
+    pHead->vgId = htonl(pHead->vgId);
+    pHead->contLen = htonl(pHead->contLen);
+
+    assert(pHead->contLen > 0);
+    SVnode *pVnode = vnodeAcquireNotClose(pHead->vgId);
+    if (pVnode != NULL) {
+      code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+      if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
+      vnodeRelease(pVnode);
+    }
+
+    leftLen -= pHead->contLen;
+    pCont -= pHead->contLen;
+  }
+
+  if (queuedMsgNum == 0) {
+    SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code};
+    rpcSendResponse(&rpcRsp);
+  }
+
+  rpcFreeCont(pMsg->pCont);
+}
+
+static void vnodeInitReadMsgFp() {
+  tsVread.msgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg;
+  tsVread.msgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg;
+}
+
+static int32_t vnodeProcessReadStart(SVnode *pVnode, SReadMsg *pRead, int32_t qtype) {
+  int32_t msgType = pRead->msgType;
+  if (tsVread.msgFp[msgType] == NULL) {
+    vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
+    return TSDB_CODE_VND_MSG_NOT_PROCESSED;
+  } else {
+    vTrace("msg:%p, app:%p type:%s will be processed", pRead, pRead->rpcAhandle, taosMsg[msgType]);
+  }
+
+  return (*tsVread.msgFp[msgType])(pVnode, pRead);
+}
+
+static void vnodeSendReadRsp(SReadMsg *pRead, int32_t code) {
+  SRpcMsg rpcRsp = {
+      .handle = pRead->rpcHandle,
+      .pCont = pRead->rspRet.rsp,
+      .contLen = pRead->rspRet.len,
+      .code = code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+}
+
+static void vnodeProcessReadEnd(SVnode *pVnode, SReadMsg *pRead, int32_t qtype, int32_t code) {
+  if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
+    vnodeSendReadRsp(pRead, code);
+  } else {
+    if (code == TSDB_CODE_QRY_HAS_RSP) {
+      vnodeSendReadRsp(pRead, pRead->code);
+    } else {  // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client
+      assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5));
+    }
+  }
+
+  vnodeFreeFromRQueue(pVnode, pRead);
+}
+
+int32_t vnodeInitRead() {
+  vnodeInitReadMsgFp();
+
+  int32_t maxFetchThreads = 4;
+  float   threadsForQuery = MAX(tsNumOfCores * tsRatioOfQueryCores, 1);
+
+  SWorkerPool *pPool = &tsVread.query;
+  pPool->name = "vquery";
+  pPool->startFp = (ProcessStartFp)vnodeProcessReadStart;
+  pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd;
+  pPool->min = (int32_t)threadsForQuery;
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) return -1;
+
+  pPool = &tsVread.fetch;
+  pPool->name = "vfetch";
+  pPool->startFp = (ProcessStartFp)vnodeProcessReadStart;
+  pPool->endFp = (ProcessEndFp)vnodeProcessReadEnd;
+  pPool->min = MIN(maxFetchThreads, tsNumOfCores);
+  pPool->max = pPool->min;
+  if (tWorkerInit(pPool) != 0) return -1;
+
+  vInfo("vread is initialized, max worker %d", pPool->max);
+  return 0;
+}
+
+void vnodeCleanupRead() {
+  tWorkerCleanup(&tsVread.fetch);
+  tWorkerCleanup(&tsVread.query);
+  vInfo("vread is closed");
+}
+
+taos_queue vnodeAllocQueryQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.query, pVnode); }
+
+taos_queue vnodeAllocFetchQueue(SVnode *pVnode) { return tWorkerAllocQueue(&tsVread.fetch, pVnode); }
+
+void vnodeFreeQueryQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.query, pQueue); }
+
+void vnodeFreeFetchQueue(taos_queue pQueue) { tWorkerFreeQueue(&tsVread.fetch, pQueue); }

src/vnode/src/vnodeStatus.c → source/server/vnode/src/vnodeStatus.c

@@ -16,9 +16,9 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
-#include "query.h"
-#include "vnodeStatus.h"
+// #include "query.h"
 #include "vnodeRead.h"
+#include "vnodeStatus.h"
 #include "vnodeWrite.h"
 
 char* vnodeStatus[] = {
@@ -29,30 +29,32 @@ char* vnodeStatus[] = {
   "reset"
 };
 
-bool vnodeSetInitStatus(SVnodeObj* pVnode) {
+bool vnodeSetInitStatus(SVnode* pVnode) {
   pthread_mutex_lock(&pVnode->statusMutex);
   pVnode->status = TAOS_VN_STATUS_INIT;
   pthread_mutex_unlock(&pVnode->statusMutex);
   return true;
 }
 
-bool vnodeSetReadyStatus(SVnodeObj* pVnode) {
+bool vnodeSetReadyStatus(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
   if (pVnode->status == TAOS_VN_STATUS_INIT || pVnode->status == TAOS_VN_STATUS_READY ||
-      pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) {
+      pVnode->status == TAOS_VN_STATUS_UPDATING) {
     pVnode->status = TAOS_VN_STATUS_READY;
     set = true;
   }
 
+#if 0
   qQueryMgmtReOpen(pVnode->qMgmt);
+#endif   
 
   pthread_mutex_unlock(&pVnode->statusMutex);
   return set;
 }
 
-static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
+static bool vnodeSetClosingStatusImp(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -65,7 +67,7 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
   return set;
 }
 
-bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
+bool vnodeSetClosingStatus(SVnode* pVnode) {
   if (pVnode->status == TAOS_VN_STATUS_CLOSING)
     return true;
 
@@ -73,15 +75,17 @@ bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
     taosMsleep(1);
   }
 
+#if 0
   // release local resources only after cutting off outside connections
   qQueryMgmtNotifyClosed(pVnode->qMgmt);
+#endif   
   vnodeWaitReadCompleted(pVnode);
   vnodeWaitWriteCompleted(pVnode);
 
   return true;
 }
 
-bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
+bool vnodeSetUpdatingStatus(SVnode* pVnode) {
   bool set = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -94,35 +98,7 @@ bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
   return set;
 }
 
-static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) {
-  bool set = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) {
-    pVnode->status = TAOS_VN_STATUS_RESET;
-    set = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return set;
-}
-
-bool vnodeSetResetStatus(SVnodeObj* pVnode) {
-  while (!vnodeSetResetStatusImp(pVnode)) {
-    taosMsleep(1);
-  }
-
-  vInfo("vgId:%d, set to reset status", pVnode->vgId);
-
-  // release local resources only after cutting off outside connections
-  qQueryMgmtNotifyClosed(pVnode->qMgmt);
-  vnodeWaitReadCompleted(pVnode);
-  vnodeWaitWriteCompleted(pVnode);
-
-  return true;
-}
-
-bool vnodeInInitStatus(SVnodeObj* pVnode) {
+bool vnodeInInitStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -134,7 +110,7 @@ bool vnodeInInitStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInReadyStatus(SVnodeObj* pVnode) {
+bool vnodeInReadyStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -146,19 +122,7 @@ bool vnodeInReadyStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) {
-  bool in = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_UPDATING) {
-    in = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return in;
-}
-
-bool vnodeInClosingStatus(SVnodeObj* pVnode) {
+bool vnodeInClosingStatus(SVnode* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);
 
@@ -170,14 +134,3 @@ bool vnodeInClosingStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInResetStatus(SVnodeObj* pVnode) {
-  bool in = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_RESET) {
-    in = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return in;
-}

source/server/vnode/test/CMakeLists.txt

+# vnodeMemAllocatorTest
+add_executable(VMATest "")
+target_sources(VMATest
+    PRIVATE
+    "../src/vnodeMemAllocator.c"
+    "vnodeMemAllocatorTest.cpp"
+)
+target_include_directories(VMATest PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc")
+target_link_libraries(VMATest os gtest_main)
\ No newline at end of file

source/server/vnode/tsdb/src/tsdb.c

@@ -15,5 +15,5 @@
 
 #include "tsdb.h"
 
-int tsdbInsert(STsdb *pTsdb, SSubmitReq *pMsg) { return 0; }
+int tsdbInsert(STsdb *tsdb, SSubmitReq *pReq, SSubmitRsp *pRsp) { return 0; }
 int tsdbCommit(STsdb *pTsdb) { return 0; }
\ No newline at end of file