From 4563289f94113c8d62bbaebe2cd720b8d4a59e9a Mon Sep 17 00:00:00 2001
From: slguan <slguan@taosdata.com>
Date: Sat, 15 Feb 2020 17:16:13 +0800
Subject: [PATCH] dnodeSystem.c

---
 src/dnode/inc/dnodeRead.h               |   2 +-
 src/dnode/inc/dnodeShell.h              |  20 +++
 src/dnode/inc/dnodeSystem.h             |  27 +++-
 src/dnode/inc/dnodeVnodeMgmt.h          |  11 +-
 src/dnode/src/dnodeModule.c             |   2 +-
 src/dnode/src/dnodeService.c            |  36 +----
 src/dnode/src/dnodeShell.c              |  50 +++---
 src/dnode/src/dnodeSystem.c             | 201 +++++++++++++++---------
 src/dnode/src/dnodeWrite.c              |   3 +-
 src/inc/dnode.h                         |  17 +-
 src/{modules/http => }/inc/http.h       |   2 +
 src/modules/http/inc/httpSystem.h       |   1 -
 src/modules/http/src/httpSystem.c       |   7 +-
 src/modules/monitor/inc/monitorSystem.h |   8 +-
 src/modules/monitor/src/monitorSystem.c |   4 +-
 src/system/CMakeLists.txt               |   8 -
 src/system/detail/CMakeLists.txt        |  46 ------
 src/system/lite/CMakeLists.txt          |  17 --
 src/vnode/common/inc/vnodePeer.h        |   2 +-
 src/vnode/detail/inc/vnode.h            |   1 -
 src/vnode/detail/inc/vnodeShell.h       |  12 --
 21 files changed, 234 insertions(+), 243 deletions(-)
 rename src/{modules/http => }/inc/http.h (98%)
 delete mode 100644 src/system/CMakeLists.txt
 delete mode 100644 src/system/detail/CMakeLists.txt
 delete mode 100644 src/system/lite/CMakeLists.txt

diff --git a/src/dnode/inc/dnodeRead.h b/src/dnode/inc/dnodeRead.h
index 1cd2e44342..b002d0eefb 100644
--- a/src/dnode/inc/dnodeRead.h
+++ b/src/dnode/inc/dnodeRead.h
@@ -23,7 +23,7 @@ extern "C" {
 /*
  * Dnode handle read messages
  * The processing result is returned by callback function with pShellObj parameter
-*/
+ */
 int32_t dnodeReadData(SQueryMeterMsg *msg, void *pShellObj, void (*callback)(SQueryMeterRsp *rspMsg, void *pShellObj));
 
 #ifdef __cplusplus
diff --git a/src/dnode/inc/dnodeShell.h b/src/dnode/inc/dnodeShell.h
index e4e26512b1..bb04767eb8 100644
--- a/src/dnode/inc/dnodeShell.h
+++ b/src/dnode/inc/dnodeShell.h
@@ -20,6 +20,26 @@
 extern "C" {
 #endif
 
+#include <stdint.h>
+#include <stdint.h>
+#include "dnode.h"
+
+typedef struct {
+  int      sid;
+  int      vnode;
+  uint32_t ip;
+  uint16_t port;
+  int32_t  count;             // track the number of imports
+  int32_t  code;              // track the code of imports
+  int32_t  numOfTotalPoints;  // track the total number of points imported
+  void    *thandle;           // handle from TAOS layer
+  void    *qhandle;
+} SShellObj;
+
+int32_t dnodeInitShell();
+
+//SDnodeStatisInfo dnodeGetStatisInfo()
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/dnode/inc/dnodeSystem.h b/src/dnode/inc/dnodeSystem.h
index eb25357c59..98a9f582f7 100644
--- a/src/dnode/inc/dnodeSystem.h
+++ b/src/dnode/inc/dnodeSystem.h
@@ -23,16 +23,31 @@ extern "C" {
 #include <stdint.h>
 #include <stdbool.h>
 #include <pthread.h>
+#include "dnode.h"
 
-extern pthread_mutex_t dmutex;
-extern bool tsDnodeStopping;
+typedef enum {
+  TSDB_DNODE_RUN_STATUS_INITIALIZE,
+  TSDB_DNODE_RUN_STATUS_RUNING,
+  TSDB_DNODE_RUN_STATUS_STOPPED
+} SDnodeRunStatus;
 
-int  dnodeInitSystem();
+extern int32_t (*dnodeInitPeers)(int32_t numOfThreads);
+extern int32_t (*dnodeCheckSystem)();
+extern int32_t (*dnodeInitStorage)();
+extern void (*dnodeCleanupStorage)();
+extern void (*dnodeParseParameterK)();
+extern int32_t tsMaxQueues;
+
+
+int32_t dnodeInitSystem();
 void dnodeCleanUpSystem();
-void dnodeCheckDbRunning(const char* dir);
+void dnodeInitPlugins();
 
-int vnodeInitStore();
-int vnodeInitPeer(int numOfThreads);
+SDnodeRunStatus dnodeGetRunStatus();
+void dnodeSetRunStatus(SDnodeRunStatus status);
+void dnodeCheckDataDirOpenned(const char *dir);
+void dnodeLockVnodes();
+void dnodeUnLockVnodes();
 
 #ifdef __cplusplus
 }
diff --git a/src/dnode/inc/dnodeVnodeMgmt.h b/src/dnode/inc/dnodeVnodeMgmt.h
index 137461607e..9fee09166b 100644
--- a/src/dnode/inc/dnodeVnodeMgmt.h
+++ b/src/dnode/inc/dnodeVnodeMgmt.h
@@ -33,7 +33,7 @@ int32_t dnodeOpenVnodes();
 /*
  * Close all Vnodes that have been created and opened
  */
-int32_t dnodeCloseVnodes();
+int32_t dnodeCleanupVnodes();
 
 /*
  * Check if vnode already exists
@@ -43,7 +43,8 @@ int32_t dnodeCheckVnodeExist(int vid);
 /*
  * Create vnode with specified configuration and open it
  */
-tsdb_repo_t* dnodeCreateVnode(int vid, SVnodeCfg *cfg);
+//tsdb_repo_t* dnodeCreateVnode(int vid, SVnodeCfg *cfg);
+void* dnodeCreateVnode(int vid, SVnodeCfg *cfg);
 
 /*
  * Modify vnode configuration information
@@ -53,7 +54,7 @@ int32_t dnodeConfigVnode(int vid, SVnodeCfg *cfg);
 /*
  * Modify vnode replication information
  */
-int32_t dnodeConfigVnodePeers(int vid, /*SVpeerCfgMsg *cfg*/);
+int32_t dnodeConfigVnodePeers(int vid/*, SVpeerCfgMsg *cfg*/);
 
 /*
  * Remove vnode from local repository
@@ -63,8 +64,8 @@ int32_t dnodeDropVnode(int vid);
 /*
  * Get the vnode object that has been opened
  */
-tsdb_repo_t* dnodeGetVnode(int vid);
-
+//tsdb_repo_t* dnodeGetVnode(int vid);
+void* dnodeGetVnode(int vid);
 
 #ifdef __cplusplus
 }
diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c
index 5999ff3239..370a96ac61 100644
--- a/src/dnode/src/dnodeModule.c
+++ b/src/dnode/src/dnodeModule.c
@@ -71,7 +71,7 @@ void dnodeCleanUpModules() {
 }
 
 void dnodeProcessModuleStatus(uint32_t status) {
-  if (tsDnodeStopping) {
+  if (tsDnodeRunStatus) {
     return;
   }
 
diff --git a/src/dnode/src/dnodeService.c b/src/dnode/src/dnodeService.c
index 763422b1dd..d7f9721fda 100644
--- a/src/dnode/src/dnodeService.c
+++ b/src/dnode/src/dnodeService.c
@@ -15,39 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "tlog.h"
 #include "tglobalcfg.h"
-#include "vnode.h"
 #include "dnodeSystem.h"
 
-#ifdef CLUSTER
-  #include "dnodeCluster.h"
-  #include "httpAdmin.h"
-  #include "mnodeAccount.h"
-  #include "mnodeBalance.h"
-  #include "mnodeCluster.h"
-  #include "sdbReplica.h"
-  #include "multilevelStorage.h"
-  #include "vnodeCluster.h"
-  #include "vnodeReplica.h"
-  #include "dnodeGrant.h"
-  void init() {
-    dnodeClusterInit();
-    httpAdminInit();
-    mnodeAccountInit();
-    mnodeBalanceInit();
-    mnodeClusterInit();
-    sdbReplicaInit();
-    multilevelStorageInit();
-    vnodeClusterInit();
-    vnodeReplicaInit();
-    dnodeGrantInit();
-  }
-#endif
-
-void dnodeParseParameterKImp() {}
-void (*dnodeParseParameterK)() = dnodeParseParameterKImp;
-
-/* Termination handler */
+/*
+ * Termination handler
+ */
 void signal_handler(int signum, siginfo_t *sigInfo, void *context) {
   if (signum == SIGUSR1) {
     tsCfgDynamicOptions("debugFlag 135");
@@ -70,6 +44,8 @@ void signal_handler(int signum, siginfo_t *sigInfo, void *context) {
 }
 
 int main(int argc, char *argv[]) {
+  dnodeInitPlugins();
+
   // Set global configuration file
   for (int i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c
index 6e43e70742..f02c7849e7 100644
--- a/src/dnode/src/dnodeShell.c
+++ b/src/dnode/src/dnodeShell.c
@@ -15,33 +15,29 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-
+#include "taoserror.h"
+#include "taosdef.h"
 #include "taosmsg.h"
-#include "vnode.h"
-#include "vnodeShell.h"
+#include "tlog.h"
 #include "tschemautil.h"
-
 #include "textbuffer.h"
 #include "trpc.h"
-#include "tscJoinProcess.h"
-#include "vnode.h"
-#include "vnodeRead.h"
-#include "vnodeUtil.h"
-#include "vnodeStore.h"
-#include "vnodeStatus.h"
+#include "dnode.h"
+#include "dnodeSystem.h"
+#include "dnodeShell.h"
 
-extern int tsMaxQueues;
 
-void *      pShellServer = NULL;
-SShellObj **shellList = NULL;
 
-int vnodeProcessRetrieveRequest(char *pMsg, int msgLen, SShellObj *pObj);
-int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj);
-int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj);
+int32_t vnodeProcessRetrieveRequest(char *pMsg, int msgLen, SShellObj *pObj);
+int32_t vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj);
+int32_t vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj);
+
 static void vnodeProcessBatchSubmitTimer(void *param, void *tmrId);
 
-int vnodeSelectReqNum = 0;
-int vnodeInsertReqNum = 0;
+static void *pShellServer = NULL;
+static SShellObj **shellList       = NULL;
+static int32_t   dnodeSelectReqNum = 0;
+static int32_t   dnodeInsertReqNum = 0;
 
 typedef struct {
   int32_t import;
@@ -131,7 +127,7 @@ void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) {
   return pObj;
 }
 
-int vnodeInitShell() {
+int32_t dnodeInitShell() {
   int      size;
   SRpcInit rpcInit;
 
@@ -399,7 +395,7 @@ _query_over:
     vnodeFreeColumnInfo(&pQueryMsg->colList[i]);
   }
 
-  atomic_fetch_add_32(&vnodeSelectReqNum, 1);
+  atomic_fetch_add_32(&dnodeSelectReqNum, 1);
   return ret;
 }
 
@@ -700,7 +696,7 @@ _submit_over:
     ret = vnodeSendShellSubmitRspMsg(pObj, code, pObj->numOfTotalPoints);
   }
 
-  atomic_fetch_add_32(&vnodeInsertReqNum, 1);
+  atomic_fetch_add_32(&dnodeInsertReqNum, 1);
   return ret;
 }
 
@@ -729,3 +725,15 @@ static void vnodeProcessBatchSubmitTimer(void *param, void *tmrId) {
     vnodeSendShellSubmitRspMsg(pShell, code, pShell->numOfTotalPoints);
   }
 }
+
+
+SDnodeStatisInfo dnodeGetStatisInfo() {
+  SDnodeStatisInfo info = {0};
+  if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
+    info.httpReqNum = httpGetReqCount();
+    info.selectReqNum = atomic_exchange_32(&dnodeSelectReqNum, 0);
+    info.insertReqNum = atomic_exchange_32(&dnodeInsertReqNum, 0);
+  }
+
+  return info;
+}
\ No newline at end of file
diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c
index 1203e15ed4..600a0c38d7 100644
--- a/src/dnode/src/dnodeSystem.c
+++ b/src/dnode/src/dnodeSystem.c
@@ -16,43 +16,78 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
+#include "taoserror.h"
+#include "tcrc32c.h"
 #include "tlog.h"
+#include "ttime.h"
 #include "ttimer.h"
+#include "tutil.h"
+#include "http.h"
 #include "dnode.h"
 #include "dnodeMgmt.h"
 #include "dnodeModule.h"
+#include "dnodeShell.h"
 #include "dnodeSystem.h"
-#include "monitorSystem.h"
-#include "httpSystem.h"
-#include "mgmtSystem.h"
-
-#include "vnode.h"
-
-pthread_mutex_t dmutex;
-extern int      vnodeSelectReqNum;
-extern int      vnodeInsertReqNum;
-void *          tsStatusTimer = NULL;
-bool            tsDnodeStopping = false;
-
-// internal global, not configurable
-void *   vnodeTmrCtrl;
-void **  rpcQhandle;
-void *   dmQhandle;
-void *   queryQhandle;
-int      tsVnodePeers = TSDB_VNODES_SUPPORT - 1;
-int      tsMaxQueues;
+#include "dnodeVnodeMgmt.h"
+
+#ifdef CLUSTER
+#include "dnodeCluster.h"
+#include "httpAdmin.h"
+#include "mnodeAccount.h"
+#include "mnodeBalance.h"
+#include "mnodeCluster.h"
+#include "sdbReplica.h"
+#include "multilevelStorage.h"
+#include "vnodeCluster.h"
+#include "vnodeReplica.h"
+#include "dnodeGrant.h"
+#endif
+
+static pthread_mutex_t tsDnodeMutex;
+static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+
+static int32_t dnodeInitRpcQHandle();
+static int32_t dnodeInitQueryQHandle();
+static int32_t dnodeInitTmrCtl();
+
+void     *tsStatusTimer = NULL;
+void     *vnodeTmrCtrl;
+void     **rpcQhandle;
+void     *dmQhandle;
+void     *queryQhandle;
+int32_t  tsVnodePeers   = TSDB_VNODES_SUPPORT - 1;
+int32_t  tsMaxQueues;
 uint32_t tsRebootTime;
 
-int32_t dnodeInitRpcQHandle();
-int32_t dnodeInitQueryQHandle();
-int32_t dnodeInitTmrCtl();
-void dnodeCountRequestImp(SCountInfo *info);
+static void dnodeInitVnodesLock() {
+  pthread_mutex_init(&tsDnodeMutex, NULL);
+}
+
+void dnodeLockVnodes() {
+  pthread_mutex_lock(&tsDnodeMutex);
+}
+
+void dnodeUnLockVnodes() {
+  pthread_mutex_unlock(&tsDnodeMutex);
+}
+
+static void dnodeCleanVnodesLock() {
+  pthread_mutex_destroy(&tsDnodeMutex);
+}
+
+SDnodeRunStatus dnodeGetRunStatus() {
+  return tsDnodeRunStatus;
+}
+
+void dnodeSetRunStatus(SDnodeRunStatus status) {
+  tsDnodeRunStatus = status;
+}
 
 void dnodeCleanUpSystem() {
-  if (tsDnodeStopping) {
+  if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_STOPPED) {
     return;
   } else {
-    tsDnodeStopping = true;
+    dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
   }
 
   if (tsStatusTimer != NULL) {
@@ -61,33 +96,48 @@ void dnodeCleanUpSystem() {
   }
 
   dnodeCleanUpModules();
-
-  vnodeCleanUpVnodes();
-
+  dnodeCleanupVnodes();
   taosCloseLogger();
-
   dnodeCleanupStorage();
+  dnodeCleanVnodesLock();
 }
 
-void dnodeCheckDbRunning(const char* dir) {
+void dnodeCheckDataDirOpenned(const char *dir) {
   char filepath[256] = {0};
   sprintf(filepath, "%s/.running", dir);
-  int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
-  int ret = flock(fd, LOCK_EX | LOCK_NB);
+  int32_t fd  = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+  int32_t ret = flock(fd, LOCK_EX | LOCK_NB);
   if (ret != 0) {
     dError("failed to lock file:%s ret:%d, database may be running, quit", filepath, ret);
     exit(0);
   }
 }
 
-int dnodeInitSystem() {
+void dnodeInitPlugins() {
+#ifdef CLUSTER
+  dnodeClusterInit();
+  httpAdminInit();
+  mnodeAccountInit();
+  mnodeBalanceInit();
+  mnodeClusterInit();
+  sdbReplicaInit();
+  multilevelStorageInit();
+  vnodeClusterInit();
+  vnodeReplicaInit();
+  dnodeGrantInit();
+#endif
+}
+
+int32_t dnodeInitSystem() {
   char        temp[128];
   struct stat dirstat;
 
+  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
+
   taosResolveCRC();
 
   tsRebootTime = taosGetTimestampSec();
-  tscEmbedded = 1;
+  tscEmbedded  = 1;
 
   // Read global configuration.
   tsReadGlobalLogConfig();
@@ -124,7 +174,7 @@ int dnodeInitSystem() {
 
   dnodeAllocModules();
 
-  pthread_mutex_init(&dmutex, NULL);
+  dnodeInitVnodesLock();
 
   dPrint("starting to initialize TDengine ...");
 
@@ -136,7 +186,7 @@ int dnodeInitSystem() {
   if (dnodeCheckSystem() < 0) {
     return -1;
   }
-  
+
   if (dnodeInitModules() < 0) {
     return -1;
   }
@@ -151,14 +201,14 @@ int dnodeInitSystem() {
     return -1;
   }
 
-  if (vnodeInitStore() < 0) {
+  if (dnodeOpenVnodes() < 0) {
     dError("failed to init vnode storage");
     return -1;
   }
 
-  int numOfThreads = (1.0 - tsRatioOfQueryThreads) * tsNumOfCores * tsNumOfThreadsPerCore / 2.0;
+  int32_t numOfThreads = (1.0 - tsRatioOfQueryThreads) * tsNumOfCores * tsNumOfThreadsPerCore / 2.0;
   if (numOfThreads < 1) numOfThreads = 1;
-  if (vnodeInitPeer(numOfThreads) < 0) {
+  if (dnodeInitPeers(numOfThreads) < 0) {
     dError("failed to init vnode peer communication");
     return -1;
   }
@@ -168,40 +218,21 @@ int dnodeInitSystem() {
     return -1;
   }
 
-  if (vnodeInitShell() < 0) {
+  if (dnodeInitShell() < 0) {
     dError("failed to init communication to shell");
     return -1;
   }
 
-  if (vnodeInitVnodes() < 0) {
-    dError("failed to init store");
-    return -1;
-  }
-
-  mnodeCountRequestFp = dnodeCountRequestImp;
-
   dnodeStartModules();
 
+  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
+
   dPrint("TDengine is initialized successfully");
 
   return 0;
 }
 
-void dnodeResetSystem() {
-  dPrint("reset the system ...");
-  for (int vnode = 0; vnode < TSDB_MAX_VNODES; ++vnode) {
-    vnodeRemoveVnode(vnode);
-  }
-  mgmtStopSystem();
-}
-
-void dnodeCountRequestImp(SCountInfo *info) {
-  httpGetReqCount(&info->httpReqNum);
-  info->selectReqNum = atomic_exchange_32(&vnodeSelectReqNum, 0);
-  info->insertReqNum = atomic_exchange_32(&vnodeInsertReqNum, 0);
-}
-
-int dnodeInitStorageImp() {
+int32_t dnodeInitStorageImp() {
   struct stat dirstat;
   strcpy(tsDirectory, dataDir);
   if (stat(dataDir, &dirstat) < 0) {
@@ -218,31 +249,34 @@ int dnodeInitStorageImp() {
 
   sprintf(mgmtDirectory, "%s/mgmt", tsDirectory);
   sprintf(tsDirectory, "%s/tsdb", dataDir);
-  dnodeCheckDbRunning(dataDir);
+  dnodeCheckDataDirOpenned(dataDir);
 
   return 0;
 }
+
 int32_t (*dnodeInitStorage)() = dnodeInitStorageImp;
 
 void dnodeCleanupStorageImp() {}
+
 void (*dnodeCleanupStorage)() = dnodeCleanupStorageImp;
 
-int32_t dnodeInitQueryQHandle() {
-  int numOfThreads = tsRatioOfQueryThreads * tsNumOfCores * tsNumOfThreadsPerCore;
+static int32_t dnodeInitQueryQHandle() {
+  int32_t numOfThreads = tsRatioOfQueryThreads * tsNumOfCores * tsNumOfThreadsPerCore;
   if (numOfThreads < 1) {
     numOfThreads = 1;
   }
 
   int32_t maxQueueSize = tsNumOfVnodesPerCore * tsNumOfCores * tsSessionsPerVnode;
-  dTrace("query task queue initialized, max slot:%d, task threads:%d", maxQueueSize,numOfThreads);
+  dTrace("query task queue initialized, max slot:%d, task threads:%d", maxQueueSize, numOfThreads);
 
   queryQhandle = taosInitSchedulerWithInfo(maxQueueSize, numOfThreads, "query", vnodeTmrCtrl);
 
   return 0;
 }
 
-int32_t dnodeInitTmrCtl() {
-  vnodeTmrCtrl = taosTmrInit(TSDB_MAX_VNODES * (tsVnodePeers + 10) + tsSessionsPerVnode + 1000, 200, 60000, "DND-vnode");
+static int32_t dnodeInitTmrCtl() {
+  vnodeTmrCtrl = taosTmrInit(TSDB_MAX_VNODES * (tsVnodePeers + 10) + tsSessionsPerVnode + 1000, 200, 60000,
+                             "DND-vnode");
   if (vnodeTmrCtrl == NULL) {
     dError("failed to init timer, exit");
     return -1;
@@ -251,22 +285,39 @@ int32_t dnodeInitTmrCtl() {
   return 0;
 }
 
-int32_t dnodeInitRpcQHandle() {
-  tsMaxQueues = (1.0 - tsRatioOfQueryThreads)*tsNumOfCores*tsNumOfThreadsPerCore / 2.0;
-  if (tsMaxQueues < 1) tsMaxQueues = 1;
+static int32_t dnodeInitRpcQHandle() {
+  tsMaxQueues = (1.0 - tsRatioOfQueryThreads) * tsNumOfCores * tsNumOfThreadsPerCore / 2.0;
+  if (tsMaxQueues < 1) {
+    tsMaxQueues = 1;
+  }
 
-  rpcQhandle = malloc(tsMaxQueues*sizeof(void *));
+  rpcQhandle = malloc(tsMaxQueues * sizeof(void *));
 
-  for (int i=0; i< tsMaxQueues; ++i )
+  for (int32_t i = 0; i < tsMaxQueues; ++i) {
     rpcQhandle[i] = taosInitScheduler(tsSessionsPerVnode, 1, "dnode");
+  }
 
   dmQhandle = taosInitScheduler(tsSessionsPerVnode, 1, "mgmt");
 
   return 0;
 }
 
+int32_t dnodeCheckSystemImp() {
+  return 0;
+}
+
+int32_t (*dnodeCheckSystem)() = dnodeCheckSystemImp;
+
+void dnodeParseParameterKImp() {}
+
+void (*dnodeParseParameterK)() = dnodeParseParameterKImp;
+
+int32_t dnodeInitPeersImp(int32_t numOfThreads) {
+  return 0;
+}
+
+int32_t (*dnodeInitPeers)(int32_t numOfThreads) = dnodeInitPeersImp;
+
 
-int dnodeCheckSystemImp() { return 0; }
-int (*dnodeCheckSystem)() = dnodeCheckSystemImp;
 
 
diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c
index 4d51faa334..268c843d0d 100644
--- a/src/dnode/src/dnodeWrite.c
+++ b/src/dnode/src/dnodeWrite.c
@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "dnodeWrite.h"
-
 #include "taoserror.h"
 
 int32_t dnodeCheckTableExist(char *tableId) {
@@ -26,7 +25,7 @@ int32_t dnodeWriteData(SShellSubmitMsg *msg) {
   return 0;
 }
 
-int32_t dnodeCreateNormalTable(SCreateNormalTableMsg *table); {
+int32_t dnodeCreateNormalTable(SCreateNormalTableMsg *table) {
   return 0;
 }
 
diff --git a/src/inc/dnode.h b/src/inc/dnode.h
index d77d583dcc..326cddcba9 100644
--- a/src/inc/dnode.h
+++ b/src/inc/dnode.h
@@ -22,6 +22,13 @@ extern "C" {
 
 #include <stdint.h>
 #include <pthread.h>
+#include "tsched.h"
+
+typedef struct {
+  int32_t selectReqNum;
+  int32_t insertReqNum;
+  int32_t httpReqNum;
+} SDnodeStatisInfo;
 
 typedef struct {
   char  id[20];
@@ -32,7 +39,7 @@ typedef struct {
 } SMgmtObj;
 
 // global variables
-extern pthread_mutex_t dmutex;
+extern uint32_t tsRebootTime;
 
 // dnodeCluster
 extern void (*dnodeStartModules)();
@@ -52,11 +59,15 @@ extern int (*dnodeInitMgmt)();
 extern int32_t (*dnodeInitStorage)();
 extern void (*dnodeCleanupStorage)();
 
-void dnodeCheckDbRunning(const char* dir);
-
+void dnodeCheckDataDirOpenned(const char* dir);
 
 void dnodeProcessMsgFromMgmtImp(SSchedMsg *sched);
 
+
+void dnodeLockVnodes();
+void dnodeUnLockVnodes();
+SDnodeStatisInfo dnodeGetStatisInfo();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/modules/http/inc/http.h b/src/inc/http.h
similarity index 98%
rename from src/modules/http/inc/http.h
rename to src/inc/http.h
index ea209f082c..d77f27d3ea 100644
--- a/src/modules/http/inc/http.h
+++ b/src/inc/http.h
@@ -42,4 +42,6 @@
 #define httpLWarn(...) taosLogWarn(__VA_ARGS__) httpWarn(__VA_ARGS__)
 #define httpLPrint(...) taosLogPrint(__VA_ARGS__) httpPrint(__VA_ARGS__)
 
+int32_t httpGetReqCount();
+
 #endif
diff --git a/src/modules/http/inc/httpSystem.h b/src/modules/http/inc/httpSystem.h
index 0a09c3f762..ccf0f3582d 100644
--- a/src/modules/http/inc/httpSystem.h
+++ b/src/modules/http/inc/httpSystem.h
@@ -20,6 +20,5 @@ int  httpInitSystem();
 int  httpStartSystem();
 void httpStopSystem();
 void httpCleanUpSystem();
-void httpGetReqCount(int32_t *httpConns);
 
 #endif
diff --git a/src/modules/http/src/httpSystem.c b/src/modules/http/src/httpSystem.c
index efc24c15dd..3171aee9ec 100644
--- a/src/modules/http/src/httpSystem.c
+++ b/src/modules/http/src/httpSystem.c
@@ -146,10 +146,9 @@ void httpCleanUpSystem() {
 #endif
 }
 
-void httpGetReqCount(int32_t *httpReqestNum) {
+int32_t httpGetReqCount() {
   if (httpServer != NULL) {
-    *httpReqestNum = atomic_exchange_32(&httpServer->requestNum, 0);
-  } else {
-    *httpReqestNum = 0;
+    return atomic_exchange_32(&httpServer->requestNum, 0);
   }
+  return 0;
 }
diff --git a/src/modules/monitor/inc/monitorSystem.h b/src/modules/monitor/inc/monitorSystem.h
index 8093379dc0..e64b7fad6e 100644
--- a/src/modules/monitor/inc/monitorSystem.h
+++ b/src/modules/monitor/inc/monitorSystem.h
@@ -23,12 +23,6 @@ int  monitorStartSystem();
 void monitorStopSystem();
 void monitorCleanUpSystem();
 
-typedef struct {
-  int selectReqNum;
-  int insertReqNum;
-  int httpReqNum;
-} SCountInfo;
-
-extern void (*mnodeCountRequestFp)(SCountInfo *info);
+extern void (*mnodeCountRequestFp)(SDnodeStatisInfo *info);
 
 #endif
\ No newline at end of file
diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c
index d3bff0e281..a5985b4393 100644
--- a/src/modules/monitor/src/monitorSystem.c
+++ b/src/modules/monitor/src/monitorSystem.c
@@ -77,7 +77,7 @@ void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t ma
                         int64_t totalOutbound, int64_t maxOutbound, int64_t totalDbs, int64_t maxDbs,
                         int64_t totalUsers, int64_t maxUsers, int64_t totalStreams, int64_t maxStreams,
                         int64_t totalConns, int64_t maxConns, int8_t accessState);
-void (*mnodeCountRequestFp)(SCountInfo *info) = NULL;
+void (*mnodeCountRequestFp)(SDnodeStatisInfo *info) = NULL;
 void monitorExecuteSQL(char *sql);
 
 void monitorCheckDiskUsage(void *para, void *unused) {
@@ -333,7 +333,7 @@ int monitorBuildBandSql(char *sql) {
 }
 
 int monitorBuildReqSql(char *sql) {
-  SCountInfo info;
+  SDnodeStatisInfo info;
   info.httpReqNum = info.insertReqNum = info.selectReqNum = 0;
   (*mnodeCountRequestFp)(&info);
 
diff --git a/src/system/CMakeLists.txt b/src/system/CMakeLists.txt
deleted file mode 100644
index 516b9e85e2..0000000000
--- a/src/system/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-ADD_SUBDIRECTORY(detail)
-
-IF (TD_EDGE)
-  ADD_SUBDIRECTORY(lite)
-ENDIF ()
\ No newline at end of file
diff --git a/src/system/detail/CMakeLists.txt b/src/system/detail/CMakeLists.txt
deleted file mode 100644
index 6268b97f91..0000000000
--- a/src/system/detail/CMakeLists.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/monitor/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/util/cluster/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(./src SRC)
-  LIST(REMOVE_ITEM SRC ./src/vnodeFileUtil.c)
-  LIST(REMOVE_ITEM SRC ./src/taosGrant.c)
-
-  ADD_EXECUTABLE(taosd ${SRC})
-
-  IF (TD_PAGMODE_LITE)
-    TARGET_LINK_LIBRARIES(taosd taos trpc tutil sdb monitor pthread http)
-  ELSE ()
-    TARGET_LINK_LIBRARIES(taosd taos_static trpc tutil sdb monitor pthread http)
-  ENDIF ()
-
-  IF (TD_EDGE)
-    TARGET_LINK_LIBRARIES(taosd taosd_edge)
-  ELSE ()
-    TARGET_LINK_LIBRARIES(taosd taosd_cluster)
-  ENDIF ()
-
-  SET(PREPARE_ENV_CMD "prepare_env_cmd")
-  SET(PREPARE_ENV_TARGET "prepare_env_target")
-  ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
-    POST_BUILD 
-    COMMAND echo "make test directory"
-    DEPENDS taosd
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/
-    COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
-    COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
-    COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
-    COMMENT "prepare taosd environment")
-  ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})
-
-ENDIF ()
diff --git a/src/system/lite/CMakeLists.txt b/src/system/lite/CMakeLists.txt
deleted file mode 100644
index 8c648747e7..0000000000
--- a/src/system/lite/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/http/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/modules/monitor/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/system/detail/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/util/cluster/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(inc)
-
-  AUX_SOURCE_DIRECTORY(./src SRC)
-  ADD_LIBRARY(taosd_edge ${SRC})
-ENDIF ()
diff --git a/src/vnode/common/inc/vnodePeer.h b/src/vnode/common/inc/vnodePeer.h
index 6c69326fbc..47aaa40377 100644
--- a/src/vnode/common/inc/vnodePeer.h
+++ b/src/vnode/common/inc/vnodePeer.h
@@ -27,7 +27,7 @@ extern "C" {
 /*
  * Initialize the resources
  */
-int32_t vnodeInitPeer(int numOfThreads);
+int32_t vnodeInitPeers(int numOfThreads);
 
 /*
  * Free the resources
diff --git a/src/vnode/detail/inc/vnode.h b/src/vnode/detail/inc/vnode.h
index 158f4b2479..3b502a9f79 100644
--- a/src/vnode/detail/inc/vnode.h
+++ b/src/vnode/detail/inc/vnode.h
@@ -302,7 +302,6 @@ typedef struct {
 
 // internal globals
 extern int        tsMeterSizeOnFile;
-extern uint32_t   tsRebootTime;
 extern void **    rpcQhandle;
 
 extern void *     queryQhandle;
diff --git a/src/vnode/detail/inc/vnodeShell.h b/src/vnode/detail/inc/vnodeShell.h
index 25646fbb1a..b03634ca8f 100644
--- a/src/vnode/detail/inc/vnodeShell.h
+++ b/src/vnode/detail/inc/vnodeShell.h
@@ -22,18 +22,6 @@ extern "C" {
 
 #include "os.h"
 
-typedef struct {
-  int      sid;
-  int      vnode;
-  uint32_t ip;
-  uint16_t port;
-  int      count;             // track the number of imports
-  int      code;              // track the code of imports
-  int      numOfTotalPoints;  // track the total number of points imported
-  void *   thandle;           // handle from TAOS layer
-  void *   qhandle;
-} SShellObj;
-
 #ifdef __cplusplus
 }
 #endif
-- 
GitLab