TD-2324

src/common/inc/tglobal.h

@@ -35,6 +35,7 @@ extern int32_t  tsNumOfMnodes;
 extern int32_t  tsEnableVnodeBak;
 extern int32_t  tsEnableTelemetryReporting;
 extern char     tsEmail[];
+extern char     tsArbitrator[];
 
 // common
 extern int      tsRpcTimer;

src/dnode/inc/dnodeCfg.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCfg();
 void    dnodeCleanupCfg();

src/dnode/inc/dnodeCheck.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitCheck();
 void    dnodeCleanupCheck();

src/dnode/inc/dnodeEps.h

@@ -19,8 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
-#include "taosmsg.h"
+#include "dnodeInt.h"
 
 int32_t dnodeInitEps();
 void    dnodeCleanupEps();

src/dnode/inc/dnodeInt.h

@@ -19,8 +19,13 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
+#include "taoserror.h"
+#include "taosmsg.h"
 #include "tlog.h"
+#include "trpc.h"
+#include "tglobal.h"
+#include "dnode.h"
+#include "vnode.h"
 
 extern int32_t dDebugFlag;
 

src/dnode/inc/dnodeMInfos.h

@@ -19,8 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-
-#include "taosmsg.h"
+#include "dnodeInt.h"
 
 int32_t dnodeInitMInfos();
 void    dnodeCleanupMInfos();
@@ -29,6 +28,10 @@ void    dnodeUpdateEpSetForPeer(SRpcEpSet *epSet);
 void    dnodeGetMInfos(SMnodeInfos *minfos);
 bool    dnodeIsMasterEp(char *ep);
 
+void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
+void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
+void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
+
 #ifdef __cplusplus
 }
 #endif

src/dnode/inc/dnodeMPeer.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMPeer();
 void    dnodeCleanupMPeer();

src/dnode/inc/dnodeMRead.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMRead();
 void    dnodeCleanupMRead();

src/dnode/inc/dnodeMWrite.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitMWrite();
 void    dnodeCleanupMWrite();

src/dnode/inc/dnodeMain.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitSystem();
 void    dnodeCleanUpSystem();

src/dnode/inc/dnodeModule.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitModules();
 void    dnodeStartModules();

src/dnode/inc/dnodePeer.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitServer();
 void    dnodeCleanupServer();

src/dnode/inc/dnodeShell.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitShell();
 void    dnodeCleanupShell();

src/dnode/inc/dnodeStep.h

@@ -19,7 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "dnode.h"
+#include "dnodeInt.h"
 
 int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize);
 void    dnodeStepCleanup(SStep *pSteps, int32_t stepSize);

src/dnode/inc/dnodeTelemetry.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitTelemetry();
 void    dnodeCleanupTelemetry();

src/dnode/inc/dnodeMgmt.h → src/dnode/inc/dnodeVMgmt.h

@@ -13,32 +13,17 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_MGMT_H
-#define TDENGINE_DNODE_MGMT_H
+#ifndef TDENGINE_DNODE_VMGMT_H
+#define TDENGINE_DNODE_VMGMT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
-#include "trpc.h"
-
-int32_t dnodeInitMgmt();
-void    dnodeCleanupMgmt();
-int32_t dnodeInitMgmtTimer();
-void    dnodeCleanupMgmtTimer();
-void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
-
-void*   dnodeGetVnode(int32_t vgId);
-int32_t dnodeGetVnodeStatus(void *pVnode);
-void*   dnodeGetVnodeRworker(void *pVnode);
-void*   dnodeGetVnodeWworker(void *pVnode);
-void*   dnodeGetVnodeWal(void *pVnode);
-void*   dnodeGetVnodeTsdb(void *pVnode);
-void    dnodeReleaseVnode(void *pVnode);
-
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
+int32_t dnodeInitVMgmt();
+void    dnodeCleanupVMgmt();
+void    dnodeDispatchToVMgmtQueue(SRpcMsg *rpcMsg);
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeVRead.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVRead();
 void    dnodeCleanupVRead();

src/dnode/inc/dnodeVWrite.h

@@ -19,6 +19,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "dnodeInt.h"
 
 int32_t dnodeInitVWrite();
 void    dnodeCleanupVWrite();

src/dnode/inc/dnodeVnodes.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_VNODES_H
+#define TDENGINE_DNODE_VNODES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "dnodeInt.h"
+
+int32_t dnodeInitVnodes();
+void    dnodeCleanupVnodes();
+int32_t dnodeInitTimer();
+void    dnodeCleanupTimer();
+void    dnodeSendStatusMsgToMnode();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/src/dnodeCfg.c

@@ -16,9 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeCfg.h"
 
 static SDnodeCfg tsCfg = {0};

src/dnode/src/dnodeCheck.c

@@ -15,8 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
-#include "dnodeInt.h"
 #include "dnodeCheck.h"
 
 typedef struct {

src/dnode/src/dnodeEps.c

@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "hash.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeEps.h"
 
 static SDnodeEps *tsEps = NULL;

src/dnode/src/dnodeMInfos.c

@@ -16,10 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "cJSON.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeMInfos.h"
 
 static SMnodeInfos tsMInfos;
@@ -286,3 +283,25 @@ static int32_t dnodeWriteMInfos() {
   dInfo("successed to write %s", file);
   return 0;
 }
+
+void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
+  SRpcConnInfo connInfo = {0};
+  rpcGetConnInfo(rpcMsg->handle, &connInfo);
+
+  SRpcEpSet epSet = {0};
+  if (forShell) {
+    dnodeGetEpSetForShell(&epSet);
+  } else {
+    dnodeGetEpSetForPeer(&epSet);
+  }
+  
+  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
+         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
+
+  for (int32_t i = 0; i < epSet.numOfEps; ++i) {
+    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
+    epSet.port[i] = htons(epSet.port[i]);
+  }
+
+  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
+}

src/dnode/src/dnodeMPeer.c

@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMWrite.h"
 
 typedef struct {

src/dnode/src/dnodeMRead.c

@@ -15,16 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "tqueue.h"
 #include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMRead.h"
 
 typedef struct {

src/dnode/src/dnodeMWrite.c

@@ -15,17 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tutil.h"
 #include "ttimer.h"
 #include "tqueue.h"
-#include "twal.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
+#include "dnodeMInfos.h"
 #include "dnodeMWrite.h"
 
 typedef struct {

src/dnode/src/dnodeMain.c

@@ -16,15 +16,10 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taos.h"
-#include "tutil.h"
 #include "tconfig.h"
-#include "tglobal.h"
 #include "tfile.h"
 #include "twal.h"
-#include "trpc.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "tfs.h"
 #include "dnodePeer.h"
 #include "dnodeModule.h"
 #include "dnodeEps.h"
@@ -33,12 +28,13 @@
 #include "dnodeCheck.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
+#include "dnodeVMgmt.h"
+#include "dnodeVnodes.h"
 #include "dnodeMRead.h"
 #include "dnodeMWrite.h"
 #include "dnodeMPeer.h"
 #include "dnodeShell.h"
 #include "dnodeTelemetry.h"
-#include "tfs.h"
 
 static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
 
@@ -49,27 +45,28 @@ static void    dnodeCheckDataDirOpenned(char *dir);
 static int     dnodeCreateDir(const char *dir);
 
 static SStep tsDnodeSteps[] = {
-  {"tfile",     tfInit,              tfCleanup},
-  {"rpc",       rpcInit,             rpcCleanup},
-  {"globalcfg" ,taosCheckGlobalCfg,  NULL},
-  {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
-  {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
-  {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},
-  {"mnodeinfos",dnodeInitMInfos,     dnodeCleanupMInfos},
-  {"wal",       walInit,             walCleanUp},
-  {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
-  {"vread",     dnodeInitVRead,      dnodeCleanupVRead},
-  {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
-  {"mread",     dnodeInitMRead,      dnodeCleanupMRead},
-  {"mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
-  {"mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
-  {"client",    dnodeInitClient,     dnodeCleanupClient},
-  {"server",    dnodeInitServer,     dnodeCleanupServer},
-  {"mgmt",      dnodeInitMgmt,       dnodeCleanupMgmt},
-  {"modules",   dnodeInitModules,    dnodeCleanupModules},
-  {"mgmt-tmr",  dnodeInitMgmtTimer,  dnodeCleanupMgmtTimer},
-  {"shell",     dnodeInitShell,      dnodeCleanupShell},
-  {"telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
+  {"dnode-tfile",     tfInit,              tfCleanup},
+  {"dnode-rpc",       rpcInit,             rpcCleanup},
+  {"dnode-globalcfg", taosCheckGlobalCfg,  NULL},
+  {"dnode-storage",   dnodeInitStorage,    dnodeCleanupStorage},
+  {"dnode-cfg",       dnodeInitCfg,        dnodeCleanupCfg},
+  {"dnode-eps",       dnodeInitEps,        dnodeCleanupEps},
+  {"dnode-minfos",    dnodeInitMInfos,     dnodeCleanupMInfos},
+  {"dnode-wal",       walInit,             walCleanUp},
+  {"dnode-check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
+  {"dnode-vread",     dnodeInitVRead,      dnodeCleanupVRead},
+  {"dnode-vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
+  {"dnode-vmgmt",     dnodeInitVMgmt,      dnodeCleanupVMgmt},
+  {"dnode-mread",     dnodeInitMRead,      dnodeCleanupMRead},
+  {"dnode-mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
+  {"dnode-mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
+  {"dnode-client",    dnodeInitClient,     dnodeCleanupClient},
+  {"dnode-server",    dnodeInitServer,     dnodeCleanupServer},
+  {"dnode-vnodes",    dnodeInitVnodes,     dnodeCleanupVnodes},
+  {"dnode-modules",   dnodeInitModules,    dnodeCleanupModules},
+  {"dnode-tmr",       dnodeInitTimer,      dnodeCleanupTimer},
+  {"dnode-shell",     dnodeInitShell,      dnodeCleanupShell},
+  {"dnode-telemetry", dnodeInitTelemetry,  dnodeCleanupTelemetry},
 };
 
 static int dnodeCreateDir(const char *dir) {

src/dnode/src/dnodeModule.c

@@ -15,15 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
 #include "http.h"
 #include "tmqtt.h"
 #include "monitor.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeModule.h"
 
 typedef struct {

src/dnode/src/dnodePeer.c

@@ -21,12 +21,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
+#include "dnodeVMgmt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMPeer.h"
 #include "dnodeMInfos.h"
@@ -45,12 +41,12 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_TABLE]  = dnodeDispatchToVWriteQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_STABLE]  = dnodeDispatchToVWriteQueue;
 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToMgmtQueue; 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeDispatchToVMgmtQueue; 
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToVMgmtQueue;
 
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;

src/dnode/src/dnodeShell.c

@@ -15,15 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosdef.h"
-#include "taosmsg.h"
-#include "tglobal.h"
-#include "tutil.h"
 #include "http.h"
 #include "mnode.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
 #include "dnodeMRead.h"

src/dnode/src/dnodeStep.c

@@ -15,9 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "dnodeInt.h"
 #include "dnodeStep.h"
 
 static SStartupStep tsStartupStep;
@@ -32,14 +29,7 @@ void dnodeSendStartupStep(SRpcMsg *pMsg) {
   dInfo("nettest msg is received, cont:%s", (char *)pMsg->pCont);
 
   SStartupStep *pStep = rpcMallocCont(sizeof(SStartupStep));
-#if 1
   memcpy(pStep, &tsStartupStep, sizeof(SStartupStep));
-#else
-  static int32_t step = 0;
-  sprintf(pStep->name, "module:%d", step++);
-  sprintf(pStep->desc, "step:%d", step++);
-  if (step > 10) pStep->finished = 1;
-#endif
 
   dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStep->name, pStep->desc, pStep->finished);
 

src/dnode/src/dnodeSystem.c

@@ -16,9 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "tgrant.h"
-#include "tutil.h"
-#include "tglobal.h"
-#include "dnodeInt.h"
 #include "dnodeMain.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);

src/dnode/src/dnodeTelemetry.c

@@ -15,9 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "tglobal.h"
-#include "tutil.h"
 #include "osTime.h"
 #include "tsocket.h"
 #include "tbuffer.h"
@@ -32,8 +29,6 @@
 #include "mnodeTable.h"
 #include "mnodeSdb.h"
 #include "mnodeAcct.h"
-#include "dnode.h"
-#include "dnodeInt.h"
 #include "dnodeTelemetry.h"
 
 static tsem_t tsExitSem;

src/dnode/src/dnodeVMgmt.c

@@ -15,56 +15,19 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "cJSON.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "ttimer.h"
-#include "tsdb.h"
-#include "twal.h"
 #include "tqueue.h"
-#include "tsync.h"
-#include "ttimer.h"
-#include "tbn.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "vnode.h"
-#include "mnode.h"
-#include "dnodeInt.h"
-#include "dnodeMgmt.h"
-#include "dnodeEps.h"
-#include "dnodeCfg.h"
-#include "dnodeMInfos.h"
-#include "dnodeVRead.h"
-#include "dnodeVWrite.h"
-#include "dnodeModule.h"
-
-typedef struct {
-  pthread_t thread;
-  int32_t   threadIndex;
-  int32_t   failed;
-  int32_t   opened;
-  int32_t   vnodeNum;
-  int32_t * vnodeList;
-} SOpenVnodeThread;
+#include "dnodeVMgmt.h"
 
 typedef struct {
   SRpcMsg rpcMsg;
   char    pCont[];
 } SMgmtMsg;
 
-void *            tsDnodeTmr = NULL;
-static void *     tsStatusTimer = NULL;
-static uint32_t   tsRebootTime;
 static taos_qset  tsMgmtQset = NULL;
 static taos_queue tsMgmtQueue = NULL;
 static pthread_t  tsQthread;
 
-static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
-static void   dnodeSendStatusMsg(void *handle, void *tmrId);
-static void  *dnodeProcessMgmtQueue(void *param);
-
-static int32_t  dnodeOpenVnodes();
-static void     dnodeCloseVnodes();
+static void *  dnodeProcessMgmtQueue(void *param);
 static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
 static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
 static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
@@ -73,7 +36,7 @@ static int32_t  dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg);
 static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg);
 static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg);
 
-int32_t dnodeInitMgmt() {
+int32_t dnodeInitVMgmt() {
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE]  = dnodeProcessAlterVnodeMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE]   = dnodeProcessDropVnodeMsg;
@@ -81,27 +44,18 @@ int32_t dnodeInitMgmt() {
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg;
   dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeProcessCreateMnodeMsg;
 
-  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP,  dnodeProcessStatusRsp);
-  tsRebootTime = taosGetTimestampSec();
-
   int32_t code = vnodeInitMgmt();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
-    return -1;
-  }
+  if (code != TSDB_CODE_SUCCESS) return -1;
 
-  // create the queue and thread to handle the message 
   tsMgmtQset = taosOpenQset();
   if (tsMgmtQset == NULL) {
     dError("failed to create the mgmt queue set");
-    dnodeCleanupMgmt();
     return -1;
   }
 
   tsMgmtQueue = taosOpenQueue();
   if (tsMgmtQueue == NULL) {
     dError("failed to create the mgmt queue");
-    dnodeCleanupMgmt();
     return -1;
   }
 
@@ -115,62 +69,20 @@ int32_t dnodeInitMgmt() {
   pthread_attr_destroy(&thAttr);
   if (code != 0) {
     dError("failed to create thread to process mgmt queue, reason:%s", strerror(errno));
-    dnodeCleanupMgmt();
-    return -1; 
-  }
-
-  code = dnodeOpenVnodes();
-  if (code != TSDB_CODE_SUCCESS) {
-    dnodeCleanupMgmt();
     return -1;
   }
 
   dInfo("dnode mgmt is initialized");
- 
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t dnodeInitMgmtTimer() {
-  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
-  if (tsDnodeTmr == NULL) {
-    dError("failed to init dnode timer");
-    dnodeCleanupMgmt();
-    return -1;
-  }
-
-  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  dInfo("dnode mgmt timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeSendStatusMsgToMnode() {
-  if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
-    dInfo("force send status msg to mnode");
-    taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
-  }
-}
-
-void dnodeCleanupMgmtTimer() {
-  if (tsStatusTimer != NULL) {
-    taosTmrStopA(&tsStatusTimer);
-    tsStatusTimer = NULL;
-  }
-
-  if (tsDnodeTmr != NULL) {
-    taosTmrCleanUp(tsDnodeTmr);
-    tsDnodeTmr = NULL;
-  }
-}
-
-void dnodeCleanupMgmt() {
-  dnodeCleanupMgmtTimer();
-  dnodeCloseVnodes();
-
+void dnodeCleanupVMgmt() {
   if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);
   if (tsQthread) pthread_join(tsQthread, NULL);
 
   if (tsMgmtQueue) taosCloseQueue(tsMgmtQueue);
   if (tsMgmtQset) taosCloseQset(tsMgmtQset);
+
   tsMgmtQset = NULL;
   tsMgmtQueue = NULL;
 
@@ -180,9 +92,7 @@ void dnodeCleanupMgmt() {
 static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
   int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
   SMgmtMsg *pMgmt = taosAllocateQitem(size);
-  if (pMgmt == NULL) {
-    return TSDB_CODE_DND_OUT_OF_MEMORY;
-  }
+  if (pMgmt == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
 
   pMgmt->rpcMsg = *pMsg;
   pMgmt->rpcMsg.pCont = pMgmt->pCont;
@@ -192,7 +102,7 @@ static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToVMgmtQueue(SRpcMsg *pMsg) {
   int32_t code = dnodeWriteToMgmtQueue(pMsg);
   if (code != TSDB_CODE_SUCCESS) {
     SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
@@ -233,135 +143,7 @@ static void *dnodeProcessMgmtQueue(void *param) {
   return NULL;
 }
 
-static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  DIR *dir = opendir(tsVnodeDir);
-  if (dir == NULL) {
-    return TSDB_CODE_DND_NO_WRITE_ACCESS;
-  }
-
-  *numOfVnodes = 0;
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-    if (de->d_type & DT_DIR) {
-      if (strncmp("vnode", de->d_name, 5) != 0) continue;
-      int32_t vnode = atoi(de->d_name + 5);
-      if (vnode == 0) continue;
-
-      (*numOfVnodes)++;
-
-      if (*numOfVnodes >= TSDB_MAX_VNODES) {
-        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
-        continue;
-      } else {
-        vnodeList[*numOfVnodes - 1] = vnode;
-      }
-    }
-  }
-  closedir(dir);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void *dnodeOpenVnode(void *param) {
-  SOpenVnodeThread *pThread = param;
-  
-  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
-
-  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
-    int32_t vgId = pThread->vnodeList[v];
-    if (vnodeOpen(vgId) < 0) {
-      dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
-      pThread->failed++;
-    } else {
-      dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
-      pThread->opened++;
-    }
-  }
-
-  dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
-         pThread->failed);
-  return NULL;
-}
-
-static int32_t dnodeOpenVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
-  int32_t numOfVnodes = 0;
-  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return status;
-  }
-
-  int32_t threadNum = tsNumOfCores;
-  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
-  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
-  for (int32_t t = 0; t < threadNum; ++t) {
-    threads[t].threadIndex = t;
-    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
-  }
-
-  for (int32_t v = 0; v < numOfVnodes; ++v) {
-    int32_t t = v % threadNum;
-    SOpenVnodeThread *pThread = &threads[t];
-    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
-  }
-
-  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
-
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum == 0) continue;
-
-    pthread_attr_t thAttr;
-    pthread_attr_init(&thAttr);
-    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
-    if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
-      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
-    }
-
-    pthread_attr_destroy(&thAttr);
-  }
-
-  int32_t openVnodes = 0;
-  int32_t failedVnodes = 0;
-  for (int32_t t = 0; t < threadNum; ++t) {
-    SOpenVnodeThread *pThread = &threads[t];
-    if (pThread->vnodeNum > 0 && pThread->thread) {
-      pthread_join(pThread->thread, NULL);
-    }
-    openVnodes += pThread->opened;
-    failedVnodes += pThread->failed;
-    free(pThread->vnodeList);
-  }
-
-  free(threads);
-  dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static void dnodeCloseVnodes() {
-  int32_t vnodeList[TSDB_MAX_VNODES]= {0};
-  int32_t numOfVnodes = 0;
-  int32_t status;
-
-  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
-
-  if (status != TSDB_CODE_SUCCESS) {
-    dInfo("get dnode list failed");
-    return;
-  }
-
-  for (int32_t i = 0; i < numOfVnodes; ++i) {
-    vnodeClose(vnodeList[i]);
-  }
-
-  dInfo("total vnodes:%d are all closed", numOfVnodes);
-}
-
-static void* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
+static SCreateVnodeMsg* dnodeParseVnodeMsg(SRpcMsg *rpcMsg) {
   SCreateVnodeMsg *pCreate = rpcMsg->pCont;
   pCreate->cfg.vgId                = htonl(pCreate->cfg.vgId);
   pCreate->cfg.cfgVersion          = htonl(pCreate->cfg.cfgVersion);
@@ -421,15 +203,6 @@ static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) {
 }
 
 static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg) {
-//  SAlterStreamMsg *pStream = pCont;
-//  pStream->uid    = htobe64(pStream->uid);
-//  pStream->stime  = htobe64(pStream->stime);
-//  pStream->vnode  = htonl(pStream->vnode);
-//  pStream->sid    = htonl(pStream->sid);
-//  pStream->status = htonl(pStream->status);
-//
-//  int32_t code = dnodeCreateStream(pStream);
-
   return 0;
 }
 
@@ -461,110 +234,3 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) {
 
   return TSDB_CODE_SUCCESS;
 }
-
-static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
-  if (pMsg->code != TSDB_CODE_SUCCESS) {
-    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    return;
-  }
-
-  SStatusRsp *pStatusRsp = pMsg->pCont;
-  SMnodeInfos *minfos = &pStatusRsp->mnodes;
-  dnodeUpdateMInfos(minfos);
-
-  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
-  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
-  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
-  pCfg->dnodeId = htonl(pCfg->dnodeId);
-  dnodeUpdateCfg(pCfg);
-
-  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
-
-  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
-  dnodeUpdateEps(pEps);
-
-  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-}
-
-static void dnodeSendStatusMsg(void *handle, void *tmrId) {
-  if (tsDnodeTmr == NULL) {
-    dError("dnode timer is already released");
-    return;
-  }
-
-  if (tsStatusTimer == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to start status timer");
-    return;
-  }
-
-  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-  SStatusMsg *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
-    dError("failed to malloc status message");
-    return;
-  }
-
-  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
-  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
-  pStatus->version          = htonl(tsVersion);
-  pStatus->lastReboot       = htonl(tsRebootTime);
-  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
-  pStatus->diskAvailable    = tsAvailDataDirGB;
-  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
-  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
-
-  // fill cluster cfg parameters
-  pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
-  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
-  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
-  pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
-  pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);
-  pStatus->clusterCfg.maxtablesPerVnode  = htonl(tsMaxTablePerVnode);
-  pStatus->clusterCfg.maxVgroupsPerDb    = htonl(tsMaxVgroupsPerDb);
-  tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
-  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
-  pStatus->clusterCfg.checkTime = 0;
-  char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
-  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
-  
-  vnodeBuildStatusMsg(pStatus);
-  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
-  pStatus->openVnodes = htons(pStatus->openVnodes);
-  
-  SRpcMsg rpcMsg = {
-    .pCont   = pStatus,
-    .contLen = contLen,
-    .msgType = TSDB_MSG_TYPE_DM_STATUS
-  };
-
-  SRpcEpSet epSet;
-  dnodeGetEpSetForPeer(&epSet);
-  dnodeSendMsgToDnode(&epSet, &rpcMsg);
-}
-
-void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
-  SRpcConnInfo connInfo = {0};
-  rpcGetConnInfo(rpcMsg->handle, &connInfo);
-
-  SRpcEpSet epSet = {0};
-  if (forShell) {
-    dnodeGetEpSetForShell(&epSet);
-  } else {
-    dnodeGetEpSetForPeer(&epSet);
-  }
-  
-  dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType],
-         taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse);
-
-  for (int i = 0; i < epSet.numOfEps; ++i) {
-    dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]);
-    epSet.port[i] = htons(epSet.port[i]);
-  }
-
-  rpcSendRedirectRsp(rpcMsg->handle, &epSet);
-}

src/dnode/src/dnodeVRead.c

@@ -15,12 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "tqueue.h"
-#include "vnode.h"
-#include "dnodeInt.h"
+#include "dnodeVRead.h"
 
 typedef struct {
   pthread_t thread;    // thread

src/dnode/src/dnodeVWrite.c

@@ -15,13 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "taoserror.h"
-#include "taosmsg.h"
-#include "tglobal.h"
 #include "tqueue.h"
-#include "twal.h"
-#include "vnode.h"
-#include "dnodeInt.h"
+#include "dnodeVWrite.h"
 
 typedef struct {
   taos_qall qall;

src/dnode/src/dnodeVnodes.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "ttimer.h"
+#include "dnodeEps.h"
+#include "dnodeCfg.h"
+#include "dnodeMInfos.h"
+#include "dnodeVnodes.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   threadIndex;
+  int32_t   failed;
+  int32_t   opened;
+  int32_t   vnodeNum;
+  int32_t * vnodeList;
+} SOpenVnodeThread;
+
+void *          tsDnodeTmr = NULL;
+static void *   tsStatusTimer = NULL;
+static uint32_t tsRebootTime = 0;
+
+static void dnodeSendStatusMsg(void *handle, void *tmrId);
+static void dnodeProcessStatusRsp(SRpcMsg *pMsg);
+
+int32_t dnodeInitTimer() {
+  tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
+  if (tsDnodeTmr == NULL) {
+    dError("failed to init dnode timer");
+    return -1;
+  }
+
+  dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp);
+
+  tsRebootTime = taosGetTimestampSec();
+  taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
+
+  dInfo("dnode timer is initialized");
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeCleanupTimer() {
+  if (tsStatusTimer != NULL) {
+    taosTmrStopA(&tsStatusTimer);
+    tsStatusTimer = NULL;
+  }
+
+  if (tsDnodeTmr != NULL) {
+    taosTmrCleanUp(tsDnodeTmr);
+    tsDnodeTmr = NULL;
+  }
+}
+
+static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
+  DIR *dir = opendir(tsVnodeDir);
+  if (dir == NULL) return TSDB_CODE_DND_NO_WRITE_ACCESS;
+
+  *numOfVnodes = 0;
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+    if (de->d_type & DT_DIR) {
+      if (strncmp("vnode", de->d_name, 5) != 0) continue;
+      int32_t vnode = atoi(de->d_name + 5);
+      if (vnode == 0) continue;
+
+      (*numOfVnodes)++;
+
+      if (*numOfVnodes >= TSDB_MAX_VNODES) {
+        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
+        continue;
+      } else {
+        vnodeList[*numOfVnodes - 1] = vnode;
+      }
+    }
+  }
+  closedir(dir);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void *dnodeOpenVnode(void *param) {
+  SOpenVnodeThread *pThread = param;
+  
+  dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
+
+  for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
+    int32_t vgId = pThread->vnodeList[v];
+    if (vnodeOpen(vgId) < 0) {
+      dError("vgId:%d, failed to open vnode by thread:%d", vgId, pThread->threadIndex);
+      pThread->failed++;
+    } else {
+      dDebug("vgId:%d, is openned by thread:%d", vgId, pThread->threadIndex);
+      pThread->opened++;
+    }
+  }
+
+  dDebug("thread:%d, total vnodes:%d, openned:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
+         pThread->failed);
+  return NULL;
+}
+
+int32_t dnodeInitVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES] = {0};
+  int32_t numOfVnodes = 0;
+  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
+
+  if (status != TSDB_CODE_SUCCESS) {
+    dInfo("get dnode list failed");
+    return status;
+  }
+
+  int32_t threadNum = tsNumOfCores;
+  int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
+  SOpenVnodeThread *threads = calloc(threadNum, sizeof(SOpenVnodeThread));
+  for (int32_t t = 0; t < threadNum; ++t) {
+    threads[t].threadIndex = t;
+    threads[t].vnodeList = calloc(vnodesPerThread, sizeof(int32_t));
+  }
+
+  for (int32_t v = 0; v < numOfVnodes; ++v) {
+    int32_t t = v % threadNum;
+    SOpenVnodeThread *pThread = &threads[t];
+    pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
+  }
+
+  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum == 0) continue;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+    if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnode, pThread) != 0) {
+      dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+  }
+
+  int32_t openVnodes = 0;
+  int32_t failedVnodes = 0;
+  for (int32_t t = 0; t < threadNum; ++t) {
+    SOpenVnodeThread *pThread = &threads[t];
+    if (pThread->vnodeNum > 0 && pThread->thread) {
+      pthread_join(pThread->thread, NULL);
+    }
+    openVnodes += pThread->opened;
+    failedVnodes += pThread->failed;
+    free(pThread->vnodeList);
+  }
+
+  free(threads);
+  dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeCleanupVnodes() {
+  int32_t vnodeList[TSDB_MAX_VNODES]= {0};
+  int32_t numOfVnodes = 0;
+  int32_t status;
+
+  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
+
+  if (status != TSDB_CODE_SUCCESS) {
+    dInfo("get dnode list failed");
+    return;
+  }
+
+  for (int32_t i = 0; i < numOfVnodes; ++i) {
+    vnodeClose(vnodeList[i]);
+  }
+
+  dInfo("total vnodes:%d are all closed", numOfVnodes);
+}
+
+static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
+  if (pMsg->code != TSDB_CODE_SUCCESS) {
+    dError("status rsp is received, error:%s", tstrerror(pMsg->code));
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    return;
+  }
+
+  SStatusRsp *pStatusRsp = pMsg->pCont;
+  SMnodeInfos *minfos = &pStatusRsp->mnodes;
+  dnodeUpdateMInfos(minfos);
+
+  SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
+  pCfg->numOfVnodes = htonl(pCfg->numOfVnodes);
+  pCfg->moduleStatus = htonl(pCfg->moduleStatus);
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
+  dnodeUpdateCfg(pCfg);
+
+  vnodeSetAccess(pStatusRsp->vgAccess, pCfg->numOfVnodes);
+
+  SDnodeEps *pEps = (SDnodeEps *)((char *)pStatusRsp->vgAccess + pCfg->numOfVnodes * sizeof(SVgroupAccess));
+  dnodeUpdateEps(pEps);
+
+  taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+}
+
+static void dnodeSendStatusMsg(void *handle, void *tmrId) {
+  if (tsDnodeTmr == NULL) {
+    dError("dnode timer is already released");
+    return;
+  }
+
+  if (tsStatusTimer == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to start status timer");
+    return;
+  }
+
+  int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
+  SStatusMsg *pStatus = rpcMallocCont(contLen);
+  if (pStatus == NULL) {
+    taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer);
+    dError("failed to malloc status message");
+    return;
+  }
+
+  dnodeGetCfg(&pStatus->dnodeId, pStatus->clusterId);
+  pStatus->dnodeId          = htonl(dnodeGetDnodeId());
+  pStatus->version          = htonl(tsVersion);
+  pStatus->lastReboot       = htonl(tsRebootTime);
+  pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
+  pStatus->diskAvailable    = tsAvailDataDirGB;
+  pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
+  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
+
+  // fill cluster cfg parameters
+  pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
+  pStatus->clusterCfg.enableBalance      = htonl(tsEnableBalance);
+  pStatus->clusterCfg.mnodeEqualVnodeNum = htonl(tsMnodeEqualVnodeNum);
+  pStatus->clusterCfg.offlineThreshold   = htonl(tsOfflineThreshold);
+  pStatus->clusterCfg.statusInterval     = htonl(tsStatusInterval);
+  pStatus->clusterCfg.maxtablesPerVnode  = htonl(tsMaxTablePerVnode);
+  pStatus->clusterCfg.maxVgroupsPerDb    = htonl(tsMaxVgroupsPerDb);
+  tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN);
+  tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
+  pStatus->clusterCfg.checkTime = 0;
+  char timestr[32] = "1970-01-01 00:00:00.00";
+  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
+  tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);  
+  
+  vnodeBuildStatusMsg(pStatus);
+  contLen = sizeof(SStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad);
+  pStatus->openVnodes = htons(pStatus->openVnodes);
+  
+  SRpcMsg rpcMsg = {
+    .pCont   = pStatus,
+    .contLen = contLen,
+    .msgType = TSDB_MSG_TYPE_DM_STATUS
+  };
+
+  SRpcEpSet epSet;
+  dnodeGetEpSetForPeer(&epSet);
+  dnodeSendMsgToDnode(&epSet, &rpcMsg);
+}
+
+void dnodeSendStatusMsgToMnode() {
+  if (tsDnodeTmr != NULL && tsStatusTimer != NULL) {
+    dInfo("force send status msg to mnode");
+    taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer);
+  }
+}