Merge pull request #2067 from taosdata/hotfix/mgmt

Hotfix/mgmt

packaging/cfg/taos.cfg

@@ -155,7 +155,7 @@
 # maxVnodeConnections   10000
 
 # mnode take into account while balance, for cluster version only 
-# mgmtEqualVnodeNum     4
+# mnodeEqualVnodeNum     4
 
 # number of seconds allowed for a dnode to be offline, for cluster version only 
 # offlineThreshold      864000

src/client/inc/tsclient.h

@@ -284,8 +284,8 @@ typedef struct {
 typedef struct STscObj {
   void *             signature;
   void *             pTimer;
-  char               mgmtIp[TSDB_USER_LEN];
-  uint16_t           mgmtPort;
+  char               mnodeIp[TSDB_USER_LEN];
+  uint16_t           mnodePort;
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
   char               acctId[TSDB_DB_NAME_LEN];

src/client/src/TSDBJNIConnector.c

@@ -23,7 +23,7 @@
 #include "ttime.h"
 
 #define jniError(...)  { if (jniDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR JNI ", jniDebugFlag, __VA_ARGS__); }}
-#define jniWarn(...)   { if (jniDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  JNI ", jniDebugFlag, __VA_ARGS__); }}
+#define jniWarn(...)   { if (jniDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN JNI ", jniDebugFlag, __VA_ARGS__); }}
 #define jniTrace(...)  { if (jniDebugFlag & DEBUG_TRACE) { taosPrintLog("JNI ", jniDebugFlag, __VA_ARGS__); }}
 #define jniPrint(...)  { taosPrintLog("JNI ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
 

src/client/src/tscParseInsert.c

@@ -1361,7 +1361,7 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
   /*
    * the pRes->code may be modified or released by another thread in tscTableMetaCallBack function,
    * so do NOT use pRes->code to determine if the getTableMeta/getMetricMeta function
-   * invokes new threads to get data from mgmt node or simply retrieves data from cache.
+   * invokes new threads to get data from mnode or simply retrieves data from cache.
    *
    * do NOT assign return code to pRes->code for the same reason since it may be released by another thread
    * pRes->code = ret;

src/client/src/tscServer.c

@@ -60,10 +60,10 @@ static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) {
 
 void tscPrintMgmtIp() {
   if (tscMgmtIpSet.numOfIps <= 0) {
-    tscError("invalid mgmt IP list:%d", tscMgmtIpSet.numOfIps);
+    tscError("invalid mnode IP list:%d", tscMgmtIpSet.numOfIps);
   } else {
     for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
-      tscTrace("mgmt index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
+      tscTrace("mnode index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
     }
   }
 }
@@ -78,7 +78,7 @@ void tscSetMgmtIpList(SRpcIpSet *pIpList) {
 
 void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
   tscMgmtIpSet = *pIpSet;
-  tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse);
+  tscTrace("mnode IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse);
   for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
     tscTrace("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
   }

src/client/src/tscSql.c

@@ -88,7 +88,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
 
   strncpy(pObj->user, user, TSDB_USER_LEN);
   taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass);
-  pObj->mgmtPort = port ? port : tsDnodeShellPort;
+  pObj->mnodePort = port ? port : tsDnodeShellPort;
 
   if (db) {
     int32_t len = strlen(db);

src/client/src/tscSystem.c

@@ -116,7 +116,7 @@ void taos_init_imp() {
   }
 
   if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) {
-    tscError("failed to init mgmt IP list");
+    tscError("failed to init mnode IP list");
     return;
   } 
 

src/common/inc/tglobal.h

@@ -100,7 +100,7 @@ extern int32_t tsMaxMgmtConnections;
 
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
-extern int32_t tsMgmtEqualVnodeNum;
+extern int32_t tsMnodeEqualVnodeNum;
 
 extern int32_t tsEnableHttpModule;
 extern int32_t tsEnableMqttModule;

src/common/inc/tulog.h

@@ -26,7 +26,7 @@ extern int32_t uDebugFlag;
 extern int32_t tscEmbedded;
 
 #define uError(...)       { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR UTL ", uDebugFlag, __VA_ARGS__); }}
-#define uWarn(...)        { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  UTL ", uDebugFlag, __VA_ARGS__); }}
+#define uWarn(...)        { if (uDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN UTL ", uDebugFlag, __VA_ARGS__); }}
 #define uTrace(...)       { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }}
 #define uDump(x, y)       { if (uDebugFlag & DEBUG_DUMP)  { taosDumpData(x, y); }}
 #define uPrint(...)       { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }

src/common/src/tglobal.c

@@ -117,7 +117,7 @@ int32_t tsMaxVnodeConnections = 10000;
 
 int32_t tsBalanceInterval = 300;  // seconds
 int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
-int32_t tsMgmtEqualVnodeNum = 4;
+int32_t tsMnodeEqualVnodeNum = 4;
 
 int32_t tsEnableHttpModule = 1;
 int32_t tsEnableMqttModule = 0;   // not finished yet, not started it by default
@@ -427,7 +427,7 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  // 0-any; 1-mgmt; 2-dnode
+  // 0-any; 1-mnode; 2-dnode
   cfg.option = "alternativeRole";
   cfg.ptr = &tsAlternativeRole;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -875,8 +875,8 @@ static void doInitGlobalConfig() {
   taosInitConfigOption(cfg);
 
   // module configs
-  cfg.option = "mgmtEqualVnodeNum";
-  cfg.ptr = &tsMgmtEqualVnodeNum;
+  cfg.option = "mnodeEqualVnodeNum";
+  cfg.ptr = &tsMnodeEqualVnodeNum;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;

src/dnode/inc/dnodeInt.h

@@ -25,7 +25,7 @@ extern "C" {
 extern int32_t dDebugFlag;
 
 #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR DND ", 255, __VA_ARGS__); }}
-#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  DND ", dDebugFlag, __VA_ARGS__); }}
+#define dWarn(...)  { if (dDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN DND ", dDebugFlag, __VA_ARGS__); }}
 #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
 #define dPrint(...) { taosPrintLog("DND ", 255, __VA_ARGS__); }
 

src/mnode/inc/mgmtDClient.h → src/dnode/inc/dnodeMPeer.h

@@ -13,17 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DCLIENT_H
-#define TDENGINE_MGMT_DCLIENT_H
+#ifndef TDENGINE_DNODE_MPEER_H
+#define TDENGINE_DNODE_MPEER_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t mgmtInitDClient();
-void    mgmtCleanupDClient();
-void    mgmtAddDClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
-void    mgmtSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg);
+int32_t dnodeInitMnodePeer();
+void    dnodeCleanupMnodePeer();
+int32_t dnodeAllocateMnodePqueue();
+void    dnodeFreeMnodePqueue();
+void    dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtUser.h → src/dnode/inc/dnodeMRead.h

@@ -13,23 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_USER_H
-#define TDENGINE_MGMT_USER_H
+#ifndef TDENGINE_DNODE_MREAD_H
+#define TDENGINE_DNODE_MREAD_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mgmtDef.h"
 
-int32_t   mgmtInitUsers();
-void      mgmtCleanUpUsers();
-SUserObj *mgmtGetUser(char *name);
-void *    mgmtGetNextUser(void *pIter, SUserObj **pUser);
-void      mgmtIncUserRef(SUserObj *pUser);
-void      mgmtDecUserRef(SUserObj *pUser);
-SUserObj *mgmtGetUserFromConn(void *pConn);
-int32_t   mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
-void      mgmtDropAllUsers(SAcctObj *pAcct);
+int32_t dnodeInitMnodeRead();
+void    dnodeCleanupMnodeRead();
+int32_t dnodeAllocateMnodeRqueue();
+void    dnodeFreeMnodeRqueue();
+void    dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeMWrite.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_DNODE_MWRITE_H
+#define TDENGINE_DNODE_MWRITE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dnodeInitMnodeWrite();
+void    dnodeCleanupMnodeWrite();
+int32_t dnodeAllocateMnodeWqueue();
+void    dnodeFreeMnodeWqueue();
+void    dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/dnode/inc/dnodeMain.h

@@ -21,7 +21,7 @@ extern "C" {
 #endif
 
 int32_t dnodeInitSystem();
-void dnodeCleanUpSystem();
+void    dnodeCleanUpSystem();
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeMgmt.h

@@ -32,6 +32,10 @@ void*   dnodeGetVnodeWal(void *pVnode);
 void*   dnodeGetVnodeTsdb(void *pVnode);
 void    dnodeReleaseVnode(void *pVnode);
 
+void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
+void    dnodeGetMnodeIpSetForPeer(void *ipSet);
+void    dnodeGetMnodeIpSetForShell(void *ipSet);
+
 #ifdef __cplusplus
 }
 #endif

src/dnode/inc/dnodePeer.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_DNODE_H
-#define TDENGINE_DNODE_DNODE_H
+#ifndef TDENGINE_DNODE_PEER_H
+#define TDENGINE_DNODE_PEER_H
 
 #ifdef __cplusplus
 extern "C" {

src/dnode/inc/dnodeVRead.h

@@ -13,15 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_READ_H
-#define TDENGINE_DNODE_READ_H
+#ifndef TDENGINE_DNODE_VREAD_H
+#define TDENGINE_DNODE_VREAD_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t dnodeInitRead();
-void    dnodeCleanupRead();
+int32_t dnodeInitVnodeRead();
+void    dnodeCleanupVnodeRead();
 void    dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg);
 
 #ifdef __cplusplus

src/dnode/inc/dnodeVWrite.h

@@ -13,17 +13,16 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_WRITE_H
-#define TDENGINE_DNODE_WRITE_H
+#ifndef TDENGINE_DNODE_VWRITE_H
+#define TDENGINE_DNODE_VWRITE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t dnodeInitWrite();
-void    dnodeCleanupWrite();
+int32_t dnodeInitVnodeWrite();
+void    dnodeCleanupVnodeWrite();
 void    dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg);
-void    dnodeSendWriteResponse(void *pVnode, void *param, int32_t code);
 
 #ifdef __cplusplus
 }

src/dnode/src/dnodeMPeer.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "taosmsg.h"
+#include "tutil.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMgmt.h"
+#include "dnodeMWrite.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SMPeerWorker;
+
+typedef struct {
+  int32_t       num;
+  SMPeerWorker *peerWorker;
+} SMPeerWorkerPool;
+
+static SMPeerWorkerPool tsMPeerPool;
+static taos_qset        tsMPeerQset;
+static taos_queue       tsMPeerQueue;
+
+static void *dnodeProcessMnodePeerQueue(void *param);
+
+int32_t dnodeInitMnodePeer() {
+  tsMPeerQset = taosOpenQset();
+  
+  tsMPeerPool.num = 1;
+  tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.num);
+
+  if (tsMPeerPool.peerWorker == NULL) return -1;
+  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+    pWorker->workerId = i;
+  }
+
+  dPrint("dnode mpeer is opened");
+  return 0;
+}
+
+void dnodeCleanupMnodePeer() {
+  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+    if (pWorker->thread) {
+      taosQsetThreadResume(tsMPeerQset);
+    }
+  }
+
+  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+    if (pWorker->thread) {
+      pthread_join(pWorker->thread, NULL);
+    }
+  }
+
+  dPrint("dnode mpeer is closed");
+}
+
+int32_t dnodeAllocateMnodePqueue() {
+  tsMPeerQueue = taosOpenQueue();
+  if (tsMPeerQueue == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
+
+  for (int32_t i = 0; i < tsMPeerPool.num; ++i) {
+    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
+      dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+    dTrace("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.num);
+  }
+
+  dTrace("dnode mpeer queue:%p is allocated", tsMPeerQueue);
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeFreeMnodePqueue() {
+  taosCloseQueue(tsMPeerQueue);
+  tsMPeerQueue = NULL;
+}
+
+void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
+  if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
+    dnodeSendRedirectMsg(pMsg, false);
+    return;
+  }
+
+  SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
+  mnodeCreateMsg(pPeer, pMsg);
+  taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
+}
+
+static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
+  mnodeCleanupMsg(pPeer);
+  taosFreeQitem(pPeer);
+}
+
+static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+
+  SRpcMsg rpcRsp = {
+    .handle  = pPeer->rpcMsg.handle,
+    .pCont   = pPeer->rpcRsp.rsp,
+    .contLen = pPeer->rpcRsp.len,
+    .code    = code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+  dnodeFreeMnodePeerMsg(pPeer);
+}
+
+static void *dnodeProcessMnodePeerQueue(void *param) {
+  SMnodeMsg *pPeerMsg;
+  int32_t    type;
+  void *     unUsed;
+  
+  while (1) {
+    if (taosReadQitemFromQset(tsMPeerQset, &type, (void **)&pPeerMsg, &unUsed) == 0) {
+      dTrace("dnodeProcessMnodePeerQueue: got no message from qset, exiting...");
+      break;
+    }
+
+    dTrace("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);    
+    int32_t code = mnodeProcessPeerReq(pPeerMsg);    
+    dnodeSendRpcMnodePeerRsp(pPeerMsg, code);    
+  }
+
+  return NULL;
+}

src/dnode/src/dnodeMRead.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "taosmsg.h"
+#include "tutil.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMgmt.h"
+#include "dnodeMRead.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SMReadWorker;
+
+typedef struct {
+  int32_t       num;
+  SMReadWorker *readWorker;
+} SMReadWorkerPool;
+
+static SMReadWorkerPool tsMReadPool;
+static taos_qset        tsMReadQset;
+static taos_queue       tsMReadQueue;
+
+static void *dnodeProcessMnodeReadQueue(void *param);
+
+int32_t dnodeInitMnodeRead() {
+  tsMReadQset = taosOpenQset();
+
+  tsMReadPool.num = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+  tsMReadPool.num = MAX(2, tsMReadPool.num);
+  tsMReadPool.num = MIN(4, tsMReadPool.num);
+  tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.num);
+
+  if (tsMReadPool.readWorker == NULL) return -1;
+  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+    pWorker->workerId = i;
+  }
+
+  dPrint("dnode mread is opened");
+  return 0;
+}
+
+void dnodeCleanupMnodeRead() {
+  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+    if (pWorker->thread) {
+      taosQsetThreadResume(tsMReadQset);
+    }
+  }
+
+  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+    if (pWorker->thread) {
+      pthread_join(pWorker->thread, NULL);
+    }
+  }
+
+  taosCloseQset(tsMReadQset);
+  free(tsMReadPool.readWorker);
+
+  dPrint("dnode mread is closed");
+}
+
+int32_t dnodeAllocateMnodeRqueue() {
+  tsMReadQueue = taosOpenQueue();
+  if (tsMReadQueue == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
+
+  for (int32_t i = 0; i < tsMReadPool.num; ++i) {
+    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
+      dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+    dTrace("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.num);
+  }
+
+  dTrace("dnode mread queue:%p is allocated", tsMReadQueue);
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeFreeMnodeRqueue() {
+  taosCloseQueue(tsMReadQueue);
+  tsMReadQueue = NULL;
+}
+
+void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
+  if (!mnodeIsRunning() || tsMReadQueue == NULL) {
+    dnodeSendRedirectMsg(pMsg, true);
+    return;
+  }
+
+  SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
+  mnodeCreateMsg(pRead, pMsg);
+  taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
+}
+
+static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
+  mnodeCleanupMsg(pRead);
+  taosFreeQitem(pRead);
+}
+
+static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+  if (code == TSDB_CODE_ACTION_NEED_REPROCESSED) {
+    // may be a auto create req, should put into write queue
+    dnodeReprocessMnodeWriteMsg(pRead);
+    return;
+  }
+
+  SRpcMsg rpcRsp = {
+    .handle  = pRead->rpcMsg.handle,
+    .pCont   = pRead->rpcRsp.rsp,
+    .contLen = pRead->rpcRsp.len,
+    .code    = code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+  dnodeFreeMnodeReadMsg(pRead);
+}
+
+static void *dnodeProcessMnodeReadQueue(void *param) {
+  SMnodeMsg *pReadMsg;
+  int32_t    type;
+  void *     unUsed;
+  
+  while (1) {
+    if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
+      dTrace("dnodeProcessMnodeReadQueue: got no message from qset, exiting...");
+      break;
+    }
+
+    dTrace("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);    
+    int32_t code = mnodeProcessRead(pReadMsg);    
+    dnodeSendRpcMnodeReadRsp(pReadMsg, code);   
+  }
+
+  return NULL;
+}

src/dnode/src/dnodeMWrite.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "taosmsg.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "dnodeInt.h"
+#include "dnodeMgmt.h"
+#include "dnodeMWrite.h"
+
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SMWriteWorker;
+
+typedef struct {
+  int32_t        num;
+  SMWriteWorker *writeWorker;
+} SMWriteWorkerPool;
+
+static SMWriteWorkerPool tsMWritePool;
+static taos_qset         tsMWriteQset;
+static taos_queue        tsMWriteQueue;
+extern void *            tsDnodeTmr;
+
+static void *dnodeProcessMnodeWriteQueue(void *param);
+
+int32_t dnodeInitMnodeWrite() {
+  tsMWriteQset = taosOpenQset();
+  
+  tsMWritePool.num = 1;
+  tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.num);
+
+  if (tsMWritePool.writeWorker == NULL) return -1;
+  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+    pWorker->workerId = i;
+  }
+
+  dPrint("dnode mwrite is opened");
+  return 0;
+}
+
+void dnodeCleanupMnodeWrite() {
+  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+    if (pWorker->thread) {
+      taosQsetThreadResume(tsMWriteQset);
+    }
+  }
+
+  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+    if (pWorker->thread) {
+      pthread_join(pWorker->thread, NULL);
+    }
+  }
+
+  dPrint("dnode mwrite is closed");
+}
+
+int32_t dnodeAllocateMnodeWqueue() {
+  tsMWriteQueue = taosOpenQueue();
+  if (tsMWriteQueue == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
+
+  taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
+
+  for (int32_t i = 0; i < tsMWritePool.num; ++i) {
+    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
+      dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+    dTrace("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.num);
+  }
+
+  dTrace("dnode mwrite queue:%p is allocated", tsMWriteQueue);
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeFreeMnodeWqueue() {
+  taosCloseQueue(tsMWriteQueue);
+  tsMWriteQueue = NULL;
+}
+
+void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
+  if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
+    dnodeSendRedirectMsg(pMsg, true);
+    return;
+  }
+
+  SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
+  mnodeCreateMsg(pWrite, pMsg);
+  taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+}
+
+static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+  mnodeCleanupMsg(pWrite);
+  taosFreeQitem(pWrite);
+}
+
+void dnodeSendRpcMnodeWriteRsp(void *pRaw, int32_t code) {
+  SMnodeMsg *pWrite = pRaw;
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
+  if (code == TSDB_CODE_ACTION_NEED_REPROCESSED) {
+    dnodeReprocessMnodeWriteMsg(pWrite);
+    return;
+  }
+
+  SRpcMsg rpcRsp = {
+    .handle  = pWrite->rpcMsg.handle,
+    .pCont   = pWrite->rpcRsp.rsp,
+    .contLen = pWrite->rpcRsp.len,
+    .code    = code,
+  };
+
+  rpcSendResponse(&rpcRsp);
+  dnodeFreeMnodeWriteMsg(pWrite);
+}
+
+static void *dnodeProcessMnodeWriteQueue(void *param) {
+  SMnodeMsg *pWriteMsg;
+  int32_t    type;
+  void *     unUsed;
+  
+  while (1) {
+    if (taosReadQitemFromQset(tsMWriteQset, &type, (void **)&pWriteMsg, &unUsed) == 0) {
+      dTrace("dnodeProcessMnodeWriteQueue: got no message from qset, exiting...");
+      break;
+    }
+
+    dTrace("%p, msg:%s will be processed in mwrite queue", pWriteMsg->rpcMsg.ahandle, taosMsg[pWriteMsg->rpcMsg.msgType]);    
+    int32_t code = mnodeProcessWrite(pWriteMsg);    
+    dnodeSendRpcMnodeWriteRsp(pWriteMsg, code);
+  }
+
+  return NULL;
+}
+
+void dnodeReprocessMnodeWriteMsg(void *pMsg) {
+  SMnodeMsg *pWrite = pMsg;
+
+  if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
+    dnodeSendRedirectMsg(pMsg, true);
+    dnodeFreeMnodeWriteMsg(pWrite);
+  } else {    
+    taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+  }
+}
+
+static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
+  dnodeReprocessMnodeWriteMsg(param);
+}
+
+void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
+  SMnodeMsg *mnodeMsg = pMsg;
+  void *unUsed = NULL;
+  taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
+}
\ No newline at end of file

src/dnode/src/dnodeMain.c

@@ -25,8 +25,11 @@
 #include "dnodePeer.h"
 #include "dnodeModule.h"
 #include "dnodeVRead.h"
-#include "dnodeShell.h"
 #include "dnodeVWrite.h"
+#include "dnodeMRead.h"
+#include "dnodeMWrite.h"
+#include "dnodeMPeer.h"
+#include "dnodeShell.h"
 
 static int32_t dnodeInitStorage();
 static void dnodeCleanupStorage();
@@ -65,8 +68,11 @@ int32_t dnodeInitSystem() {
   dPrint("start to initialize TDengine on %s", tsLocalEp);
 
   if (dnodeInitStorage() != 0) return -1;
-  if (dnodeInitRead() != 0) return -1;
-  if (dnodeInitWrite() != 0) return -1;
+  if (dnodeInitVnodeRead() != 0) return -1;
+  if (dnodeInitVnodeWrite() != 0) return -1;
+  if (dnodeInitMnodeRead() != 0) return -1;
+  if (dnodeInitMnodeWrite() != 0) return -1;
+  if (dnodeInitMnodePeer() != 0) return -1;
   if (dnodeInitClient() != 0) return -1;
   if (dnodeInitServer() != 0) return -1;
   if (dnodeInitMgmt() != 0) return -1;
@@ -89,8 +95,11 @@ void dnodeCleanUpSystem() {
     dnodeCleanupMgmt();
     dnodeCleanupServer();
     dnodeCleanupClient();
-    dnodeCleanupWrite();
-    dnodeCleanupRead();
+    dnodeCleanupMnodePeer();
+    dnodeCleanupMnodeWrite();
+    dnodeCleanupMnodeRead();
+    dnodeCleanupVnodeWrite();
+    dnodeCleanupVnodeRead();
     dnodeCleanupStorage();
     taos_cleanup();
     taosCloseLog();

src/dnode/src/dnodeMgmt.c

@@ -20,7 +20,6 @@
 #include "taosmsg.h"
 #include "ttime.h"
 #include "ttimer.h"
-#include "trpc.h"
 #include "tsdb.h"
 #include "twal.h"
 #include "tsync.h"
@@ -39,6 +38,15 @@
 
 #define MPEER_CONTENT_LEN 2000
 
+void *          tsDnodeTmr = NULL;
+static void *   tsStatusTimer = NULL;
+static uint32_t tsRebootTime;
+
+static SRpcIpSet     tsDMnodeIpSetForPeer = {0};
+static SRpcIpSet     tsDMnodeIpSetForShell = {0};
+static SDMMnodeInfos tsDMnodeInfos = {0};
+static SDMDnodeCfg   tsDnodeCfg = {0};
+
 static void   dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes);
 static bool   dnodeReadMnodeInfos();
 static void   dnodeSaveMnodeInfos();
@@ -48,14 +56,6 @@ static void   dnodeSaveDnodeCfg();
 static void   dnodeProcessStatusRsp(SRpcMsg *pMsg);
 static void   dnodeSendStatusMsg(void *handle, void *tmrId);
 
-static void    *tsDnodeTmr = NULL;
-static void    *tsStatusTimer = NULL;
-static uint32_t tsRebootTime;
-
-static SRpcIpSet     tsMnodeIpSet  = {0};
-static SDMMnodeInfos tsMnodeInfos = {0};
-static SDMDnodeCfg   tsDnodeCfg = {0};
-
 static int32_t  dnodeOpenVnodes();
 static void     dnodeCloseVnodes();
 static int32_t  dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
@@ -81,22 +81,40 @@ int32_t dnodeInitMgmt() {
   }
   
   if (!dnodeReadMnodeInfos()) {
-    memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
-    memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
-    tsMnodeIpSet.numOfIps = 1;
-    taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
-    tsMnodeIpSet.port[0] += TSDB_PORT_DNODEDNODE;
+    memset(&tsDMnodeIpSetForPeer, 0, sizeof(SRpcIpSet));
+    memset(&tsDMnodeIpSetForShell, 0, sizeof(SRpcIpSet));
+    memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos));
+
+    tsDMnodeIpSetForPeer.numOfIps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsDMnodeIpSetForPeer.fqdn[0], &tsDMnodeIpSetForPeer.port[0]);
+    tsDMnodeIpSetForPeer.port[0] += TSDB_PORT_DNODEDNODE;
+    
+    tsDMnodeIpSetForShell.numOfIps = 1;
+    taosGetFqdnPortFromEp(tsFirst, tsDMnodeIpSetForShell.fqdn[0], &tsDMnodeIpSetForShell.port[0]);
+    tsDMnodeIpSetForShell.port[0] += TSDB_PORT_DNODESHELL;
+    
     if (strcmp(tsSecond, tsFirst) != 0) {
-      tsMnodeIpSet.numOfIps = 2;
-      taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
-      tsMnodeIpSet.port[1] += TSDB_PORT_DNODEDNODE;
+      tsDMnodeIpSetForPeer.numOfIps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsDMnodeIpSetForPeer.fqdn[1], &tsDMnodeIpSetForPeer.port[1]);
+      tsDMnodeIpSetForPeer.port[1] += TSDB_PORT_DNODEDNODE;
+
+      tsDMnodeIpSetForShell.numOfIps = 2;
+      taosGetFqdnPortFromEp(tsSecond, tsDMnodeIpSetForShell.fqdn[1], &tsDMnodeIpSetForShell.port[1]);
+      tsDMnodeIpSetForShell.port[1] += TSDB_PORT_DNODESHELL;
     }
   } else {
-    tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
-    tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
-    for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-      taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
-      tsMnodeIpSet.port[i] += TSDB_PORT_DNODEDNODE;
+    tsDMnodeIpSetForPeer.inUse = tsDMnodeInfos.inUse;
+    tsDMnodeIpSetForPeer.numOfIps = tsDMnodeInfos.nodeNum;
+    for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+      taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSetForPeer.fqdn[i], &tsDMnodeIpSetForPeer.port[i]);
+      tsDMnodeIpSetForPeer.port[i] += TSDB_PORT_DNODEDNODE;
+    }
+
+    tsDMnodeIpSetForShell.inUse = tsDMnodeInfos.inUse;
+    tsDMnodeIpSetForShell.numOfIps = tsDMnodeInfos.nodeNum;
+    for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+      taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSetForShell.fqdn[i], &tsDMnodeIpSetForShell.port[i]);
+      tsDMnodeIpSetForShell.port[i] += TSDB_PORT_DNODESHELL;
     }
   }
 
@@ -265,23 +283,23 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
   return taosCfgDynamicOptions(pCfg->config);
 }
 
-void dnodeUpdateIpSet(SRpcIpSet *pIpSet) {
-  dPrint("mnode IP list is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse);
+void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet) {
+  dPrint("mnode IP list for peer is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse);
   for (int i = 0; i < pIpSet->numOfIps; ++i) {
     dPrint("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i])
   }
 
-  tsMnodeIpSet = *pIpSet;
+  tsDMnodeIpSetForPeer = *pIpSet;
 }
 
-void dnodeGetMnodeDnodeIpSet(void *ipSetRaw) {
+void dnodeGetMnodeIpSetForPeer(void *ipSetRaw) {
   SRpcIpSet *ipSet = ipSetRaw;
-  ipSet->numOfIps = tsMnodeInfos.nodeNum;
-  ipSet->inUse = tsMnodeInfos.inUse;
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; ++i) {
-    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, ipSet->fqdn[i], &ipSet->port[i]);
-    ipSet->port[i] += TSDB_PORT_DNODEDNODE;
-  }
+  *ipSet = tsDMnodeIpSetForPeer;
+}
+
+void dnodeGetMnodeIpSetForShell(void *ipSetRaw) {
+  SRpcIpSet *ipSet = ipSetRaw;
+  *ipSet = tsDMnodeIpSetForShell;
 }
 
 static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
@@ -321,22 +339,29 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
 }
 
 static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
-  bool mnodesChanged = (memcmp(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
-  bool mnodesNotInit = (tsMnodeInfos.nodeNum == 0);
+  bool mnodesChanged = (memcmp(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos)) != 0);
+  bool mnodesNotInit = (tsDMnodeInfos.nodeNum == 0);
   if (!(mnodesChanged || mnodesNotInit)) return;
 
-  memcpy(&tsMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
+  memcpy(&tsDMnodeInfos, pMnodes, sizeof(SDMMnodeInfos));
+  dPrint("mnode infos is changed, nodeNum:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
+  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+    dPrint("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
+  }
 
-  tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
-  tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
-    tsMnodeIpSet.port[i] += TSDB_PORT_DNODEDNODE;
+  tsDMnodeIpSetForPeer.inUse = tsDMnodeInfos.inUse;
+  tsDMnodeIpSetForPeer.numOfIps = tsDMnodeInfos.nodeNum;
+  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+    taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSetForPeer.fqdn[i], &tsDMnodeIpSetForPeer.port[i]);
+    tsDMnodeIpSetForPeer.port[i] += TSDB_PORT_DNODEDNODE;
+    dPrint("mnode index:%d, for peer %s %d", i, tsDMnodeIpSetForPeer.fqdn[i], tsDMnodeIpSetForPeer.port[i]);
   }
 
-  dPrint("mnodes is changed, nodeNum:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
+  tsDMnodeIpSetForShell.inUse = tsDMnodeInfos.inUse;
+  tsDMnodeIpSetForShell.numOfIps = tsDMnodeInfos.nodeNum;
+  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+    taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSetForShell.fqdn[i], &tsDMnodeIpSetForShell.port[i]);
+    dPrint("mnode index:%d, for shell %s %d", i, tsDMnodeIpSetForShell.fqdn[i], tsDMnodeIpSetForShell.port[i]);
   }
 
   dnodeSaveMnodeInfos();
@@ -345,10 +370,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) {
 
 static bool dnodeReadMnodeInfos() {
   char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
+  sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir);
   FILE *fp = fopen(ipFile, "r");
   if (!fp) {
-    dTrace("failed to read mnode mgmtIpList.json, file not exist");
+    dTrace("failed to read mnodeIpList.json, file not exist");
     return false;
   }
 
@@ -359,39 +384,39 @@ static bool dnodeReadMnodeInfos() {
   if (len <= 0) {
     free(content);
     fclose(fp);
-    dError("failed to read mnode mgmtIpList.json, content is null");
+    dError("failed to read mnodeIpList.json, content is null");
     return false;
   }
 
   cJSON* root = cJSON_Parse(content);
   if (root == NULL) {
-    dError("failed to read mnode mgmtIpList.json, invalid json format");
+    dError("failed to read mnodeIpList.json, invalid json format");
     goto PARSE_OVER;
   }
 
   cJSON* inUse = cJSON_GetObjectItem(root, "inUse");
   if (!inUse || inUse->type != cJSON_Number) {
-    dError("failed to read mnode mgmtIpList.json, inUse not found");
+    dError("failed to read mnodeIpList.json, inUse not found");
     goto PARSE_OVER;
   }
-  tsMnodeInfos.inUse = inUse->valueint;
+  tsDMnodeInfos.inUse = inUse->valueint;
 
   cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum");
   if (!nodeNum || nodeNum->type != cJSON_Number) {
-    dError("failed to read mnode mgmtIpList.json, nodeNum not found");
+    dError("failed to read mnodeIpList.json, nodeNum not found");
     goto PARSE_OVER;
   }
-  tsMnodeInfos.nodeNum = nodeNum->valueint;
+  tsDMnodeInfos.nodeNum = nodeNum->valueint;
 
   cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
   if (!nodeInfos || nodeInfos->type != cJSON_Array) {
-    dError("failed to read mnode mgmtIpList.json, nodeInfos not found");
+    dError("failed to read mnodeIpList.json, nodeInfos not found");
     goto PARSE_OVER;
   }
 
   int size = cJSON_GetArraySize(nodeInfos);
-  if (size != tsMnodeInfos.nodeNum) {
-    dError("failed to read mnode mgmtIpList.json, nodeInfos size not matched");
+  if (size != tsDMnodeInfos.nodeNum) {
+    dError("failed to read mnodeIpList.json, nodeInfos size not matched");
     goto PARSE_OVER;
   }
 
@@ -401,24 +426,24 @@ static bool dnodeReadMnodeInfos() {
 
     cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId");
     if (!nodeId || nodeId->type != cJSON_Number) {
-      dError("failed to read mnode mgmtIpList.json, nodeId not found");
+      dError("failed to read mnodeIpList.json, nodeId not found");
       goto PARSE_OVER;
     }
-    tsMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
+    tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint;
 
     cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp");
     if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) {
-      dError("failed to read mnode mgmtIpList.json, nodeName not found");
+      dError("failed to read mnodeIpList.json, nodeName not found");
       goto PARSE_OVER;
     }
-    strncpy(tsMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
+    strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN);
  }
 
   ret = true;
 
-  dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsMnodeInfos.nodeNum, tsMnodeInfos.inUse);
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    dPrint("mnode:%d, %s", tsMnodeInfos.nodeInfos[i].nodeId, tsMnodeInfos.nodeInfos[i].nodeEp);
+  dPrint("read mnode iplist successed, numOfIps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse);
+  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+    dPrint("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp);
   }
 
 PARSE_OVER:
@@ -430,7 +455,7 @@ PARSE_OVER:
 
 static void dnodeSaveMnodeInfos() {
   char ipFile[TSDB_FILENAME_LEN] = {0};
-  sprintf(ipFile, "%s/mgmtIpList.json", tsDnodeDir);
+  sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir);
   FILE *fp = fopen(ipFile, "w");
   if (!fp) return;
 
@@ -439,13 +464,13 @@ static void dnodeSaveMnodeInfos() {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsMnodeInfos.inUse);
-  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsMnodeInfos.nodeNum);
+  len += snprintf(content + len, maxLen - len, "  \"inUse\": %d,\n", tsDMnodeInfos.inUse);
+  len += snprintf(content + len, maxLen - len, "  \"nodeNum\": %d,\n", tsDMnodeInfos.nodeNum);
   len += snprintf(content + len, maxLen - len, "  \"nodeInfos\": [{\n");
-  for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
-    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsMnodeInfos.nodeInfos[i].nodeId);
-    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsMnodeInfos.nodeInfos[i].nodeEp);
-    if (i < tsMnodeInfos.nodeNum -1) {
+  for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) {
+    len += snprintf(content + len, maxLen - len, "    \"nodeId\": %d,\n", tsDMnodeInfos.nodeInfos[i].nodeId);
+    len += snprintf(content + len, maxLen - len, "    \"nodeEp\": \"%s\"\n", tsDMnodeInfos.nodeInfos[i].nodeEp);
+    if (i < tsDMnodeInfos.nodeNum -1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");  
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");  
@@ -461,11 +486,11 @@ static void dnodeSaveMnodeInfos() {
 }
 
 char *dnodeGetMnodeMasterEp() {
-  return tsMnodeInfos.nodeInfos[tsMnodeIpSet.inUse].nodeEp;
+  return tsDMnodeInfos.nodeInfos[tsDMnodeIpSetForPeer.inUse].nodeEp;
 }
 
 void* dnodeGetMnodeInfos() {
-  return &tsMnodeInfos;
+  return &tsDMnodeInfos;
 }
 
 static void dnodeSendStatusMsg(void *handle, void *tmrId) {
@@ -508,7 +533,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
     .msgType = TSDB_MSG_TYPE_DM_STATUS
   };
 
-  dnodeSendMsgToDnode(&tsMnodeIpSet, &rpcMsg);
+  dnodeSendMsgToDnode(&tsDMnodeIpSetForPeer, &rpcMsg);
 }
 
 static bool dnodeReadDnodeCfg() {
@@ -590,3 +615,24 @@ int32_t dnodeGetDnodeId() {
   return tsDnodeCfg.dnodeId;
 }
 
+void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) {
+  SRpcConnInfo connInfo;
+  rpcGetConnInfo(rpcMsg->handle, &connInfo);
+
+  SRpcIpSet ipSet = {0};
+  if (forShell) {
+    dnodeGetMnodeIpSetForShell(&ipSet);
+  } else {
+    dnodeGetMnodeIpSetForPeer(&ipSet);
+  }
+  
+  dTrace("msg:%s will be redirected, dnodeIp:%s user:%s, numOfIps:%d inUse:%d", taosMsg[rpcMsg->msgType],
+         taosIpStr(connInfo.clientIp), connInfo.user, ipSet.numOfIps, ipSet.inUse);
+
+  for (int i = 0; i < ipSet.numOfIps; ++i) {
+    dTrace("mnode index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);
+    ipSet.port[i] = htons(ipSet.port[i]);
+  }
+
+  rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
+}

src/dnode/src/dnodeModule.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taosdef.h"
 #include "tglobal.h"
-#include "trpc.h"
 #include "mnode.h"
 #include "http.h"
 #include "mqtt.h"
@@ -46,12 +45,12 @@ static void dnodeUnSetModuleStatus(int32_t module) {
 }
 
 static void dnodeAllocModules() {
-  tsModule[TSDB_MOD_MGMT].enable       = false;
-  tsModule[TSDB_MOD_MGMT].name         = "mgmt";
-  tsModule[TSDB_MOD_MGMT].initFp       = mgmtInitSystem;
-  tsModule[TSDB_MOD_MGMT].cleanUpFp    = mgmtCleanUpSystem;
-  tsModule[TSDB_MOD_MGMT].startFp      = mgmtStartSystem;
-  tsModule[TSDB_MOD_MGMT].stopFp       = mgmtStopSystem;
+  tsModule[TSDB_MOD_MNODE].enable       = false;
+  tsModule[TSDB_MOD_MNODE].name         = "mnode";
+  tsModule[TSDB_MOD_MNODE].initFp       = mnodeInitSystem;
+  tsModule[TSDB_MOD_MNODE].cleanUpFp    = mnodeCleanupSystem;
+  tsModule[TSDB_MOD_MNODE].startFp      = mnodeStartSystem;
+  tsModule[TSDB_MOD_MNODE].stopFp       = mnodeStopSystem;
 
   tsModule[TSDB_MOD_HTTP].enable       = (tsEnableHttpModule == 1);
   tsModule[TSDB_MOD_HTTP].name         = "http";
@@ -94,8 +93,8 @@ void dnodeCleanUpModules() {
     }
   }
 
-  if (tsModule[TSDB_MOD_MGMT].enable && tsModule[TSDB_MOD_MGMT].cleanUpFp) {
-    (*tsModule[TSDB_MOD_MGMT].cleanUpFp)();
+  if (tsModule[TSDB_MOD_MNODE].enable && tsModule[TSDB_MOD_MNODE].cleanUpFp) {
+    (*tsModule[TSDB_MOD_MNODE].cleanUpFp)();
   }
 }
 
@@ -125,7 +124,7 @@ void dnodeStartModules() {
 }
 
 void dnodeProcessModuleStatus(uint32_t moduleStatus) {
-  for (int32_t module = TSDB_MOD_MGMT; module < TSDB_MOD_HTTP; ++module) {
+  for (int32_t module = TSDB_MOD_MNODE; module < TSDB_MOD_HTTP; ++module) {
     bool enableModule = moduleStatus & (1 << module);
     if (!tsModule[module].enable && enableModule) {
       dPrint("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);

src/dnode/src/dnodePeer.c

@@ -22,14 +22,14 @@
 #include "os.h"
 #include "taosmsg.h"
 #include "tglobal.h"
-#include "trpc.h"
+#include "mnode.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
 #include "dnodeVWrite.h"
-#include "mnode.h"
+#include "dnodeMPeer.h"
 
-extern void dnodeUpdateIpSet(SRpcIpSet *pIpSet);
+extern void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet);
 static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *);
 static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
@@ -48,11 +48,11 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToDnodeMgmt;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToDnodeMgmt;
 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = mgmtProcessReqMsgFromDnode;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = mgmtProcessReqMsgFromDnode;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = mgmtProcessReqMsgFromDnode;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = mgmtProcessReqMsgFromDnode;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = mgmtProcessReqMsgFromDnode;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMnodePeerQueue;
   
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
@@ -101,16 +101,14 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
     rpcSendResponse(&rspMsg);
     return;
   }
- 
+
   if (dnodeProcessReqMsgFp[pMsg->msgType]) {
     (*dnodeProcessReqMsgFp[pMsg->msgType])(pMsg);
   } else {
+    dTrace("RPC %p, message:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]);
     rspMsg.code = TSDB_CODE_MSG_NOT_PROCESSED;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);
-    dTrace("RPC %p, message:%s not processed", pMsg->handle, taosMsg[pMsg->msgType]); 
-    return;
-
   }
 }
 
@@ -146,12 +144,14 @@ void dnodeCleanupClient() {
 }
 
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
+  if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) {
+    dnodeUpdateMnodeIpSetForPeer(pIpSet);
+  }
 
-  if (dnodeProcessRspMsgFp[pMsg->msgType]) {
-    if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) dnodeUpdateIpSet(pIpSet);
+  if (dnodeProcessRspMsgFp[pMsg->msgType]) {    
     (*dnodeProcessRspMsgFp[pMsg->msgType])(pMsg);
   } else {
-    dError("RPC %p, msg:%s is not processed", pMsg->handle, taosMsg[pMsg->msgType]);
+    mnodeProcessPeerRsp(pMsg);
   }
 
   rpcFreeCont(pMsg->pCont);
@@ -167,6 +167,6 @@ void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) {
 
 void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcIpSet ipSet = {0};
-  dnodeGetMnodeDnodeIpSet(&ipSet);
+  dnodeGetMnodeIpSetForPeer(&ipSet);
   rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp);
 }

src/dnode/src/dnodeShell.c

@@ -18,7 +18,6 @@
 #include "taoserror.h"
 #include "taosdef.h"
 #include "taosmsg.h"
-#include "trpc.h"
 #include "tglobal.h"
 #include "http.h"
 #include "mnode.h"
@@ -26,6 +25,8 @@
 #include "dnodeInt.h"
 #include "dnodeVRead.h"
 #include "dnodeVWrite.h"
+#include "dnodeMRead.h"
+#include "dnodeMWrite.h"
 #include "dnodeShell.h"
 
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
@@ -35,43 +36,41 @@ static void  * tsDnodeShellRpc = NULL;
 static int32_t tsDnodeQueryReqNum  = 0;
 static int32_t tsDnodeSubmitReqNum = 0;
 
-void mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
-
 int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVnodeWriteQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY]  = dnodeDispatchToVnodeReadQueue;
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH]  = dnodeDispatchToVnodeReadQueue;
 
   // the following message shall be treated as mnode write
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = mgmtProcessMsgFromShell; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = mgmtProcessMsgFromShell; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = mgmtProcessMsgFromShell;
-
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = dnodeDispatchToMnodeWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
+  
   // the following message shall be treated as mnode query 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = mgmtProcessMsgFromShell; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= mgmtProcessMsgFromShell;   
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = mgmtProcessMsgFromShell;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= mgmtProcessMsgFromShell;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = dnodeDispatchToMnodeReadQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;   
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMnodeReadQueue;
 
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
   numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@@ -139,7 +138,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) {
 }
 
 static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  int code = mgmtRetriveAuth(user, spi, encrypt, secret, ckey);
+  int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey);
   if (code != TSDB_CODE_NOT_READY) return code;
 
   SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg));

src/dnode/src/dnodeVRead.c

@@ -19,7 +19,6 @@
 #include "taosmsg.h"
 #include "tutil.h"
 #include "tqueue.h"
-#include "trpc.h"
 #include "twal.h"
 #include "tglobal.h"
 #include "dnodeInt.h"
@@ -53,7 +52,7 @@ static void  dnodeHandleIdleReadWorker(SReadWorker *);
 static SReadWorkerPool readPool;
 static taos_qset       readQset;
 
-int32_t dnodeInitRead() {
+int32_t dnodeInitVnodeRead() {
   readQset = taosOpenQset();
 
   readPool.min = 2;
@@ -71,7 +70,7 @@ int32_t dnodeInitRead() {
   return 0;
 }
 
-void dnodeCleanupRead() {
+void dnodeCleanupVnodeRead() {
   for (int i=0; i < readPool.max; ++i) {
     SReadWorker *pWorker = readPool.readWorker + i;
     if (pWorker->thread) {
@@ -142,7 +141,7 @@ void dnodeDispatchToVnodeReadQueue(SRpcMsg *pMsg) {
   }
 }
 
-void *dnodeAllocateRqueue(void *pVnode) {
+void *dnodeAllocateVnodeRqueue(void *pVnode) {
   taos_queue queue = taosOpenQueue();
   if (queue == NULL) return NULL;
 
@@ -172,7 +171,7 @@ void *dnodeAllocateRqueue(void *pVnode) {
   return queue;
 }
 
-void dnodeFreeRqueue(void *rqueue) {
+void dnodeFreeVnodeRqueue(void *rqueue) {
   taosCloseQueue(rqueue);
 
   // dynamically adjust the number of threads

src/dnode/src/dnodeVWrite.c

@@ -22,9 +22,9 @@
 #include "trpc.h"
 #include "tsdb.h"
 #include "twal.h"
+#include "tdataformat.h"
 #include "tglobal.h"
 #include "vnode.h"
-#include "tdataformat.h"
 #include "dnodeInt.h"
 #include "dnodeVWrite.h"
 #include "dnodeMgmt.h"
@@ -54,7 +54,7 @@ static void  dnodeHandleIdleWorker(SWriteWorker *pWorker);
 
 SWriteWorkerPool wWorkerPool;
 
-int32_t dnodeInitWrite() {
+int32_t dnodeInitVnodeWrite() {
   wWorkerPool.max = tsNumOfCores;
   wWorkerPool.writeWorker = (SWriteWorker *)calloc(sizeof(SWriteWorker), wWorkerPool.max);
   if (wWorkerPool.writeWorker == NULL) return -1;
@@ -67,7 +67,7 @@ int32_t dnodeInitWrite() {
   return 0;
 }
 
-void dnodeCleanupWrite() {
+void dnodeCleanupVnodeWrite() {
   for (int32_t i = 0; i < wWorkerPool.max; ++i) {
     SWriteWorker *pWorker =  wWorkerPool.writeWorker + i;
     if (pWorker->thread) {
@@ -122,7 +122,7 @@ void dnodeDispatchToVnodeWriteQueue(SRpcMsg *pMsg) {
   }
 }
 
-void *dnodeAllocateWqueue(void *pVnode) {
+void *dnodeAllocateVnodeWqueue(void *pVnode) {
   SWriteWorker *pWorker = wWorkerPool.writeWorker + wWorkerPool.nextId;
   void *queue = taosOpenQueue();
   if (queue == NULL) return NULL;
@@ -157,13 +157,13 @@ void *dnodeAllocateWqueue(void *pVnode) {
   return queue;
 }
 
-void dnodeFreeWqueue(void *wqueue) {
+void dnodeFreeVnodeWqueue(void *wqueue) {
   taosCloseQueue(wqueue);
 
   // dynamically adjust the number of threads
 }
 
-void dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code) {
+void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) {
   SWriteMsg *pWrite = (SWriteMsg *)param;
 
   if (code > 0) return;
@@ -223,7 +223,7 @@ static void *dnodeProcessWriteQueue(void *param) {
       taosGetQitem(pWorker->qall, &type, &item);
       if (type == TAOS_QTYPE_RPC) {
         pWrite = (SWriteMsg *)item;
-        dnodeSendRpcWriteRsp(pVnode, item, pWrite->rpcMsg.code); 
+        dnodeSendRpcVnodeWriteRsp(pVnode, item, pWrite->rpcMsg.code); 
       } else {
         taosFreeQitem(item);
         vnodeRelease(pVnode);

src/inc/dnode.h

@@ -37,22 +37,32 @@ typedef enum {
 SDnodeRunStatus dnodeGetRunStatus();
 SDnodeStatisInfo dnodeGetStatisInfo();
 
-void *dnodeAllocateWqueue(void *pVnode);
-void  dnodeFreeWqueue(void *queue);
-void *dnodeAllocateRqueue(void *pVnode);
-void  dnodeFreeRqueue(void *rqueue);
-void  dnodeSendRpcWriteRsp(void *pVnode, void *param, int32_t code);
-
-bool     dnodeIsFirstDeploy();
-char    *dnodeGetMnodeMasterEp();
-void     dnodeGetMnodeDnodeIpSet(void *ipSet);
-void *   dnodeGetMnodeInfos();
-int32_t  dnodeGetDnodeId();
-
-void     dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
-void     dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
-void     dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg);
-void     dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
+bool    dnodeIsFirstDeploy();
+char *  dnodeGetMnodeMasterEp();
+void    dnodeGetMnodeIpSetForPeer(void *ipSet);
+void    dnodeGetMnodeIpSetForShell(void *ipSet);
+void *  dnodeGetMnodeInfos();
+int32_t dnodeGetDnodeId();
+
+void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg);
+void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp);
+
+void *dnodeAllocateVnodeWqueue(void *pVnode);
+void  dnodeFreeVnodeWqueue(void *queue);
+void *dnodeAllocateVnodeRqueue(void *pVnode);
+void  dnodeFreeVnodeRqueue(void *rqueue);
+void  dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code);
+
+int32_t dnodeAllocateMnodePqueue();
+void    dnodeFreeMnodePqueue();
+int32_t dnodeAllocateMnodeRqueue();
+void    dnodeFreeMnodeRqueue();
+int32_t dnodeAllocateMnodeWqueue();
+void    dnodeFreeMnodeWqueue();
+void    dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
+void    dnodeReprocessMnodeWriteMsg(void *pMsg);
+void    dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
 
 #ifdef __cplusplus
 }

src/inc/mnode.h

@@ -13,23 +13,58 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_H
-#define TDENGINE_MGMT_H
+#ifndef TDENGINE_MNODE_H
+#define TDENGINE_MNODE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-int32_t mgmtInitSystem();
-int32_t mgmtStartSystem();
-void    mgmtCleanUpSystem();
-void    mgmtStopSystem();
-void    sdbUpdateSync();
+#include "trpc.h"
+
+struct SAcctObj;
+struct SDnodeObj;
+struct SUserObj;
+struct SDbObj;
+struct SVgObj;
+struct STableObj;
+
+typedef struct {
+  int32_t len;
+  void *  rsp;
+} SMnodeRsp;
 
-int32_t mgmtRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-void    mgmtProcessMsgFromShell(SRpcMsg *rpcMsg);
-void    mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg);
+typedef struct SMnodeMsg {
+  SRpcMsg   rpcMsg;
+  SMnodeRsp rpcRsp;
+  int8_t    received;
+  int8_t    successed;
+  int8_t    expected;
+  int8_t    retry;
+  int32_t   code;
+  struct SAcctObj * pAcct;
+  struct SDnodeObj *pDnode;
+  struct SUserObj * pUser;
+  struct SDbObj *   pDb;
+  struct SVgObj *   pVgroup;
+  struct STableObj *pTable;
+} SMnodeMsg;
 
+void    mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
+int32_t mnodeInitMsg(SMnodeMsg *pMsg);
+void    mnodeCleanupMsg(SMnodeMsg *pMsg);
+
+int32_t mnodeInitSystem();
+int32_t mnodeStartSystem();
+void    mnodeCleanupSystem();
+void    mnodeStopSystem();
+void    sdbUpdateSync();
+bool    mnodeIsRunning();
+int32_t mnodeProcessRead(SMnodeMsg *pMsg);
+int32_t mnodeProcessWrite(SMnodeMsg *pMsg);
+int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg);
+void    mnodeProcessPeerRsp(SRpcMsg *pMsg);
+int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey);
 
 #ifdef __cplusplus
 }

src/inc/taosdef.h

@@ -203,6 +203,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_COL_NAME_LEN         64
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
+#define TSDB_MAX_SQL_SHOW_LEN     256
 #define TSDB_MAX_ALLOWED_SQL_LEN  (8*1024*1024U)          // sql length should be less than 6mb
 
 #define TSDB_MAX_BYTES_PER_ROW    TSDB_MAX_COLUMNS * 64
@@ -373,7 +374,7 @@ typedef enum {
 } ETableType;
 
 typedef enum {
-  TSDB_MOD_MGMT,
+  TSDB_MOD_MNODE,
   TSDB_MOD_HTTP,
   TSDB_MOD_MONITOR,
   TSDB_MOD_MQTT,

src/inc/tbalance.h

@@ -26,7 +26,7 @@ struct SDnodeObj;
 int32_t balanceInit();
 void    balanceCleanUp();
 void    balanceNotify();
-void    balanceUpdateMgmt();
+void    balanceUpdateMnode();
 void    balanceReset();
 int32_t balanceAllocVnodes(struct SVgObj *pVgroup);
 int32_t balanceDropDnode(struct SDnodeObj *pDnode);

src/inc/trpc.h

@@ -28,21 +28,21 @@ extern "C" {
 
 extern int tsRpcHeadSize;
 
-typedef struct {
+typedef struct SRpcIpSet {
   int8_t    inUse; 
   int8_t    numOfIps;
   uint16_t  port[TSDB_MAX_REPLICA];
   char      fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN];
 } SRpcIpSet;
 
-typedef struct {
+typedef struct SRpcConnInfo {
   uint32_t  clientIp;
   uint16_t  clientPort;
   uint32_t  serverIp;
   char      user[TSDB_USER_LEN];
 } SRpcConnInfo;
 
-typedef struct {
+typedef struct SRpcMsg {
   uint8_t msgType;
   void   *pCont;
   int     contLen;
@@ -51,7 +51,7 @@ typedef struct {
   void   *ahandle;  //app handle set by client, for debug purpose
 } SRpcMsg;
 
-typedef struct {
+typedef struct SRpcInit {
   uint16_t localPort; // local port
   char  *label;        // for debug purpose
   int    numOfThreads; // number of threads to handle connections

src/mnode/inc/mgmtShell.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_MGMT_SHELL_H
-#define TDENGINE_MGMT_SHELL_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "mgmtDef.h"
-
-int32_t mgmtInitShell();
-void    mgmtCleanUpShell();
-void    mgmtAddShellMsgHandle(uint8_t msgType, void (*fp)(SQueuedMsg *queuedMsg));
-
-typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-void mgmtAddShellShowMetaHandle(uint8_t showType, SShowMetaFp fp);
-void mgmtAddShellShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
-
-void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
-void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg);
-void mgmtSendSimpleResp(void *thandle, int32_t code);
-
-bool  mgmtCheckQhandle(uint64_t qhandle);
-void *mgmtSaveQhandle(void *qhandle, int32_t size);
-void  mgmtFreeQhandle(void *qhandle, bool forceRemove);
-
-void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
-void *mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
-void  mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/mnode/inc/mgmtAcct.h → src/mnode/inc/mnodeAcct.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_ACCT_H
-#define TDENGINE_MGMT_ACCT_H
+#ifndef TDENGINE_MNODE_ACCT_H
+#define TDENGINE_MNODE_ACCT_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -22,16 +22,16 @@ extern "C" {
 
 #include "tacct.h"
 
-int32_t mgmtInitAccts();
-void    mgmtCleanUpAccts();
-void *  mgmtGetAcct(char *acctName);
-void *  mgmtGetNextAcct(void *pIter, SAcctObj **pAcct);
-void    mgmtIncAcctRef(SAcctObj *pAcct);
-void    mgmtDecAcctRef(SAcctObj *pAcct);
-void    mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);
-void    mgmtDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb);
-void    mgmtAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser);
-void    mgmtDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser);
+int32_t mnodeInitAccts();
+void    mnodeCleanupAccts();
+void *  mnodeGetAcct(char *acctName);
+void *  mnodeGetNextAcct(void *pIter, SAcctObj **pAcct);
+void    mnodeIncAcctRef(SAcctObj *pAcct);
+void    mnodeDecAcctRef(SAcctObj *pAcct);
+void    mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);
+void    mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb);
+void    mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser);
+void    mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtDb.h → src/mnode/inc/mnodeDb.h

@@ -13,14 +13,14 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DB_H
-#define TDENGINE_MGMT_DB_H
+#ifndef TDENGINE_MNODE_DB_H
+#define TDENGINE_MNODE_DB_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "mgmtDef.h"
+#include "mnodeDef.h"
 
 enum _TSDB_DB_STATUS {
   TSDB_DB_STATUS_READY,
@@ -28,26 +28,26 @@ enum _TSDB_DB_STATUS {
 };
 
 // api
-int32_t mgmtInitDbs();
-void    mgmtCleanUpDbs();
-SDbObj *mgmtGetDb(char *db);
-SDbObj *mgmtGetDbByTableId(char *db);
-void *  mgmtGetNextDb(void *pIter, SDbObj **pDb);
-void    mgmtIncDbRef(SDbObj *pDb);
-void    mgmtDecDbRef(SDbObj *pDb);
-bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);
-void    mgmtDropAllDbs(SAcctObj *pAcct);
+int32_t mnodeInitDbs();
+void    mnodeCleanupDbs();
+SDbObj *mnodeGetDb(char *db);
+SDbObj *mnodeGetDbByTableId(char *db);
+void *  mnodeGetNextDb(void *pIter, SDbObj **pDb);
+void    mnodeIncDbRef(SDbObj *pDb);
+void    mnodeDecDbRef(SDbObj *pDb);
+bool    mnodeCheckIsMonitorDB(char *db, char *monitordb);
+void    mnodeDropAllDbs(SAcctObj *pAcct);
 
 // util func
-void mgmtAddSuperTableIntoDb(SDbObj *pDb);
-void mgmtRemoveSuperTableFromDb(SDbObj *pDb);
-void mgmtAddTableIntoDb(SDbObj *pDb);
-void mgmtRemoveTableFromDb(SDbObj *pDb);
-void mgmtAddVgroupIntoDb(SVgObj *pVgroup);
-void mgmtAddVgroupIntoDbTail(SVgObj *pVgroup);
-void mgmtRemoveVgroupFromDb(SVgObj *pVgroup);
-void mgmtMoveVgroupToTail(SVgObj *pVgroup);
-void mgmtMoveVgroupToHead(SVgObj *pVgroup);
+void mnodeAddSuperTableIntoDb(SDbObj *pDb);
+void mnodeRemoveSuperTableFromDb(SDbObj *pDb);
+void mnodeAddTableIntoDb(SDbObj *pDb);
+void mnodeRemoveTableFromDb(SDbObj *pDb);
+void mnodeAddVgroupIntoDb(SVgObj *pVgroup);
+void mnodeAddVgroupIntoDbTail(SVgObj *pVgroup);
+void mnodeRemoveVgroupFromDb(SVgObj *pVgroup);
+void mnodeMoveVgroupToTail(SVgObj *pVgroup);
+void mnodeMoveVgroupToHead(SVgObj *pVgroup);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtDef.h → src/mnode/inc/mnodeDef.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DEF_H
-#define TDENGINE_MGMT_DEF_H
+#ifndef TDENGINE_MNODE_DEF_H
+#define TDENGINE_MNODE_DEF_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,7 +65,7 @@ typedef struct SMnodeObj {
   int8_t     role;
 } SMnodeObj;
 
-typedef struct {
+typedef struct STableObj {
   char  *tableId;
   int8_t type;
 } STableObj;
@@ -233,26 +233,6 @@ typedef struct {
   char     payload[];
 } SShowObj;
 
-typedef struct {
-  uint8_t  msgType;
-  int8_t   received;
-  int8_t   successed;
-  int8_t   expected;
-  int8_t   retry;
-  int8_t   maxRetry;
-  int32_t  contLen;
-  int32_t  code;
-  void     *ahandle;
-  void     *thandle;
-  void     *pCont;
-  SAcctObj *pAcct;
-  SDnodeObj*pDnode;
-  SUserObj *pUser;
-  SDbObj   *pDb;
-  SVgObj   *pVgroup;
-  STableObj *pTable;
-} SQueuedMsg;
-
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtDnode.h → src/mnode/inc/mnodeDnode.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_CLUSTER_H
-#define TDENGINE_CLUSTER_H
+#ifndef TDENGINE_MNODE_DNODE_H
+#define TDENGINE_MNODE_DNODE_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -27,21 +27,21 @@ typedef enum {
   TAOS_DN_STATUS_READY
 } EDnodeStatus;
 
-int32_t mgmtInitDnodes();
-void    mgmtCleanupDnodes();
+int32_t mnodeInitDnodes();
+void    mnodeCleanupDnodes();
 
-char*   mgmtGetDnodeStatusStr(int32_t dnodeStatus);
+char*   mnodeGetDnodeStatusStr(int32_t dnodeStatus);
 void    mgmtMonitorDnodeModule();
 
-int32_t mgmtGetDnodesNum();
-int32_t mgmtGetOnlinDnodesNum();
-void *  mgmtGetNextDnode(void *pIter, SDnodeObj **pDnode);
-void    mgmtIncDnodeRef(SDnodeObj *pDnode);
-void    mgmtDecDnodeRef(SDnodeObj *pDnode);
-void *  mgmtGetDnode(int32_t dnodeId);
-void *  mgmtGetDnodeByEp(char *ep);
-void    mgmtUpdateDnode(SDnodeObj *pDnode);
-int32_t mgmtDropDnode(SDnodeObj *pDnode);
+int32_t mnodeGetDnodesNum();
+int32_t mnodeGetOnlinDnodesNum();
+void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
+void    mnodeIncDnodeRef(SDnodeObj *pDnode);
+void    mnodeDecDnodeRef(SDnodeObj *pDnode);
+void *  mnodeGetDnode(int32_t dnodeId);
+void *  mnodeGetDnodeByEp(char *ep);
+void    mnodeUpdateDnode(SDnodeObj *pDnode);
+int32_t mnodeDropDnode(SDnodeObj *pDnode);
 
 extern int32_t tsAccessSquence;
 

src/mnode/inc/mgmtInt.h → src/mnode/inc/mnodeInt.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_LOG_H
-#define TDENGINE_MGMT_LOG_H
+#ifndef TDENGINE_MNODE_LOG_H
+#define TDENGINE_MNODE_LOG_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -28,7 +28,7 @@ extern int32_t sdbDebugFlag;
 
 // mnode log function
 #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND ", 255, __VA_ARGS__); }}
-#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND ", mDebugFlag, __VA_ARGS__); }}
+#define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN MND ", mDebugFlag, __VA_ARGS__); }}
 #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
 #define mPrint(...) { taosPrintLog("MND ", 255, __VA_ARGS__); }
 
@@ -37,7 +37,7 @@ extern int32_t sdbDebugFlag;
 #define mLPrint(...) { monitorSaveLog(0, __VA_ARGS__); mPrint(__VA_ARGS__) }
 
 #define sdbError(...) { if (sdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MND-SDB ", 255, __VA_ARGS__); }}
-#define sdbWarn(...)  { if (sdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MND-SDB ", sdbDebugFlag, __VA_ARGS__); }}
+#define sdbWarn(...)  { if (sdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN MND-SDB ", sdbDebugFlag, __VA_ARGS__); }}
 #define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("MND-SDB ", sdbDebugFlag, __VA_ARGS__);}}
 #define sdbPrint(...) { taosPrintLog("MND-SDB ", 255, __VA_ARGS__); }
 

src/mnode/inc/mgmtMnode.h → src/mnode/inc/mnodeMnode.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_MNODE_H
-#define TDENGINE_MGMT_MNODE_H
+#ifndef TDENGINE_MNODE_MNODE_H
+#define TDENGINE_MNODE_MNODE_H
 
 #ifdef __cplusplus
 extern "C" {
@@ -28,23 +28,25 @@ typedef enum {
   TAOS_MN_STATUS_READY
 } EMnodeStatus;
 
-int32_t mgmtInitMnodes();
-void    mgmtCleanupMnodes();
+int32_t mnodeInitMnodes();
+void    mnodeCleanupMnodes();
 
-int32_t mgmtAddMnode(int32_t dnodeId);
-int32_t mgmtDropMnode(int32_t dnodeId);
-void    mgmtDropMnodeLocal(int32_t dnodeId);
+int32_t mnodeAddMnode(int32_t dnodeId);
+int32_t mnodeDropMnode(int32_t dnodeId);
+void    mnodeDropMnodeLocal(int32_t dnodeId);
 
-void *  mgmtGetMnode(int32_t mnodeId);
-int32_t mgmtGetMnodesNum();
-void *  mgmtGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
-void    mgmtIncMnodeRef(struct SMnodeObj *pMnode);
-void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
+void *  mnodeGetMnode(int32_t mnodeId);
+int32_t mnodeGetMnodesNum();
+void *  mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
+void    mnodeIncMnodeRef(struct SMnodeObj *pMnode);
+void    mnodeDecMnodeRef(struct SMnodeObj *pMnode);
 
-char *  mgmtGetMnodeRoleStr();
-void    mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
-void    mgmtGetMnodeInfos(void *mnodes);
-void    mgmtUpdateMnodeIpSet();
+char *  mnodeGetMnodeRoleStr();
+void    mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet);
+void    mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet);
+
+void    mnodeGetMnodeInfos(void *mnodes);
+void    mnodeUpdateMnodeIpSet();
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodePeer.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_PEER_H
+#define TDENGINE_MNODE_PEER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeDef.h"
+
+void    mnodeAddPeerRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
+void    mnodeAddPeerMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg));
+int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg);
+void    mnodeProcessPeerRsp(SRpcMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mgmtProfile.h → src/mnode/inc/mnodeProfile.h

@@ -13,16 +13,16 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_PROFILE_H
-#define TDENGINE_MGMT_PROFILE_H
+#ifndef TDENGINE_MNODE_PROFILE_H
+#define TDENGINE_MNODE_PROFILE_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "mgmtDef.h"
+#include "mnodeDef.h"
 
-int32_t mgmtInitProfile();
-void    mgmtCleanUpProfile();
+int32_t mnodeInitProfile();
+void    mnodeCleanupProfile();
 
 #ifdef __cplusplus
 }

src/mnode/inc/mgmtServer.h → src/mnode/inc/mnodeRead.h

@@ -13,15 +13,16 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_DSERVER_H
-#define TDENGINE_MGMT_DSERVER_H
+#ifndef TDENGINE_MNODE_READ_H
+#define TDENGINE_MNODE_READ_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
+#include "mnodeDef.h"
 
-int32_t mgmtInitServer();
-void    mgmtCleanupServer();
+void    mnodeAddReadMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg));
+int32_t mnodeProcessRead(SMnodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeShow.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_SHELL_H
+#define TDENGINE_MNODE_SHELL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeDef.h"
+
+int32_t mnodeInitShow();
+void    mnodeCleanUpShow();
+
+typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp);
+void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mgmtTable.h → src/mnode/inc/mnodeTable.h

@@ -20,18 +20,18 @@
 extern "C" {
 #endif
 
-#include "mgmtDef.h"
+#include "mnodeDef.h"
 
-int32_t mgmtInitTables();
-void    mgmtCleanUpTables();
-void *  mgmtGetTable(char *tableId);
-void    mgmtIncTableRef(void *pTable);
-void    mgmtDecTableRef(void *pTable);
-void *  mgmtGetNextChildTable(void *pIter, SChildTableObj **pTable);
-void *  mgmtGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
-void    mgmtDropAllChildTables(SDbObj *pDropDb);
-void    mgmtDropAllSuperTables(SDbObj *pDropDb);
-void    mgmtDropAllChildTablesInVgroups(SVgObj *pVgroup);
+int32_t mnodeInitTables();
+void    mnodeCleanupTables();
+void *  mnodeGetTable(char *tableId);
+void    mnodeIncTableRef(void *pTable);
+void    mnodeDecTableRef(void *pTable);
+void *  mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable);
+void *  mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
+void    mnodeDropAllChildTables(SDbObj *pDropDb);
+void    mnodeDropAllSuperTables(SDbObj *pDropDb);
+void    mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeUser.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_USER_H
+#define TDENGINE_MNODE_USER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeDef.h"
+
+int32_t   mnodeInitUsers();
+void      mnodeCleanupUsers();
+SUserObj *mnodeGetUser(char *name);
+void *    mnodeGetNextUser(void *pIter, SUserObj **pUser);
+void      mnodeIncUserRef(SUserObj *pUser);
+void      mnodeDecUserRef(SUserObj *pUser);
+SUserObj *mnodeGetUserFromConn(void *pConn);
+int32_t   mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass);
+void      mnodeDropAllUsers(SAcctObj *pAcct);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/inc/mgmtVgroup.h → src/mnode/inc/mnodeVgroup.h

@@ -13,41 +13,41 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_MGMT_VGROUP_H
-#define TDENGINE_MGMT_VGROUP_H
+#ifndef TDENGINE_MNODE_VGROUP_H
+#define TDENGINE_MNODE_VGROUP_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include "mgmtDef.h"
-
-int32_t mgmtInitVgroups();
-void    mgmtCleanUpVgroups();
-SVgObj *mgmtGetVgroup(int32_t vgId);
-void    mgmtIncVgroupRef(SVgObj *pVgroup);
-void    mgmtDecVgroupRef(SVgObj *pVgroup);
-void    mgmtDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
-void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
-void    mgmtUpdateAllDbVgroups(SDbObj *pAlterDb);
-
-void *  mgmtGetNextVgroup(void *pIter, SVgObj **pVgroup);
-void    mgmtUpdateVgroup(SVgObj *pVgroup);
-void    mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
-
-void    mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb);
-void    mgmtDropVgroup(SVgObj *pVgroup, void *ahandle);
-void    mgmtAlterVgroup(SVgObj *pVgroup, void *ahandle);
-SVgObj *mgmtGetAvailableVgroup(SDbObj *pDb);
-
-void    mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
-void    mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
-void    mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle);
-void    mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);
-void    mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
-
-SRpcIpSet mgmtGetIpSetFromVgroup(SVgObj *pVgroup);
-SRpcIpSet mgmtGetIpSetFromIp(char *ep);
+struct SMnodeMsg;
+
+int32_t mnodeInitVgroups();
+void    mnodeCleanupVgroups();
+SVgObj *mnodeGetVgroup(int32_t vgId);
+void    mnodeIncVgroupRef(SVgObj *pVgroup);
+void    mnodeDecVgroupRef(SVgObj *pVgroup);
+void    mnodeDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
+void    mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
+void    mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
+
+void *  mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
+void    mnodeUpdateVgroup(SVgObj *pVgroup);
+void    mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *dnodeId, SVnodeLoad *pVload);
+
+int32_t mnodeCreateVgroup(struct SMnodeMsg *pMsg, SDbObj *pDb);
+void    mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
+void    mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
+SVgObj *mnodeGetAvailableVgroup(SDbObj *pDb);
+
+void    mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
+void    mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
+void    mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle);
+void    mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle);
+void    mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
+
+SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup);
+SRpcIpSet mnodeGetIpSetFromIp(char *ep);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeWrite.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_MNODE_WRITE_H
+#define TDENGINE_MNODE_WRITE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "mnodeDef.h"
+
+void    mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg));
+int32_t mnodeProcessRead(SMnodeMsg *pMsg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/mnode/src/mgmtServer.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "taoserror.h"
-#include "trpc.h"
-#include "tsched.h"
-#include "tsystem.h"
-#include "tutil.h"
-#include "tgrant.h"
-#include "tbalance.h"
-#include "tglobal.h"
-#include "dnode.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtDb.h"
-#include "mgmtMnode.h"
-#include "mgmtProfile.h"
-#include "mgmtShell.h"
-#include "mgmtSdb.h"
-#include "mgmtTable.h"
-#include "mgmtVgroup.h"
-
-static void (*mgmtProcessDnodeMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
-static void  *tsMgmtServerQhandle = NULL;
-
-int32_t mgmtInitServer() {
-
-  tsMgmtServerQhandle = taosInitScheduler(tsMaxShellConns, 1, "MS");
-
-  mPrint("server connection to dnode is opened");
-  return 0;
-}
-
-void mgmtCleanupServer() {
-  if (tsMgmtServerQhandle) {
-    taosCleanUpScheduler(tsMgmtServerQhandle);
-    tsMgmtServerQhandle = NULL;
-  }
-}
-
-void dnodeAddServerMsgHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
-  mgmtProcessDnodeMsgFp[msgType] = fp;
-}
-
-static void mgmtProcessRequestFromDnode(SSchedMsg *sched) {
-  SRpcMsg *pMsg = sched->msg;
-  (*mgmtProcessDnodeMsgFp[pMsg->msgType])(pMsg);
-  rpcFreeCont(pMsg->pCont);
-  free(pMsg);
-}
-
-static void mgmtAddToServerQueue(SRpcMsg *pMsg) {
-  SSchedMsg schedMsg;
-  schedMsg.msg = pMsg;
-  schedMsg.fp  = mgmtProcessRequestFromDnode;
-  taosScheduleTask(tsMgmtServerQhandle, &schedMsg);
-}
-
-void mgmtProcessReqMsgFromDnode(SRpcMsg *rpcMsg) {
-  if (mgmtProcessDnodeMsgFp[rpcMsg->msgType] == NULL) {
-    mError("%s is not processed in mnode", taosMsg[rpcMsg->msgType]);
-    mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_MSG_NOT_PROCESSED);
-    rpcFreeCont(rpcMsg->pCont);
-  }
-
-  if (rpcMsg->pCont == NULL) {
-    mgmtSendSimpleResp(rpcMsg->handle, TSDB_CODE_INVALID_MSG_LEN);
-    return;
-  }
-
-  if (!sdbIsMaster()) {
-    SRpcConnInfo connInfo;
-    rpcGetConnInfo(rpcMsg->handle, &connInfo);
-    
-    SRpcIpSet ipSet = {0};
-    dnodeGetMnodeDnodeIpSet(&ipSet);
-    for (int i = 0; i < ipSet.numOfIps; ++i) 
-      ipSet.port[i] = htons(ipSet.port[i]);
-    
-    mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
-    for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
-      mTrace("mnode index:%d %s:%d", i, ipSet.fqdn[i], htons(ipSet.port[i]));
-    }
-    rpcSendRedirectRsp(rpcMsg->handle, &ipSet);
-    return;
-  }
-  
-  SRpcMsg *pMsg = malloc(sizeof(SRpcMsg));
-  memcpy(pMsg, rpcMsg, sizeof(SRpcMsg));
-  mgmtAddToServerQueue(pMsg);
-}
-

src/mnode/src/mgmtAcct.c → src/mnode/src/mnodeAcct.c

@@ -17,59 +17,58 @@
 #include "os.h"
 #include "taoserror.h"
 #include "ttime.h"
-#include "tutil.h"
 #include "dnode.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtAcct.h"
-#include "mgmtDb.h"
-#include "mgmtSdb.h"
-#include "mgmtUser.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeAcct.h"
+#include "mnodeDb.h"
+#include "mnodeSdb.h"
+#include "mnodeUser.h"
 
 void *  tsAcctSdb = NULL;
 static int32_t tsAcctUpdateSize;
-static void    mgmtCreateRootAcct();
+static void    mnodeCreateRootAcct();
 
-static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {
+static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
   pthread_mutex_destroy(&pAcct->mutex);
   tfree(pOper->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtAcctActionInsert(SSdbOper *pOper) {
+static int32_t mnodeAcctActionInsert(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
   memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
   pthread_mutex_init(&pAcct->mutex, NULL);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtActionAcctDelete(SSdbOper *pOper) {
+static int32_t mnodeAcctActionDelete(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
-  mgmtDropAllUsers(pAcct);
-  mgmtDropAllDbs(pAcct);
+  mnodeDropAllUsers(pAcct);
+  mnodeDropAllDbs(pAcct);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtActionAcctUpdate(SSdbOper *pOper) {
+static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
-  SAcctObj *pSaved = mgmtGetAcct(pAcct->user);
+  SAcctObj *pSaved = mnodeGetAcct(pAcct->user);
   if (pAcct != pSaved) {
     memcpy(pSaved, pAcct, tsAcctUpdateSize);
     free(pAcct);
   }
-  mgmtDecAcctRef(pSaved);
+  mnodeDecAcctRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtActionActionEncode(SSdbOper *pOper) {
+static int32_t mnodeActionActionEncode(SSdbOper *pOper) {
   SAcctObj *pAcct = pOper->pObj;
   memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
   pOper->rowSize = tsAcctUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtActionAcctDecode(SSdbOper *pOper) {
+static int32_t mnodeAcctActionDecode(SSdbOper *pOper) {
   SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj));
   if (pAcct == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
 
@@ -78,16 +77,16 @@ static int32_t mgmtActionAcctDecode(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtActionAcctRestored() {
+static int32_t mnodeAcctActionRestored() {
   if (dnodeIsFirstDeploy()) {
-    mgmtCreateRootAcct();
+    mnodeCreateRootAcct();
   }
 
   acctInit();
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t mgmtInitAccts() {
+int32_t mnodeInitAccts() {
   SAcctObj tObj;
   tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
@@ -98,13 +97,13 @@ int32_t mgmtInitAccts() {
     .maxRowSize   = tsAcctUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,
-    .insertFp     = mgmtAcctActionInsert,
-    .deleteFp     = mgmtActionAcctDelete,
-    .updateFp     = mgmtActionAcctUpdate,
-    .encodeFp     = mgmtActionActionEncode,
-    .decodeFp     = mgmtActionAcctDecode,
-    .destroyFp    = mgmtActionAcctDestroy,
-    .restoredFp   = mgmtActionAcctRestored
+    .insertFp     = mnodeAcctActionInsert,
+    .deleteFp     = mnodeAcctActionDelete,
+    .updateFp     = mnodeAcctActionUpdate,
+    .encodeFp     = mnodeActionActionEncode,
+    .decodeFp     = mnodeAcctActionDecode,
+    .destroyFp    = mnodeAcctActionDestroy,
+    .restoredFp   = mnodeAcctActionRestored
   };
 
   tsAcctSdb = sdbOpenTable(&tableDesc);
@@ -117,52 +116,52 @@ int32_t mgmtInitAccts() {
   return TSDB_CODE_SUCCESS;
 }
 
-void mgmtCleanUpAccts() {
+void mnodeCleanupAccts() {
   sdbCloseTable(tsAcctSdb);
   acctCleanUp();
 }
 
-void *mgmtGetAcct(char *name) {
+void *mnodeGetAcct(char *name) {
   return sdbGetRow(tsAcctSdb, name);
 }
 
-void *mgmtGetNextAcct(void *pIter, SAcctObj **pAcct) {
+void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) {
   return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); 
 }
 
-void mgmtIncAcctRef(SAcctObj *pAcct) {
+void mnodeIncAcctRef(SAcctObj *pAcct) {
   sdbIncRef(tsAcctSdb, pAcct);
 }
 
-void mgmtDecAcctRef(SAcctObj *pAcct) {
+void mnodeDecAcctRef(SAcctObj *pAcct) {
   sdbDecRef(tsAcctSdb, pAcct);
 }
 
-void mgmtAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb) {
+void mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb) {
   atomic_add_fetch_32(&pAcct->acctInfo.numOfDbs, 1);
   pDb->pAcct = pAcct;
-  mgmtIncAcctRef(pAcct);
+  mnodeIncAcctRef(pAcct);
 }
 
-void mgmtDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
+void mnodeDropDbFromAcct(SAcctObj *pAcct, SDbObj *pDb) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfDbs, 1);
   pDb->pAcct = NULL;
-  mgmtDecAcctRef(pAcct);
+  mnodeDecAcctRef(pAcct);
 }
 
-void mgmtAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser) {
+void mnodeAddUserToAcct(SAcctObj *pAcct, SUserObj *pUser) {
   atomic_add_fetch_32(&pAcct->acctInfo.numOfUsers, 1);
   pUser->pAcct = pAcct;
-  mgmtIncAcctRef(pAcct);
+  mnodeIncAcctRef(pAcct);
 }
 
-void mgmtDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
+void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
   atomic_sub_fetch_32(&pAcct->acctInfo.numOfUsers, 1);
   pUser->pAcct = NULL;
-  mgmtDecAcctRef(pAcct);
+  mnodeDecAcctRef(pAcct);
 }
 
-static void mgmtCreateRootAcct() {
+static void mnodeCreateRootAcct() {
   int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb);
   if (numOfAccts != 0) return;
 

src/mnode/src/mgmtBalance.c → src/mnode/src/mnodeBalance.c

@@ -15,22 +15,18 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "trpc.h"
-#include "tbalance.h"
 #include "tglobal.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtMnode.h"
-#include "mgmtDnode.h"
-#include "mgmtSdb.h"
-#include "mgmtVgroup.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeSdb.h"
 
 #ifndef _SYNC
 
 int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
 void    balanceCleanUp() {}
 void    balanceNotify() {}
-void    balanceUpdateMgmt() {}
+void    balanceUpdateMnode() {}
 void    balanceReset() {}
 
 int32_t balanceAllocVnodes(SVgObj *pVgroup) {
@@ -40,12 +36,12 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   float      vnodeUsage = 1000.0;
 
   while (1) {
-    pIter = mgmtGetNextDnode(pIter, &pDnode);
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
 
     if (pDnode->totalVnodes > 0 && pDnode->openVnodes < pDnode->totalVnodes) {
       float openVnodes = pDnode->openVnodes;
-      if (pDnode->isMgmt) openVnodes += tsMgmtEqualVnodeNum;
+      if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
 
       float usage = openVnodes / pDnode->totalVnodes;
       if (usage <= vnodeUsage) {
@@ -53,7 +49,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
         vnodeUsage = usage;
       }
     }
-    mgmtDecDnodeRef(pDnode);
+    mnodeDecDnodeRef(pDnode);
   }
 
   sdbFreeIter(pIter);

src/mnode/src/mgmtGrant.c → src/mnode/src/mnodeGrant.c

@@ -18,7 +18,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tgrant.h"
-#include "mgmtInt.h"
+#include "mnodeInt.h"
 
 int32_t grantInit() { return TSDB_CODE_SUCCESS; }
 void    grantCleanUp() {}

src/mnode/src/mnodeInt.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "taoserror.h"
+#include "trpc.h"
+#include "tcache.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeAcct.h"
+#include "mnodeDb.h"
+#include "mnodeDnode.h"
+#include "mnodeMnode.h"
+#include "mnodeProfile.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeTable.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
+  pMsg->rpcMsg = *rpcMsg;
+}
+
+int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
+  pMsg->pUser = mnodeGetUserFromConn(pMsg->rpcMsg.handle);
+  if (pMsg->pUser == NULL) {
+    return TSDB_CODE_INVALID_USER;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void mnodeCleanupMsg(SMnodeMsg *pMsg) {
+  if (pMsg != NULL) {
+    if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont);
+    if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser);
+    if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
+    if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);
+    if (pMsg->pTable) mnodeDecTableRef(pMsg->pTable);
+    if (pMsg->pAcct) mnodeDecAcctRef(pMsg->pAcct);
+    if (pMsg->pDnode) mnodeDecDnodeRef(pMsg->pDnode);
+  }
+}

src/mnode/src/mgmtMain.c → src/mnode/src/mnodeMain.c

@@ -22,65 +22,72 @@
 #include "ttimer.h"
 #include "tglobal.h"
 #include "dnode.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtServer.h"
-#include "mgmtAcct.h"
-#include "mgmtDnode.h"
-#include "mgmtMnode.h"
-#include "mgmtDb.h"
-#include "mgmtSdb.h"
-#include "mgmtVgroup.h"
-#include "mgmtUser.h"
-#include "mgmtTable.h"
-#include "mgmtShell.h"
-
-extern void *tsMgmtTmr;
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeAcct.h"
+#include "mnodeDnode.h"
+#include "mnodeMnode.h"
+#include "mnodeDb.h"
+#include "mnodeSdb.h"
+#include "mnodeVgroup.h"
+#include "mnodeUser.h"
+#include "mnodeTable.h"
+#include "mnodeShow.h"
+
+void *tsMnodeTmr;
 static bool tsMgmtIsRunning = false;
 
-int32_t mgmtStartSystem() {
+static void mnodeInitTimer();
+static void mnodeCleanupTimer();
+static bool mnodeNeedStart() ;
+
+int32_t mnodeStartSystem() {
   if (tsMgmtIsRunning) {
-    mPrint("TDengine mgmt module already started...");
+    mPrint("mnode module already started...");
     return 0;
   }
 
-  mPrint("starting to initialize TDengine mgmt ...");
+  mPrint("starting to initialize mnode ...");
   struct stat dirstat;
   if (stat(tsMnodeDir, &dirstat) < 0) {
     mkdir(tsMnodeDir, 0755);
   }
 
-  if (mgmtInitAccts() < 0) {
+  dnodeAllocateMnodeWqueue();
+  dnodeAllocateMnodeRqueue();
+  dnodeAllocateMnodePqueue();
+
+  if (mnodeInitAccts() < 0) {
     mError("failed to init accts");
     return -1;
   }
 
-  if (mgmtInitUsers() < 0) {
+  if (mnodeInitUsers() < 0) {
     mError("failed to init users");
     return -1;
   }
 
-  if (mgmtInitDnodes() < 0) {
+  if (mnodeInitDnodes() < 0) {
     mError("failed to init dnodes");
     return -1;
   }
 
-  if (mgmtInitDbs() < 0) {
+  if (mnodeInitDbs() < 0) {
     mError("failed to init dbs");
     return -1;
   }
 
-  if (mgmtInitVgroups() < 0) {
+  if (mnodeInitVgroups() < 0) {
     mError("failed to init vgroups");
     return -1;
   }
 
-  if (mgmtInitTables() < 0) {
+  if (mnodeInitTables() < 0) {
     mError("failed to init tables");
     return -1;
   }
 
-  if (mgmtInitMnodes() < 0) {
+  if (mnodeInitMnodes() < 0) {
     mError("failed to init mnodes");
     return -1;
   }
@@ -99,63 +106,83 @@ int32_t mgmtStartSystem() {
     return -1;
   }
 
-  if (mgmtInitServer() < 0) {
+  if (mnodeInitShow() < 0) {
+    mError("failed to init show");
     return -1;
   }
 
   grantReset(TSDB_GRANT_ALL, 0);
   tsMgmtIsRunning = true;
 
-  mPrint("TDengine mgmt is initialized successfully");
+  mPrint("mnode is initialized successfully");
 
   return 0;
 }
 
-int32_t mgmtInitSystem() {
-  if (mgmtInitShell() != 0) {
-    mError("failed to init shell");
-    return -1;
-  }
-
-  struct stat dirstat;
-  bool fileExist  = (stat(tsMnodeDir, &dirstat) == 0);
-  bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0);
-
-  if (asMaster || fileExist) {
-    if (mgmtStartSystem() != 0) {
-      return -1;
-    }
-  }
-
-  return 0;
+int32_t mnodeInitSystem() {
+  mnodeInitTimer();
+  if (!mnodeNeedStart()) return 0;
+  return mnodeStartSystem();
 }
 
-void mgmtCleanUpSystem() {
-  mPrint("starting to clean up mgmt");
+void mnodeCleanupSystem() {
+  mPrint("starting to clean up mnode");
   tsMgmtIsRunning = false;
-  mgmtCleanUpShell();
-  mgmtCleanupServer();
+
+  dnodeFreeMnodeWqueue();
+  dnodeFreeMnodeRqueue();
+  dnodeFreeMnodePqueue();
+  mnodeCleanupTimer();
+  mnodeCleanUpShow();
   grantCleanUp();
   balanceCleanUp();
   sdbCleanUp();
-  mgmtCleanupMnodes();
-  mgmtCleanUpTables();
-  mgmtCleanUpVgroups();
-  mgmtCleanUpDbs();
-  mgmtCleanupDnodes();
-  mgmtCleanUpUsers();
-  mgmtCleanUpAccts();
-  mPrint("mgmt is cleaned up");
+  mnodeCleanupMnodes();
+  mnodeCleanupTables();
+  mnodeCleanupVgroups();
+  mnodeCleanupDbs();
+  mnodeCleanupDnodes();
+  mnodeCleanupUsers();
+  mnodeCleanupAccts();
+  mPrint("mnode is cleaned up");
 }
 
-void mgmtStopSystem() {
+void mnodeStopSystem() {
   if (sdbIsMaster()) {
-    mTrace("it is a master mgmt node, it could not be stopped");
+    mTrace("it is a master mnode, it could not be stopped");
     return;
   }
+  
+  mnodeCleanupSystem();
+  mPrint("mnode file is removed");
+  remove(tsMnodeDir);
+}
+
+static void mnodeInitTimer() {
+  if (tsMnodeTmr == NULL) {
+    tsMnodeTmr = taosTmrInit((tsMaxShellConns)*3, 200, 3600000, "MND");
+  }
+}
 
-  mgmtCleanUpSystem();
+static void mnodeCleanupTimer() {
+  if (tsMnodeTmr != NULL) {
+    taosTmrCleanUp(tsMnodeTmr);
+    tsMnodeTmr = NULL;
+  }
+}
 
-  mPrint("mgmt file is removed");
-  remove(tsMnodeDir);
+static bool mnodeNeedStart() {
+  struct stat dirstat;
+  bool fileExist = (stat(tsMnodeDir, &dirstat) == 0);
+  bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0);
+
+  if (asMaster || fileExist) {
+    return true;
+  }
+
+  return false;
 }
+
+bool mnodeIsRunning() {
+  return tsMgmtIsRunning;
+}
\ No newline at end of file

src/mnode/src/mgmtMnode.c → src/mnode/src/mnodeMnode.c

@@ -23,85 +23,85 @@
 #include "ttime.h"
 #include "tsocket.h"
 #include "tdataformat.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtMnode.h"
-#include "mgmtDnode.h"
-#include "mgmtSdb.h"
-#include "mgmtShell.h"
-#include "mgmtUser.h"
-
-static void *  tsMnodeSdb = NULL;
-static int32_t tsMnodeUpdateSize = 0;
-static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
-static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
-
-static SRpcIpSet tsMnodeRpcIpSet;
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeMnode.h"
+#include "mnodeDnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+
+static void *        tsMnodeSdb = NULL;
+static int32_t       tsMnodeUpdateSize = 0;
+static SRpcIpSet     tsMnodeIpSetForShell;
+static SRpcIpSet     tsMnodeIpSetForPeer;
 static SDMMnodeInfos tsMnodeInfos;
+static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 
 #if defined(LINUX)
-  static pthread_rwlock_t        tsMnodeLock;
-  #define mgmtMnodeWrLock()      pthread_rwlock_wrlock(&tsMnodeLock)
-  #define mgmtMnodeRdLock()      pthread_rwlock_rdlock(&tsMnodeLock)
-  #define mgmtMnodeUnLock()      pthread_rwlock_unlock(&tsMnodeLock)
-  #define mgmtMnodeInitLock()    pthread_rwlock_init(&tsMnodeLock, NULL)
-  #define mgmtMnodeDestroyLock() pthread_rwlock_destroy(&tsMnodeLock)
+  static pthread_rwlock_t         tsMnodeLock;
+  #define mnodeMnodeWrLock()      pthread_rwlock_wrlock(&tsMnodeLock)
+  #define mnodeMnodeRdLock()      pthread_rwlock_rdlock(&tsMnodeLock)
+  #define mnodeMnodeUnLock()      pthread_rwlock_unlock(&tsMnodeLock)
+  #define mnodeMnodeInitLock()    pthread_rwlock_init(&tsMnodeLock, NULL)
+  #define mnodeMnodeDestroyLock() pthread_rwlock_destroy(&tsMnodeLock)
 #else
-  static pthread_mutex_t         tsMnodeLock;
-  #define mgmtMnodeWrLock()      pthread_mutex_lock(&tsMnodeLock)
-  #define mgmtMnodeRdLock()      pthread_mutex_lock(&tsMnodeLock)
-  #define mgmtMnodeUnLock()      pthread_mutex_unlock(&tsMnodeLock)
-  #define mgmtMnodeInitLock()    pthread_mutex_init(&tsMnodeLock, NULL)
-  #define mgmtMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
+  static pthread_mutex_t          tsMnodeLock;
+  #define mnodeMnodeWrLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mnodeMnodeRdLock()      pthread_mutex_lock(&tsMnodeLock)
+  #define mnodeMnodeUnLock()      pthread_mutex_unlock(&tsMnodeLock)
+  #define mnodeMnodeInitLock()    pthread_mutex_init(&tsMnodeLock, NULL)
+  #define mnodeMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
 #endif
 
-static int32_t mgmtMnodeActionDestroy(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) {
   tfree(pOper->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionInsert(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {
   SMnodeObj *pMnode = pOper->pObj;
-  SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+  SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
 
   pDnode->isMgmt = true;
-  mgmtDecDnodeRef(pDnode);
+  mnodeDecDnodeRef(pDnode);
   
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionDelete(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionDelete(SSdbOper *pOper) {
   SMnodeObj *pMnode = pOper->pObj;
 
-  SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+  SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_DNODE_NOT_EXIST;
   pDnode->isMgmt = false;
-  mgmtDecDnodeRef(pDnode);
+  mnodeDecDnodeRef(pDnode);
 
   mTrace("mnode:%d, is dropped from sdb", pMnode->mnodeId);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionUpdate(SSdbOper *pOper) {
   SMnodeObj *pMnode = pOper->pObj;
-  SMnodeObj *pSaved = mgmtGetMnode(pMnode->mnodeId);
+  SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId);
   if (pMnode != pSaved) {
     memcpy(pSaved, pMnode, pOper->rowSize);
     free(pMnode);
   }
-  mgmtDecMnodeRef(pSaved);
+  mnodeDecMnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionEncode(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionEncode(SSdbOper *pOper) {
   SMnodeObj *pMnode = pOper->pObj;
   memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize);
   pOper->rowSize = tsMnodeUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionDecode(SSdbOper *pOper) {
   SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
   if (pMnode == NULL) return TSDB_CODE_SERV_OUT_OF_MEMORY;
 
@@ -110,24 +110,24 @@ static int32_t mgmtMnodeActionDecode(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mgmtMnodeActionRestored() {
-  if (mgmtGetMnodesNum() == 1) {
+static int32_t mnodeMnodeActionRestored() {
+  if (mnodeGetMnodesNum() == 1) {
     SMnodeObj *pMnode = NULL;
-    void *pIter = mgmtGetNextMnode(NULL, &pMnode);
+    void *pIter = mnodeGetNextMnode(NULL, &pMnode);
     if (pMnode != NULL) {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
-      mgmtDecMnodeRef(pMnode);
+      mnodeDecMnodeRef(pMnode);
     }
     sdbFreeIter(pIter);
   }
 
-  mgmtUpdateMnodeIpSet();
+  mnodeUpdateMnodeIpSet();
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t mgmtInitMnodes() {
-  mgmtMnodeInitLock();
+int32_t mnodeInitMnodes() {
+  mnodeMnodeInitLock();
 
   SMnodeObj tObj;
   tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
@@ -139,13 +139,13 @@ int32_t mgmtInitMnodes() {
     .maxRowSize   = tsMnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_INT,
-    .insertFp     = mgmtMnodeActionInsert,
-    .deleteFp     = mgmtMnodeActionDelete,
-    .updateFp     = mgmtMnodeActionUpdate,
-    .encodeFp     = mgmtMnodeActionEncode,
-    .decodeFp     = mgmtMnodeActionDecode,
-    .destroyFp    = mgmtMnodeActionDestroy,
-    .restoredFp   = mgmtMnodeActionRestored
+    .insertFp     = mnodeMnodeActionInsert,
+    .deleteFp     = mnodeMnodeActionDelete,
+    .updateFp     = mnodeMnodeActionUpdate,
+    .encodeFp     = mnodeMnodeActionEncode,
+    .decodeFp     = mnodeMnodeActionDecode,
+    .destroyFp    = mnodeMnodeActionDestroy,
+    .restoredFp   = mnodeMnodeActionRestored
   };
 
   tsMnodeSdb = sdbOpenTable(&tableDesc);
@@ -154,39 +154,39 @@ int32_t mgmtInitMnodes() {
     return -1;
   }
 
-  mgmtAddShellShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mgmtGetMnodeMeta);
-  mgmtAddShellShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mgmtRetrieveMnodes);
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes);
 
   mTrace("table:mnodes table is created");
   return TSDB_CODE_SUCCESS;
 }
 
-void mgmtCleanupMnodes() {
+void mnodeCleanupMnodes() {
   sdbCloseTable(tsMnodeSdb);
-  mgmtMnodeDestroyLock();
+  mnodeMnodeDestroyLock();
 }
 
-int32_t mgmtGetMnodesNum() { 
+int32_t mnodeGetMnodesNum() { 
   return sdbGetNumOfRows(tsMnodeSdb); 
 }
 
-void *mgmtGetMnode(int32_t mnodeId) {
+void *mnodeGetMnode(int32_t mnodeId) {
   return sdbGetRow(tsMnodeSdb, &mnodeId);
 }
 
-void mgmtIncMnodeRef(SMnodeObj *pMnode) {
+void mnodeIncMnodeRef(SMnodeObj *pMnode) {
   sdbIncRef(tsMnodeSdb, pMnode);
 }
 
-void mgmtDecMnodeRef(SMnodeObj *pMnode) {
+void mnodeDecMnodeRef(SMnodeObj *pMnode) {
   sdbDecRef(tsMnodeSdb, pMnode);
 }
 
-void *mgmtGetNextMnode(void *pIter, SMnodeObj **pMnode) { 
+void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { 
   return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); 
 }
 
-char *mgmtGetMnodeRoleStr(int32_t role) {
+char *mnodeGetMnodeRoleStr(int32_t role) {
   switch (role) {
     case TAOS_SYNC_ROLE_OFFLINE:
       return "offline";
@@ -201,68 +201,77 @@ char *mgmtGetMnodeRoleStr(int32_t role) {
   }
 }
 
-void mgmtUpdateMnodeIpSet() {
-  SRpcIpSet *ipSet = &tsMnodeRpcIpSet;
-  SDMMnodeInfos *mnodes = &tsMnodeInfos;
+void mnodeUpdateMnodeIpSet() {
+  mPrint("update mnodes ipset, numOfIps:%d ", mnodeGetMnodesNum());
 
-  mPrint("update mnodes ipset, numOfIps:%d ", mgmtGetMnodesNum());
+  mnodeMnodeWrLock();
 
-  mgmtMnodeWrLock();
-
-  memset(ipSet, 0, sizeof(tsMnodeRpcIpSet));
-  memset(mnodes, 0, sizeof(SDMMnodeInfos));
+  memset(&tsMnodeIpSetForShell, 0, sizeof(SRpcIpSet));
+  memset(&tsMnodeIpSetForPeer, 0, sizeof(SRpcIpSet));
+  memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
 
   int32_t index = 0;
   void *  pIter = NULL;
   while (1) {
     SMnodeObj *pMnode = NULL;
-    pIter = mgmtGetNextMnode(pIter, &pMnode);
+    pIter = mnodeGetNextMnode(pIter, &pMnode);
     if (pMnode == NULL) break;
 
-    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+    SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
     if (pDnode != NULL) {
-      strcpy(ipSet->fqdn[ipSet->numOfIps], pDnode->dnodeFqdn);
-      ipSet->port[ipSet->numOfIps] = htons(pDnode->dnodePort);
+      strcpy(tsMnodeIpSetForShell.fqdn[index], pDnode->dnodeFqdn);
+      tsMnodeIpSetForShell.port[index] = htons(pDnode->dnodePort);
+      mTrace("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForShell.fqdn[index], htons(tsMnodeIpSetForShell.port[index]));      
+
+      strcpy(tsMnodeIpSetForPeer.fqdn[index], pDnode->dnodeFqdn);
+      tsMnodeIpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE);
+      mTrace("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForPeer.fqdn[index], htons(tsMnodeIpSetForPeer.port[index]));
 
-      mnodes->nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
-      strcpy(mnodes->nodeInfos[index].nodeEp, pDnode->dnodeEp);
+      tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId);
+      strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp);
 
       if (pMnode->role == TAOS_SYNC_ROLE_MASTER) {
-        ipSet->inUse = ipSet->numOfIps;
-        mnodes->inUse = index;
+        tsMnodeIpSetForShell.inUse = index;
+        tsMnodeIpSetForPeer.inUse = index;
+        tsMnodeInfos.inUse = index;
       }
 
-      mPrint("mnode:%d, ep:%s %s", index, pDnode->dnodeEp,
-             pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
-
-      ipSet->numOfIps++;
+      mPrint("mnode:%d, ep:%s %s", pDnode->dnodeId, pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : "");
       index++;
     }
 
-    mgmtDecDnodeRef(pDnode);
-    mgmtDecMnodeRef(pMnode);
+    mnodeDecDnodeRef(pDnode);
+    mnodeDecMnodeRef(pMnode);
   }
 
-  mnodes->nodeNum = index;
+  tsMnodeInfos.nodeNum = index;
+  tsMnodeIpSetForShell.numOfIps = index;
+  tsMnodeIpSetForPeer.numOfIps = index;
 
   sdbFreeIter(pIter);
 
-  mgmtMnodeUnLock();
+  mnodeMnodeUnLock();
+}
+
+void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet) {
+  mnodeMnodeRdLock();
+  *ipSet = tsMnodeIpSetForPeer;
+  mnodeMnodeUnLock();
 }
 
-void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
-  mgmtMnodeRdLock();
-  *ipSet = tsMnodeRpcIpSet;
-  mgmtMnodeUnLock();
+void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet) {
+  mnodeMnodeRdLock();
+  *ipSet = tsMnodeIpSetForShell;
+  mnodeMnodeUnLock();
 }
 
-void mgmtGetMnodeInfos(void *mnodeInfos) {
-  mgmtMnodeRdLock();
+void mnodeGetMnodeInfos(void *mnodeInfos) {
+  mnodeMnodeRdLock();
   *(SDMMnodeInfos *)mnodeInfos = tsMnodeInfos;
-  mgmtMnodeUnLock();
+  mnodeMnodeUnLock();
 }
 
-int32_t mgmtAddMnode(int32_t dnodeId) {
+int32_t mnodeAddMnode(int32_t dnodeId) {
   SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
   pMnode->mnodeId = dnodeId;
   pMnode->createdTime = taosGetTimestampMs();
@@ -279,24 +288,24 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
     code = TSDB_CODE_SDB_ERROR;
   }
 
-  mgmtUpdateMnodeIpSet();
+  mnodeUpdateMnodeIpSet();
 
   return code;
 }
 
-void mgmtDropMnodeLocal(int32_t dnodeId) {
-  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
+void mnodeDropMnodeLocal(int32_t dnodeId) {
+  SMnodeObj *pMnode = mnodeGetMnode(dnodeId);
   if (pMnode != NULL) {
     SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
     sdbDeleteRow(&oper);
-    mgmtDecMnodeRef(pMnode);
+    mnodeDecMnodeRef(pMnode);
   }
 
-  mgmtUpdateMnodeIpSet();
+  mnodeUpdateMnodeIpSet();
 }
 
-int32_t mgmtDropMnode(int32_t dnodeId) {
-  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
+int32_t mnodeDropMnode(int32_t dnodeId) {
+  SMnodeObj *pMnode = mnodeGetMnode(dnodeId);
   if (pMnode == NULL) {
     return TSDB_CODE_DNODE_NOT_EXIST;
   }
@@ -314,18 +323,18 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
 
   sdbDecRef(tsMnodeSdb, pMnode);
 
-  mgmtUpdateMnodeIpSet();
+  mnodeUpdateMnodeIpSet();
 
   return code;
 }
 
-static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   sdbUpdateMnodeRoles();
-  SUserObj *pUser = mgmtGetUserFromConn(pConn);
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->pAcct->user, "root") != 0)  {
-    mgmtDecUserRef(pUser);
+    mnodeDecUserRef(pUser);
     return TSDB_CODE_NO_RIGHTS;
   }
 
@@ -364,22 +373,22 @@ static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo
     pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
   }
 
-  pShow->numOfRows = mgmtGetMnodesNum();
+  pShow->numOfRows = mnodeGetMnodesNum();
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pIter = NULL;
-  mgmtDecUserRef(pUser);
+  mnodeDecUserRef(pUser);
 
   return 0;
 }
 
-static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
   int32_t    numOfRows = 0;
   int32_t    cols      = 0;
   SMnodeObj *pMnode   = NULL;
   char      *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pIter = mgmtGetNextMnode(pShow->pIter, &pMnode);
+    pShow->pIter = mnodeGetNextMnode(pShow->pIter, &pMnode);
     if (pMnode == NULL) break;
 
     cols = 0;
@@ -390,18 +399,18 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     
-    SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+    SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
     if (pDnode != NULL) {
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - VARSTR_HEADER_SIZE);
     } else {
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols] - VARSTR_HEADER_SIZE);
     }
-    mgmtDecDnodeRef(pDnode);
+    mnodeDecDnodeRef(pDnode);
 
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    char* roles = mgmtGetMnodeRoleStr(pMnode->role);
+    char* roles = mnodeGetMnodeRoleStr(pMnode->role);
     STR_TO_VARSTR(pWrite, roles);
     cols++;
 
@@ -411,7 +420,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     
     numOfRows++;
 
-    mgmtDecMnodeRef(pMnode);
+    mnodeDecMnodeRef(pMnode);
   }
 
   pShow->numOfReads += numOfRows;

src/mnode/src/mnodePeer.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tsched.h"
+#include "tsystem.h"
+#include "tutil.h"
+#include "tgrant.h"
+#include "tbalance.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeProfile.h"
+#include "mnodeShow.h"
+#include "mnodeSdb.h"
+#include "mnodeTable.h"
+#include "mnodeVgroup.h"
+
+static int32_t (*tsMnodeProcessPeerMsgFp[TSDB_MSG_TYPE_MAX])(SMnodeMsg *);
+static void (*tsMnodeProcessPeerRspFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
+
+void mnodeAddPeerMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg)) {
+  tsMnodeProcessPeerMsgFp[msgType] = fp;
+}
+
+void mnodeAddPeerRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
+  tsMnodeProcessPeerRspFp[msgType] = fp;
+}
+
+int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
+  if (pMsg->rpcMsg.pCont == NULL) {
+    mError("%p, msg:%s in mpeer queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    return TSDB_CODE_INVALID_MSG_LEN;
+  }
+
+  if (!sdbIsMaster()) {
+    SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
+    SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet));
+    mnodeGetMnodeIpSetForPeer(ipSet);
+    rpcRsp->rsp = ipSet;
+    rpcRsp->len = sizeof(SRpcIpSet);
+
+    mTrace("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse);
+    for (int32_t i = 0; i < ipSet->numOfIps; ++i) {
+      mTrace("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i]));
+    }
+
+    return TSDB_CODE_REDIRECT;
+  }
+
+  if (tsMnodeProcessPeerMsgFp[pMsg->rpcMsg.msgType] == NULL) {
+    mError("%p, msg:%s in mpeer queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    return TSDB_CODE_MSG_NOT_PROCESSED;
+  }
+
+  return (*tsMnodeProcessPeerMsgFp[pMsg->rpcMsg.msgType])(pMsg);
+}
+
+void mnodeProcessPeerRsp(SRpcMsg *pMsg) {
+  if (tsMnodeProcessPeerRspFp[pMsg->msgType]) {
+    (*tsMnodeProcessPeerRspFp[pMsg->msgType])(pMsg);
+  } else {
+    mError("%p, msg:%s is not processed", pMsg->ahandle, taosMsg[pMsg->msgType]);
+  }
+}

src/mnode/src/mnodeRead.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosdef.h"
+#include "tsched.h"
+#include "tbalance.h"
+#include "tgrant.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "mnode.h"
+#include "dnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeAcct.h"
+#include "mnodeDnode.h"
+#include "mnodeMnode.h"
+#include "mnodeDb.h"
+#include "mnodeSdb.h"
+#include "mnodeVgroup.h"
+#include "mnodeUser.h"
+#include "mnodeTable.h"
+#include "mnodeShow.h"
+
+static int32_t (*tsMnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SMnodeMsg *);
+
+void mnodeAddReadMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *pMsg)) {
+  tsMnodeProcessReadMsgFp[msgType] = fp;
+}
+
+int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
+  if (pMsg->rpcMsg.pCont == NULL) {
+    mError("%p, msg:%s in mread queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    return TSDB_CODE_INVALID_MSG_LEN;
+  }
+
+  if (!sdbIsMaster()) {
+    SMnodeRsp *rpcRsp = &pMsg->rpcRsp;
+    SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet));
+    mnodeGetMnodeIpSetForShell(ipSet);
+    rpcRsp->rsp = ipSet;
+    rpcRsp->len = sizeof(SRpcIpSet);
+
+    mTrace("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse);
+    for (int32_t i = 0; i < ipSet->numOfIps; ++i) {
+      mTrace("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i]));
+    }
+
+    return TSDB_CODE_REDIRECT;
+  }
+
+  if (tsMnodeProcessReadMsgFp[pMsg->rpcMsg.msgType] == NULL) {
+    mError("%p, msg:%s in mread queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    return TSDB_CODE_MSG_NOT_PROCESSED;
+  }
+
+  int32_t code = mnodeInitMsg(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    mError("%p, msg:%s in mread queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code));
+    return code;
+  }
+
+  return (*tsMnodeProcessReadMsgFp[pMsg->rpcMsg.msgType])(pMsg);
+}

src/mnode/src/mgmtSdb.c → src/mnode/src/mnodeSdb.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taoserror.h"
 #include "hash.h"
-#include "trpc.h"
 #include "tutil.h"
 #include "tbalance.h"
 #include "tqueue.h"
@@ -25,11 +24,11 @@
 #include "tsync.h"
 #include "tglobal.h"
 #include "dnode.h"
-#include "mgmtDef.h"
-#include "mgmtInt.h"
-#include "mgmtMnode.h"
-#include "mgmtDnode.h"
-#include "mgmtSdb.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeMnode.h"
+#include "mnodeDnode.h"
+#include "mnodeSdb.h"
 
 typedef enum {
   SDB_ACTION_INSERT,
@@ -187,18 +186,18 @@ void sdbUpdateMnodeRoles() {
   SNodesRole roles = {0};
   syncGetNodesRole(tsSdbObj.sync, &roles);
 
-  sdbPrint("update mnodes:%d sync roles", tsSdbObj.cfg.replica);
+  sdbPrint("update mnodes sync roles, total:%d", tsSdbObj.cfg.replica);
   for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) {
-    SMnodeObj *pMnode = mgmtGetMnode(roles.nodeId[i]);
+    SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]);
     if (pMnode != NULL) {
       pMnode->role = roles.role[i];
-      sdbPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
+      sdbPrint("mnode:%d, role:%s", pMnode->mnodeId, mnodeGetMnodeRoleStr(pMnode->role));
       if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbObj.role = pMnode->role;
-      mgmtDecMnodeRef(pMnode);
+      mnodeDecMnodeRef(pMnode);
     }
   }
 
-  mgmtUpdateMnodeIpSet();
+  mnodeUpdateMnodeIpSet();
 }
 
 static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
@@ -211,7 +210,7 @@ static int sdbGetWalInfo(void *ahandle, char *name, uint32_t *index) {
 }
 
 static void sdbNotifyRole(void *ahandle, int8_t role) {
-  sdbPrint("mnode role changed from %s to %s", mgmtGetMnodeRoleStr(tsSdbObj.role), mgmtGetMnodeRoleStr(role));
+  sdbPrint("mnode role changed from %s to %s", mnodeGetMnodeRoleStr(tsSdbObj.role), mnodeGetMnodeRoleStr(role));
 
   if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) {
     balanceReset();
@@ -256,20 +255,20 @@ void sdbUpdateSync() {
     void *pIter = NULL;
     while (1) {
       SMnodeObj *pMnode = NULL;
-      pIter = mgmtGetNextMnode(pIter, &pMnode);
+      pIter = mnodeGetNextMnode(pIter, &pMnode);
       if (pMnode == NULL) break;
 
       syncCfg.nodeInfo[index].nodeId = pMnode->mnodeId;
 
-      SDnodeObj *pDnode = mgmtGetDnode(pMnode->mnodeId);
+      SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
       if (pDnode != NULL) {
         syncCfg.nodeInfo[index].nodePort = pDnode->dnodePort + TSDB_PORT_SYNC;
         strcpy(syncCfg.nodeInfo[index].nodeFqdn, pDnode->dnodeEp);
         index++;
       }
 
-      mgmtDecDnodeRef(pDnode);
-      mgmtDecMnodeRef(pMnode);
+      mnodeDecDnodeRef(pDnode);
+      mnodeDecMnodeRef(pMnode);
     }
     sdbFreeIter(pIter);
   }
@@ -324,7 +323,7 @@ int32_t sdbInit() {
   
   sdbRestoreTables();
 
-  if (mgmtGetMnodesNum() == 1) {
+  if (mnodeGetMnodesNum() == 1) {
     tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
   }
 

src/plugins/http/inc/httpLog.h

@@ -21,7 +21,7 @@
 extern int32_t httpDebugFlag;
 
 #define httpError(...) { if (httpDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR HTP ", 255, __VA_ARGS__); }}
-#define httpWarn(...)  { if (httpDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  HTP ", httpDebugFlag, __VA_ARGS__); }}
+#define httpWarn(...)  { if (httpDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN HTP ", httpDebugFlag, __VA_ARGS__); }}
 #define httpTrace(...) { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLog("HTP ", httpDebugFlag, __VA_ARGS__); }}
 #define httpDump(...)  { if (httpDebugFlag & DEBUG_TRACE) { taosPrintLongString("HTP ", httpDebugFlag, __VA_ARGS__); }}
 #define httpPrint(...) { taosPrintLog("HTP ", 255, __VA_ARGS__); }

src/plugins/monitor/src/monitorMain.c

@@ -28,7 +28,7 @@
 #include "monitor.h"
 
 #define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR MON ", 255, __VA_ARGS__); }}
-#define monitorWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define monitorWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN MON ", monitorDebugFlag, __VA_ARGS__); }}
 #define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
 #define monitorPrint(...) { taosPrintLog("MON ", 255, __VA_ARGS__); }
 

src/plugins/mqtt/inc/mqttLog.h

@@ -26,7 +26,7 @@ extern int32_t mqttDebugFlag;
   }
 #define mqttWarn(...)                                  \
   if ( mqttDebugFlag & DEBUG_WARN) {                    \
-    taosPrintLog("WARN  MQT ",  mqttDebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN MQT ",  mqttDebugFlag, __VA_ARGS__); \
   }
 #define  mqttTrace(...)                           \
   if ( mqttDebugFlag & DEBUG_TRACE) {             \

src/rpc/inc/rpcLog.h

@@ -25,7 +25,7 @@ extern "C" {
 extern int32_t rpcDebugFlag;
 
 #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR RPC ", rpcDebugFlag, __VA_ARGS__); }}
-#define tWarn(...)  { if (rpcDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN  RPC ", rpcDebugFlag, __VA_ARGS__); }}
+#define tWarn(...)  { if (rpcDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN RPC ", rpcDebugFlag, __VA_ARGS__); }}
 #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", rpcDebugFlag, __VA_ARGS__); }}
 #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP)  { taosDumpData((unsigned char *)x, y); }}
 #define tPrint(...) { taosPrintLog("RPC ", 255, __VA_ARGS__); }

src/util/src/ttimer.c

@@ -27,7 +27,7 @@
 
 #define tmrWarn(...)                                  \
   do { if (tmrDebugFlag & DEBUG_WARN) {               \
-    taosPrintLog("WARN  TMR ", tmrDebugFlag, __VA_ARGS__); \
+    taosPrintLog("WARN TMR ", tmrDebugFlag, __VA_ARGS__); \
   } } while(0)
 
 #define tmrTrace(...)                           \

src/vnode/inc/vnodeInt.h

@@ -27,10 +27,10 @@ extern "C" {
 
 extern int32_t vDebugFlag;
 
-#define vError(...) if (vDebugFlag & DEBUG_ERROR) {taosPrintLog("ERROR VND ", 255, __VA_ARGS__); }
-#define vWarn(...) if (vDebugFlag & DEBUG_WARN) {taosPrintLog("WARN  VND ", vDebugFlag, __VA_ARGS__); }
-#define vTrace(...) if (vDebugFlag & DEBUG_TRACE) {taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }
-#define vPrint(...) {taosPrintLog("VND ", 255, __VA_ARGS__); }
+#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR VND ", 255, __VA_ARGS__); }}
+#define vWarn(...)  { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN VND ", vDebugFlag, __VA_ARGS__); }}
+#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
+#define vPrint(...) { taosPrintLog("VND ", 255, __VA_ARGS__); }
 
 typedef struct {
   int32_t      vgId;      // global vnode group ID

tests/pytest/util/dnodes.py

@@ -159,7 +159,7 @@ class TDDnode:
         self.cfg("dataDir", self.dataDir)
         self.cfg("logDir", self.logDir)
         self.cfg("numOfLogLines", "100000000")
-        self.cfg("mgmtEqualVnodeNum", "0")
+        self.cfg("mnodeEqualVnodeNum", "0")
         self.cfg("clog", "1")
         self.cfg("statusInterval", "1")
         self.cfg("numOfTotalVnodes", "64")

tests/script/general/alter/count.sim

@@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1
 
 system sh/cfg.sh -n dnode1 -c wallevel -v 0
 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode1 -c mgmtEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
 
 print ========= start dnode1 as master
 system sh/exec.sh -n dnode1 -s start