From 0c91ffae759e8a6b38b386bed1f63ed61dc898ea Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 4 Jul 2020 14:44:25 +0800
Subject: [PATCH] [TD-825] vnodehash may be null while close all vnodes

---
 src/dnode/src/dnodeMgmt.c         | 17 ++++++++++------
 src/inc/vnode.h                   |  4 +++-
 src/plugins/mqtt/src/mqttSystem.c |  2 +-
 src/vnode/src/vnodeMain.c         | 34 +++++++++++++++++++++++++------
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c
index 9cf024ba83..10eb77058b 100644
--- a/src/dnode/src/dnodeMgmt.c
+++ b/src/dnode/src/dnodeMgmt.c
@@ -176,6 +176,7 @@ void dnodeCleanupMgmt() {
   tsMgmtQset = NULL;
   tsMgmtQueue = NULL;
 
+  vnodeCleanupResources();
 }
 
 void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
@@ -242,8 +243,14 @@ static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
       int32_t vnode = atoi(de->d_name + 5);
       if (vnode == 0) continue;
 
-      vnodeList[*numOfVnodes] = vnode;
       (*numOfVnodes)++;
+
+      if (*numOfVnodes >= TSDB_MAX_VNODES) {
+        dError("vgId:%d, too many vnode directory in disk, exist:%d max:%d", vnode, *numOfVnodes, TSDB_MAX_VNODES);
+        continue;
+      } else {
+        vnodeList[*numOfVnodes - 1] = vnode;
+      }
     }
   }
   closedir(dir);
@@ -337,7 +344,7 @@ static int32_t dnodeOpenVnodes() {
 void dnodeStartStream() {
   int32_t vnodeList[TSDB_MAX_VNODES];
   int32_t numOfVnodes = 0;
-  int32_t status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
+  int32_t status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
 
   if (status != TSDB_CODE_SUCCESS) {
     dInfo("get dnode list failed");
@@ -352,15 +359,14 @@ void dnodeStartStream() {
 }
 
 static void dnodeCloseVnodes() {
-  int32_t *vnodeList = (int32_t *)malloc(sizeof(int32_t) * TSDB_MAX_VNODES);
+  int32_t vnodeList[TSDB_MAX_VNODES];
   int32_t numOfVnodes;
   int32_t status;
 
-  status = dnodeGetVnodeList(vnodeList, &numOfVnodes);
+  status = vnodeGetVnodeList(vnodeList, &numOfVnodes);
 
   if (status != TSDB_CODE_SUCCESS) {
     dInfo("get dnode list failed");
-    free(vnodeList);
     return;
   }
 
@@ -368,7 +374,6 @@ static void dnodeCloseVnodes() {
     vnodeClose(vnodeList[i]);
   }
 
-  free(vnodeList);
   dInfo("total vnodes:%d are all closed", numOfVnodes);
 }
 
diff --git a/src/inc/vnode.h b/src/inc/vnode.h
index 9f0c8cc241..971f341258 100644
--- a/src/inc/vnode.h
+++ b/src/inc/vnode.h
@@ -57,8 +57,10 @@ void*   vnodeGetWqueue(int32_t vgId);
 void*   vnodeGetWal(void *pVnode);
 
 int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item);
-void    vnodeBuildStatusMsg(void * param);
+int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes);
+void    vnodeBuildStatusMsg(void *param);
 void    vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes);
+void    vnodeCleanupResources();
 
 int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg);
 
diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c
index 3df62f8bf4..2687106124 100644
--- a/src/plugins/mqtt/src/mqttSystem.c
+++ b/src/plugins/mqtt/src/mqttSystem.c
@@ -111,7 +111,7 @@ void mqttStopSystem() {
 }
 
 void mqttCleanUpSystem() {
-  mqttInfo("starting to clean  up mqtt");
+  mqttInfo("starting to cleanup mqtt");
   free(recntStatus.user_name);
   free(recntStatus.password);
   free(recntStatus.hostname);
diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c
index 5eb78fda52..ec97cbc48c 100644
--- a/src/vnode/src/vnodeMain.c
+++ b/src/vnode/src/vnodeMain.c
@@ -69,6 +69,12 @@ static void vnodeInit() {
   }
 }
 
+void vnodeCleanupResources() {
+  taosHashCleanup(tsDnodeVnodesHash);
+  vnodeModuleInit = PTHREAD_ONCE_INIT;
+  tsDnodeVnodesHash = NULL;
+}
+
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
   int32_t code;
   pthread_once(&vnodeModuleInit, vnodeInit);
@@ -362,12 +368,6 @@ void vnodeRelease(void *pVnodeRaw) {
 
   int32_t count = atomic_sub_fetch_32(&tsOpennedVnodes, 1);
   vDebug("vgId:%d, vnode is released, vnodes:%d", vgId, count);
-
-  if (count <= 0) {
-    taosHashCleanup(tsDnodeVnodesHash);
-    vnodeModuleInit = PTHREAD_ONCE_INIT;
-    tsDnodeVnodesHash = NULL;
-  }
 }
 
 void *vnodeGetVnode(int32_t vgId) {
@@ -424,6 +424,28 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) {
   pLoad->replica = pVnode->syncCfg.replica;  
 }
 
+int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
+  if (tsDnodeVnodesHash == NULL) return TSDB_CODE_SUCCESS;
+
+  SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+  while (taosHashIterNext(pIter)) {
+    SVnodeObj **pVnode = taosHashIterGet(pIter);
+    if (pVnode == NULL) continue;
+    if (*pVnode == NULL) continue;
+
+    (*numOfVnodes)++;
+    if (*numOfVnodes >= TSDB_MAX_VNODES) {
+      vError("vgId:%d, too many open vnodes, exist:%d max:%d", (*pVnode)->vgId, *numOfVnodes, TSDB_MAX_VNODES);
+      continue;
+    } else {
+      vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
+    }
+  }
+
+  taosHashDestroyIter(pIter);
+  return TSDB_CODE_SUCCESS;
+}
+
 void vnodeBuildStatusMsg(void *param) {
   SDMStatusMsg *pStatus = param;
   SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
-- 
GitLab