Merge pull request #2080 from taosdata/feature/crash_gen

Feature/crash gen

src/client/src/tscSql.c

@@ -579,9 +579,9 @@ void taos_free_result_imp(TAOS_RES *res, int keepCmd) {
   if ((pCmd->command == TSDB_SQL_SELECT ||
        pCmd->command == TSDB_SQL_SHOW ||
        pCmd->command == TSDB_SQL_RETRIEVE ||
-       pCmd->command == TSDB_SQL_FETCH) &&
-       (pRes->code != TSDB_CODE_QUERY_CANCELLED && ((pCmd->command < TSDB_SQL_LOCAL && pRes->completed == false) ||
-       (pRes->code == TSDB_CODE_SUCCESS && pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)))) {
+       pCmd->command == TSDB_SQL_FETCH) && pRes->code == TSDB_CODE_SUCCESS &&
+       ((pCmd->command < TSDB_SQL_LOCAL && pRes->completed == false) ||
+       (pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL))) {
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
 
     tscTrace("%p send msg to free qhandle in vnode, code:%d, numOfRows:%d, command:%s", pSql, pRes->code, pRes->numOfRows,

src/client/src/tscUtil.c

@@ -1850,8 +1850,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   STableMetaInfo* pFinalInfo = NULL;
 
   if (pPrevSql == NULL) {
-    STableMeta* pTableMeta = taosCacheAcquireByName(tscCacheHandle, name);
-    // todo handle error
+    STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
     assert(pTableMeta != NULL);
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.

src/mnode/inc/mnodeDef.h

@@ -220,6 +220,7 @@ typedef struct SAcctObj {
 
 typedef struct {
   int8_t   type;
+  int32_t  index;
   char     db[TSDB_DB_NAME_LEN + 1];
   void *   pIter;
   int16_t  numOfColumns;
@@ -228,7 +229,6 @@ typedef struct {
   int32_t  numOfReads;
   int16_t  offset[TSDB_MAX_COLUMNS];
   int16_t  bytes[TSDB_MAX_COLUMNS];
-  void *   signature;
   uint16_t payloadLen;
   char     payload[];
 } SShowObj;

src/mnode/inc/mnodeVgroup.h

@@ -27,7 +27,8 @@ void    mnodeCleanupVgroups();
 SVgObj *mnodeGetVgroup(int32_t vgId);
 void    mnodeIncVgroupRef(SVgObj *pVgroup);
 void    mnodeDecVgroupRef(SVgObj *pVgroup);
-void    mnodeDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg);
+void    mnodeDropAllDbVgroups(SDbObj *pDropDb);
+void    mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb);
 void    mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 void    mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
 

src/mnode/src/mnodeDb.c

@@ -81,10 +81,10 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
   SDbObj *pDb = pOper->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
 
-  mnodeDropDbFromAcct(pAcct, pDb);
   mnodeDropAllChildTables(pDb);
   mnodeDropAllSuperTables(pDb);
-  mnodeDropAllDbVgroups(pDb, false);
+  mnodeDropAllDbVgroups(pDb);
+  mnodeDropDbFromAcct(pAcct, pDb);
   mnodeDecAcctRef(pAcct);
   
   return TSDB_CODE_SUCCESS;
@@ -998,19 +998,7 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) {
     return code;
   }
 
-#if 1
-  mnodeDropAllDbVgroups(pMsg->pDb, true);
-#else
-  SVgObj *pVgroup = pMsg->pDb->pHead;
-  if (pVgroup != NULL) {
-    mPrint("vgId:%d, will be dropped", pVgroup->vgId);
-    SMnodeMsg *newMsg = mnodeCloneMsg(pMsg);
-    newMsg->ahandle = pVgroup;
-    newMsg->expected = pVgroup->numOfVnodes;
-    mnodeDropVgroup(pVgroup, newMsg);
-    return;
-  }
-#endif  
+  mnodeSendDropAllDbVgroupsMsg(pMsg->pDb);
 
   mTrace("db:%s, all vgroups is dropped", pMsg->pDb->name);
   return mnodeDropDb(pMsg);

src/mnode/src/mnodeShow.c

@@ -47,13 +47,14 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *mnodeMsg);
 static int32_t mnodeProcessUseMsg(SMnodeMsg *mnodeMsg);
 
 static void  mnodeFreeShowObj(void *data);
-static bool  mnodeCheckShowObj(SShowObj *pShow);
+static bool  mnodeAccquireShowObj(SShowObj *pShow);
 static bool  mnodeCheckShowFinished(SShowObj *pShow);
-static void *mnodeSaveShowObj(SShowObj *pShow, int32_t size);
-static void  mnodeCleanupShowObj(void *pShow, bool forceRemove);
+static void *mnodePutShowObj(SShowObj *pShow, int32_t size);
+static void  mnodeReleaseShowObj(void *pShow, bool forceRemove);
 
 extern void *tsMnodeTmr;
-static void *tsQhandleCache = NULL;
+static void *tsMnodeShowCache = NULL;
+static int32_t tsShowObjIndex = 0;
 static SShowMetaFp     tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
 
@@ -64,14 +65,15 @@ int32_t mnodeInitShow() {
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mnodeProcessConnectMsg);
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mnodeProcessUseMsg);
   
-  tsQhandleCache = taosCacheInitWithCb(tsMnodeTmr, 10, mnodeFreeShowObj);
+  tsMnodeShowCache = taosCacheInitWithCb(tsMnodeTmr, 10, mnodeFreeShowObj);
   return 0;
 }
 
 void mnodeCleanUpShow() {
-  if (tsQhandleCache != NULL) {
-    taosCacheCleanup(tsQhandleCache);
-    tsQhandleCache = NULL;
+  if (tsMnodeShowCache != NULL) {
+    mPrint("show cache is cleanup");
+    taosCacheCleanup(tsMnodeShowCache);
+    tsMnodeShowCache = NULL;
   }
 }
 
@@ -118,13 +120,12 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
 
   int32_t showObjSize = sizeof(SShowObj) + htons(pShowMsg->payloadLen);
   SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
-  pShow->signature  = pShow;
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
   strcpy(pShow->db, pShowMsg->db);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
-  pShow = mnodeSaveShowObj(pShow, showObjSize);
+  pShow = mnodePutShowObj(pShow, showObjSize);
   if (pShow == NULL) {    
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
@@ -132,21 +133,22 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
   int32_t size = sizeof(SCMShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE;
   SCMShowRsp *pShowRsp = rpcMallocCont(size);
   if (pShowRsp == NULL) {
-    mnodeFreeShowObj(pShow);
+    mnodeReleaseShowObj(pShow, true);
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
   pShowRsp->qhandle = htobe64((uint64_t) pShow);
 
-  mTrace("show:%p, type:%s, start to get meta", pShow, mnodeGetShowType(pShowMsg->type));
+  mTrace("%p, show type:%s, start to get meta", pShow, mnodeGetShowType(pShowMsg->type));
   int32_t code = (*tsMnodeShowMetaFp[pShowMsg->type])(&pShowRsp->tableMeta, pShow, pMsg->rpcMsg.handle);
   if (code == 0) {
     pMsg->rpcRsp.rsp = pShowRsp;
     pMsg->rpcRsp.len = sizeof(SCMShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
+    mnodeReleaseShowObj(pShow, false);
     return TSDB_CODE_SUCCESS;
   } else {
-    mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mnodeGetShowType(pShowMsg->type), tstrerror(code));
+    mError("%p, show type:%s, failed to get meta, reason:%s", pShow, mnodeGetShowType(pShowMsg->type), tstrerror(code));
     rpcFreeCont(pShowRsp);
-    mnodeCleanupShowObj(pShow, true);
+    mnodeReleaseShowObj(pShow, true);
     return code;
   }
 }
@@ -159,22 +161,20 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
   SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
-  mTrace("show:%p, type:%s, retrieve data", pShow, mnodeGetShowType(pShow->type));
+  mTrace("%p, show type:%s, retrieve data", pShow, mnodeGetShowType(pShow->type));
 
   /*
    * in case of server restart, apps may hold qhandle created by server before
    * restart, which is actually invalid, therefore, signature check is required.
    */
-  if (!mnodeCheckShowObj(pShow)) {
-    mError("retrieve:%p, qhandle:%p is invalid", pRetrieve, pShow);
+  if (!mnodeAccquireShowObj(pShow)) {
+    mError("%p, show is invalid", pShow);
     return TSDB_CODE_INVALID_QHANDLE;
   }
   
   if (mnodeCheckShowFinished(pShow)) {
-    mTrace("retrieve:%p, qhandle:%p already read finished, numOfReads:%d numOfRows:%d", pRetrieve, pShow, pShow->numOfReads, pShow->numOfRows);
+    mTrace("%p, show is already read finished, numOfReads:%d numOfRows:%d", pShow, pShow->numOfReads, pShow->numOfRows);
     pShow->numOfReads = pShow->numOfRows;
-    //mnodeCleanupShowObj(pShow, true);
-    //return TSDB_CODE_SUCCESS;
   }
 
   if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
@@ -200,7 +200,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
 
   if (rowsRead < 0) {
     rpcFreeCont(pRsp);
-    mnodeCleanupShowObj(pShow, false);
+    mnodeReleaseShowObj(pShow, false);
     assert(false);
     return TSDB_CODE_ACTION_IN_PROGRESS;
   }
@@ -211,10 +211,11 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
   pMsg->rpcRsp.rsp = pRsp;
   pMsg->rpcRsp.len = size;
 
-  if (rowsToRead == 0) {
-    mnodeCleanupShowObj(pShow, true);
+  if (rowsToRead == 0 || rowsRead == rowsToRead) {
+    pRsp->completed = 1;
+    mnodeReleaseShowObj(pShow, true);
   } else {
-    mnodeCleanupShowObj(pShow, false);
+    mnodeReleaseShowObj(pShow, false);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -318,24 +319,29 @@ static bool mnodeCheckShowFinished(SShowObj *pShow) {
   return false;
 }
 
-static bool mnodeCheckShowObj(SShowObj *pShow) {
-  SShowObj *pSaved = taosCacheAcquireByData(tsQhandleCache, pShow);
+static bool mnodeAccquireShowObj(SShowObj *pShow) {
+  char key[10];
+  sprintf(key, "%d", pShow->index);
+
+  SShowObj *pSaved = taosCacheAcquireByName(tsMnodeShowCache, key);
   if (pSaved == pShow) {
+    mTrace("%p, show is accquired from cache", pShow);
     return true;
   } else {
-    mTrace("show:%p, is already released", pShow);
     return false;
   }
 }
 
-static void *mnodeSaveShowObj(SShowObj *pShow, int32_t size) {
-  if (tsQhandleCache != NULL) {
-    char key[24];
-    sprintf(key, "show:%p", pShow);
-    SShowObj *newQhandle = taosCachePut(tsQhandleCache, key, pShow, size, 60);
+static void *mnodePutShowObj(SShowObj *pShow, int32_t size) {
+  if (tsMnodeShowCache != NULL) {
+    char key[10];
+    pShow->index = atomic_add_fetch_32(&tsShowObjIndex, 1);
+    sprintf(key, "%d", pShow->index);
+
+    SShowObj *newQhandle = taosCachePut(tsMnodeShowCache, key, pShow, size, 60);
     free(pShow);
 
-    mTrace("show:%p, is saved", newQhandle);
+    mTrace("%p, show is put into cache", newQhandle);
     return newQhandle;
   }
 
@@ -345,10 +351,10 @@ static void *mnodeSaveShowObj(SShowObj *pShow, int32_t size) {
 static void mnodeFreeShowObj(void *data) {
   SShowObj *pShow = data;
   sdbFreeIter(pShow->pIter);
-  mTrace("show:%p, is destroyed", pShow);
+  mTrace("%p, show is destroyed", pShow);
 }
 
-static void mnodeCleanupShowObj(void *pShow, bool forceRemove) {
-  mTrace("show:%p, is released, force:%s", pShow, forceRemove ? "true" : "false");
-  taosCacheRelease(tsQhandleCache, &pShow, forceRemove);
+static void mnodeReleaseShowObj(void *pShow, bool forceRemove) {
+  mTrace("%p, show is released, force:%s", pShow, forceRemove ? "true" : "false");
+  taosCacheRelease(tsMnodeShowCache, &pShow, forceRemove);
 }

src/mnode/src/mnodeTable.c

@@ -148,37 +148,30 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
     return TSDB_CODE_INVALID_VGROUP_ID;
   }
 
-  SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId);
-  if (pVgroup == NULL) {
-    return TSDB_CODE_INVALID_VGROUP_ID;
-  }
-  mnodeDecVgroupRef(pVgroup);
+  SVgObj *pVgroup = NULL;
+  SDbObj *pDb = NULL;
+  SAcctObj *pAcct = NULL;
   
-  SDbObj *pDb = mnodeGetDb(pVgroup->dbName);
-  if (pDb == NULL) {
-    mError("ctable:%s, vgId:%d not in DB:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName);
-    return TSDB_CODE_INVALID_DB;
-  }
-  mnodeDecDbRef(pDb);
-
-  SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
-  if (pAcct == NULL) {
-    mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct);
-    return TSDB_CODE_INVALID_ACCT;
-  }
-  mnodeDecAcctRef(pAcct);
+  pVgroup = mnodeGetVgroup(pTable->vgId);
+  if (pVgroup != NULL) pDb = mnodeGetDb(pVgroup->dbName);
+  if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
-    pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
+    if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
     mnodeRemoveTableFromStable(pTable->superTable, pTable);
     mnodeDecTableRef(pTable->superTable);
   } else {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
-    pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
+    if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
   }
-  mnodeRemoveTableFromDb(pDb);
-  mnodeRemoveTableFromVgroup(pVgroup, pTable);
+  
+  if (pDb != NULL) mnodeRemoveTableFromDb(pDb);
+  if (pVgroup != NULL) mnodeRemoveTableFromVgroup(pVgroup, pTable);
+
+  mnodeDecVgroupRef(pVgroup);
+  mnodeDecDbRef(pDb);
+  mnodeDecAcctRef(pAcct);
  
   return TSDB_CODE_SUCCESS;
 }
@@ -693,10 +686,10 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
   }
 
   if (pCreate->numOfTags != 0) {
-    mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
+    mTrace("table:%s, create stable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
     return mnodeProcessCreateSuperTableMsg(pMsg);
   } else {
-    mTrace("table:%s, create msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
+    mTrace("table:%s, create ctable msg is received from thandle:%p", pCreate->tableId, pMsg->rpcMsg.handle);
     return mnodeProcessCreateChildTableMsg(pMsg);
   }
 }
@@ -1276,7 +1269,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
   for (int32_t i = 0; i < numOfTable; ++i) {
     char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
     SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
-    if (pTable->vgHash != NULL) {
+    if (pTable != NULL && pTable->vgHash != NULL) {
       contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo));
     } 
     mnodeDecTableRef(pTable);
@@ -1287,12 +1280,23 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
     return TSDB_CODE_SERV_OUT_OF_MEMORY;
   }
 
-  pRsp->numOfTables = htonl(numOfTable);
-  char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg);
+  pRsp->numOfTables = 0;
+  char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg);
 
   for (int32_t i = 0; i < numOfTable; ++i) {
     char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
     SSuperTableObj *pTable = mnodeGetSuperTable(stableName);
+    if (pTable == NULL) {
+      mError("stable:%s, not exist while get stable vgroup info", stableName);
+      mnodeDecTableRef(pTable);
+      continue;
+    }
+    if (pTable->vgHash == NULL) {
+      mError("stable:%s, not vgroup exist while get stable vgroup info", stableName);
+      mnodeDecTableRef(pTable);
+      continue;
+    }
+
     SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg;
 
     SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
@@ -1318,17 +1322,25 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
     }
 
     taosHashDestroyIter(pIter);
+    mnodeDecTableRef(pTable);
 
     pVgroupInfo->numOfVgroups = htonl(vgSize);
 
     // one table is done, try the next table
     msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo);
+    pRsp->numOfTables++;
   }
 
+  if (pRsp->numOfTables != numOfTable) {
+    rpcFreeCont(pRsp);
+    return TSDB_CODE_INVALID_TABLE_ID;
+  } else {
+    pRsp->numOfTables = htonl(pRsp->numOfTables);
     pMsg->rpcRsp.rsp = pRsp;
     pMsg->rpcRsp.len = msg - (char *)pRsp;
 
     return TSDB_CODE_SUCCESS;
+  }
 }
 
 static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) {
@@ -1429,7 +1441,7 @@ static SChildTableObj* mnodeDoCreateChildTable(SCMCreateTableMsg *pCreate, SVgOb
     SSuperTableObj *pSuperTable = mnodeGetSuperTable(pTagData->name);
     if (pSuperTable == NULL) {
       mError("table:%s, corresponding super table:%s does not exist", pCreate->tableId, pTagData->name);
-      free(pTable);
+      mnodeDestroyChildTable(pTable);
       terrno = TSDB_CODE_INVALID_TABLE_ID;
       return NULL;
     }
@@ -1738,7 +1750,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
 
 static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
-  STagData* pTag = (STagData*)pInfo->tags;
+  STagData *pTag = (STagData *)pInfo->tags;
 
   int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + ntohl(pTag->dataLen);
   SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen);
@@ -1754,14 +1766,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
   pCreateMsg->contLen = htonl(contLen);
 
   memcpy(pCreateMsg->schema, pInfo->tags, contLen - sizeof(SCMCreateTableMsg));
+  mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, ((STagData *)(pCreateMsg->schema))->name);
 
   rpcFreeCont(pMsg->rpcMsg.pCont);
   pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE;
   pMsg->rpcMsg.pCont = pCreateMsg;
   pMsg->rpcMsg.contLen = contLen;
   
-  mTrace("table:%s, start to create on demand, stable:%s", pInfo->tableId, pInfo->tags);
-
   return TSDB_CODE_ACTION_NEED_REPROCESSED;
 }
 

src/mnode/src/mnodeVgroup.c

@@ -784,7 +784,7 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
   mPrint("db:%s, all vgroups is updated in sdb", pAlterDb->name);
 }
 
-void mnodeDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
+void mnodeDropAllDbVgroups(SDbObj *pDropDb) {
   void *  pIter = NULL;
   int32_t numOfVgroups = 0;
   SVgObj *pVgroup = NULL;
@@ -802,10 +802,28 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
       };
       sdbDeleteRow(&oper);
       numOfVgroups++;
+    }
 
-      if (sendMsg) {
-        mnodeSendDropVgroupMsg(pVgroup, NULL);
+    mnodeDecVgroupRef(pVgroup);
   }
+
+  sdbFreeIter(pIter);
+
+  mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
+}
+
+void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
+  void *  pIter = NULL;
+  int32_t numOfVgroups = 0;
+  SVgObj *pVgroup = NULL;
+
+  mPrint("db:%s, all vgroups will be dropped in dnode", pDropDb->name);
+  while (1) {
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->pDb == pDropDb) {
+      mnodeSendDropVgroupMsg(pVgroup, NULL);
     }
 
     mnodeDecVgroupRef(pVgroup);
@@ -813,5 +831,5 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb, bool sendMsg) {
 
   sdbFreeIter(pIter);
 
-  mPrint("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
+  mPrint("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
 }
\ No newline at end of file