Merge pull request #3466 from taosdata/feature/crash

add syncInit and syncCleanUp functions

src/dnode/src/dnodeModule.c

@@ -16,11 +16,13 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
+#include "taosmsg.h"
 #include "tglobal.h"
 #include "mnode.h"
 #include "http.h"
 #include "tmqtt.h"
 #include "monitor.h"
+#include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeModule.h"
 
@@ -129,17 +131,32 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) {
   for (int32_t module = TSDB_MOD_MNODE; module < TSDB_MOD_HTTP; ++module) {
     bool enableModule = moduleStatus & (1 << module);
     if (!tsModule[module].enable && enableModule) {
-      dInfo("module status:%u is received, start %s module", tsModuleStatus, tsModule[module].name);
+      dInfo("module status:%u is set, start %s module", moduleStatus, tsModule[module].name);
       tsModule[module].enable = true;
       dnodeSetModuleStatus(module);
       (*tsModule[module].startFp)();
     }
 
     if (tsModule[module].enable && !enableModule) {
-      dInfo("module status:%u is received, stop %s module", tsModuleStatus, tsModule[module].name);
+      dInfo("module status:%u is set, stop %s module", moduleStatus, tsModule[module].name);
       tsModule[module].enable = false;
       dnodeUnSetModuleStatus(module);
       (*tsModule[module].stopFp)();
     }
   }
 }
+
+void dnodeCheckModules() {
+  if (tsModuleStatus & TSDB_MOD_MNODE) return;
+
+  SDMMnodeInfos *mnodes = dnodeGetMnodeInfos();
+  for (int32_t i = 0; i < mnodes->nodeNum; ++i) {
+    SDMMnodeInfo *node = &mnodes->nodeInfos[i];
+    if (node->nodeId == dnodeGetDnodeId()) {
+      uint32_t moduleStatus = tsModuleStatus | (1 << TSDB_MOD_MNODE);;
+      dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus);
+      dnodeProcessModuleStatus(moduleStatus);
+      break;
+    }
+  }
+}

src/inc/dnode.h

@@ -43,6 +43,7 @@ void    dnodeGetMnodeEpSetForPeer(void *epSet);
 void    dnodeGetMnodeEpSetForShell(void *epSet);
 void *  dnodeGetMnodeInfos();
 int32_t dnodeGetDnodeId();
+void    dnodeCheckModules();
 
 void  dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg));
 void  dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg);

src/inc/tsync.h

@@ -103,6 +103,9 @@ typedef struct {
 
 typedef void* tsync_h;
 
+int32_t syncInit();
+void    syncCleanUp();
+
 tsync_h syncStart(const SSyncInfo *);
 void    syncStop(tsync_h shandle);
 int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);

src/mnode/src/mnodeSdb.c

@@ -22,6 +22,7 @@
 #include "tqueue.h"
 #include "twal.h"
 #include "tsync.h"
+#include "ttimer.h"
 #include "tglobal.h"
 #include "dnode.h"
 #include "mnode.h"
@@ -88,6 +89,8 @@ typedef struct {
   SSdbWriteWorker *writeWorker;
 } SSdbWriteWorkerPool;
 
+extern void *     tsMnodeTmr;
+static void *     tsUpdateSyncTmr;
 static SSdbObject tsSdbObj = {0};
 static taos_qset  tsSdbWriteQset;
 static taos_qall  tsSdbWriteQall;
@@ -290,11 +293,16 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   taosFreeQitem(pOper);
 }
 
+static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(); }
+
 void sdbUpdateSync() {
   if (!mnodeIsRunning()) {
     mDebug("mnode not start yet, update sync info later");
+    dnodeCheckModules();
+    taosTmrReset(sdbUpdateSyncTmrFp, 1000, NULL, tsMnodeTmr, &tsUpdateSyncTmr);
     return;
   }
+  mDebug("update sync info in sdb");
 
   SSyncCfg syncCfg = {0};
   int32_t  index = 0;
@@ -387,8 +395,6 @@ int32_t sdbInit() {
     tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
   }
 
-  sdbUpdateSync();
-
   tsSdbObj.status = SDB_STATUS_SERVING;
   return TSDB_CODE_SUCCESS;
 }

src/vnode/src/vnodeMain.c

@@ -57,6 +57,9 @@ void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
 #endif
 
 int32_t vnodeInitResources() {
+  int code = syncInit();
+  if (code != 0) return code;
+
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
@@ -70,11 +73,12 @@ int32_t vnodeInitResources() {
 }
 
 void vnodeCleanupResources() {
-
   if (tsDnodeVnodesHash != NULL) {
     taosHashCleanup(tsDnodeVnodesHash);
     tsDnodeVnodesHash = NULL;
   }
+
+  syncCleanUp();
 }
 
 int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {

tests/script/unique/cluster/balance3.sim

@@ -105,6 +105,15 @@ if $dnode4Vnodes != null then
   goto show1
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step2
 print ========= start dnode4
 sql create dnode $hostname4
@@ -132,6 +141,15 @@ if $dnode4Vnodes != 2 then
   goto show2
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step3
 print ========= drop dnode2
 sql drop dnode $hostname2
@@ -167,6 +185,15 @@ if $dnode4Vnodes != 3 then
   goto show3
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 
 print ============================== step4
@@ -195,6 +222,15 @@ if $dnode5Vnodes != 2 then
   goto show4
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step5
 print ========= drop dnode3
 sql drop dnode $hostname3
@@ -232,6 +268,15 @@ endi
 
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step6
 sql create dnode $hostname6
 system sh/exec.sh -n dnode6 -s start
@@ -258,6 +303,15 @@ if $dnode6Vnodes != 2 then
   goto show6
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step7
 print ========= drop dnode4
 sql drop dnode $hostname4
@@ -294,6 +348,14 @@ if $dnode4Vnodes != null then
 endi
 
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
 
 print ============================== step8
 sql create dnode $hostname7
@@ -321,6 +383,15 @@ if $dnode7Vnodes != 2 then
   goto show8
 endi
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step9
 print ========= drop dnode1
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
@@ -335,15 +406,20 @@ sql show mnodes
 $dnode1Role = $data2_1
 $dnode4Role = $data2_4
 $dnode5Role = $data2_5
-print dnode1 ==> $dnode1Role
-print dnode4 ==> $dnode4Role
-print dnode5 ==> $dnode5Role
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
 
 if $dnode1Role != offline then
   return -1
 endi
 
 print ============================== step9.1
+sleep 2000
 system sh/exec.sh -n dnode1 -s start
 
 $x = 0
@@ -353,6 +429,19 @@ show9:
 	if $x == 20 then
 	  return -1
 	endi
+
+sql show mnodes
+$dnode1Role = $data2_1
+$dnode4Role = $data2_4
+$dnode5Role = $data2_5
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 sql show dnodes -x show9
 $dnode5Vnodes = $data2_5
 print dnode5 $dnode5Vnodes
@@ -374,6 +463,15 @@ endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 sleep 5000
 
+sql show mnodes
+print dnode1 ==> $data2_1
+print dnode2 ==> $data2_2
+print dnode3 ==> $data2_3
+print dnode4 ==> $data2_4
+print dnode5 ==> $data2_5
+print dnode6 ==> $data2_6
+print dnode7 ==> $data2_7
+
 print ============================== step11
 print ========= add db4
 

tests/tsim/src/simExe.c

@@ -667,7 +667,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
 
   TAOS_RES* pSql = NULL;
   
-  for (int attempt = 0; attempt < 3; ++attempt) {
+  for (int attempt = 0; attempt < 10; ++attempt) {
     simLogSql(rest, false);
     pSql = taos_query(script->taos, rest);
     ret = taos_errno(pSql);