TD-2429

src/balance/src/bnMain.c

@@ -224,19 +224,34 @@ static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
     return false;
   }
 
+  int32_t rmVnodeVer = 0;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
+    if (pVnode == pRmVnode) {
+      rmVnodeVer = mnodeGetVgidVer(pVnode->vver);
+      mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d is watching", pVgroup->vgId, i,
+             pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], rmVnodeVer);
+    }
+  }
+
   bool isReady = false;
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pVnode = pVgroup->vnodeGid + i;
     if (pVnode == pRmVnode) continue;
+    int32_t vver = mnodeGetVgidVer(pVnode->vver);
 
-    mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId,
-           pVnode->pDnode->status, syncRole[pVnode->role]);
+    mTrace("vgId:%d, check vgroup status, vindex:%d dnode:%d status:%s role:%s vver:%d, rmvver:%d" , pVgroup->vgId, i,
+           pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
     if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
     if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
+    if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) continue;
 
-    if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) {
-      isReady = true;
+    if (rmVnodeVer == 0 || vver >= rmVnodeVer) {
+      mInfo("vgId:%d, is ready for vindex:%d in dnode:%d status:%s role:%s vver:%d larger than rmvver:%d", pVgroup->vgId, i,
+             pVnode->dnodeId, dnodeStatus[pVnode->pDnode->status], syncRole[pVnode->role], vver, rmVnodeVer);
     }
+
+    isReady = true;
   }
 
   return isReady;
@@ -256,7 +271,7 @@ static int32_t bnRemoveVnode(SVgObj *pVgroup) {
     mDebug("vgId:%d, is not ready", pVgroup->vgId);
     return -1;
   } else {
-    mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
+    mInfo("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
     bnDiscardVnode(pVgroup, pSelVnode);
     return TSDB_CODE_SUCCESS;
   }

src/inc/taosmsg.h

@@ -523,11 +523,12 @@ typedef struct {
   int64_t  totalStorage;
   int64_t  compStorage;
   int64_t  pointsWritten;
+  uint64_t vnodeVersion;
+  int32_t  vgCfgVersion;
   uint8_t  status;
   uint8_t  role;
   uint8_t  replica;
   uint8_t  reserved;
-  int32_t vgCfgVersion;
 } SVnodeLoad;
 
 typedef struct {

src/mnode/inc/mnodeDef.h

@@ -128,8 +128,8 @@ typedef struct {
 typedef struct {
   int32_t    dnodeId;
   int8_t     role;
-  int8_t     reserved[3];
-  SDnodeObj* pDnode;
+  int8_t     vver[3];  // To ensure compatibility, 3 bits are used to represent the remainder of 64 bit version
+  SDnodeObj *pDnode;
 } SVnodeGid;
 
 typedef struct SVgObj {

src/mnode/inc/mnodeVgroup.h

@@ -53,6 +53,9 @@ void    mnodeSendAlterVgroupMsg(SVgObj *pVgroup);
 SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup);
 SRpcEpSet mnodeGetEpSetFromIp(char *ep);
 
+int32_t mnodeGetVgidVer(int8_t *vver);
+void    mnodeSetVgidVer(int8_t *cver, uint64_t iver);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/src/mnodeDnode.c

@@ -571,6 +571,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
     pVload->vgId = htonl(pVload->vgId);
     pVload->dbCfgVersion = htonl(pVload->dbCfgVersion);
     pVload->vgCfgVersion = htonl(pVload->vgCfgVersion);
+    pVload->vnodeVersion = htobe64(pVload->vnodeVersion);
 
     SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId);
     if (pVgroup == NULL) {

src/mnode/src/mnodeVgroup.c

@@ -184,6 +184,7 @@ static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) {
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     pTmpVgroup->vnodeGid[i].pDnode = NULL;
     pTmpVgroup->vnodeGid[i].role = 0;
+    memset(pTmpVgroup->vnodeGid[i].vver, 0, sizeof(pTmpVgroup->vnodeGid[i].vver));
   }
 
   pRow->rowSize = tsVgUpdateSize;
@@ -317,9 +318,10 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
     if (pVgid->pDnode == pDnode) {
-      mTrace("dnode:%d, receive status from dnode, vgId:%d status:%s last:%s", pDnode->dnodeId, pVgroup->vgId,
-             syncRole[pVload->role], syncRole[pVgid->role]);
+      mTrace("vgId:%d, receive vnode status from dnode:%d, status:%s last:%s vver:%" PRIu64, pVgroup->vgId,
+             pDnode->dnodeId, syncRole[pVload->role], syncRole[pVgid->role], pVload->vnodeVersion);
       pVgid->role = pVload->role;
+      mnodeSetVgidVer(pVgid->vver, pVload->vnodeVersion);
       if (pVload->role == TAOS_SYNC_ROLE_MASTER) {
         pVgroup->inUse = i;
       }
@@ -1179,3 +1181,14 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
 
   mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
 }
+
+int32_t mnodeGetVgidVer(int8_t *cver) {
+  int32_t iver = ((int32_t)cver[0]) * 10000 + ((int32_t)cver[1]) * 100 + (int32_t)cver[2];
+  return iver;
+}
+
+void mnodeSetVgidVer(int8_t *cver, uint64_t iver) {
+  cver[0] = (int8_t)((int32_t)(iver % 1000000) / 10000);
+  cver[1] = (int8_t)((int32_t)(iver % 100000) / 100);
+  cver[2] = (int8_t)(iver % 100);
+}
\ No newline at end of file

src/vnode/src/vnodeMgmt.c

@@ -142,6 +142,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
   pLoad->totalStorage = htobe64(totalStorage);
   pLoad->compStorage = htobe64(compStorage);
   pLoad->pointsWritten = htobe64(pointsWritten);
+  pLoad->vnodeVersion = htobe64(pVnode->version);
   pLoad->status = pVnode->status;
   pLoad->role = pVnode->role;
   pLoad->replica = pVnode->syncCfg.replica;  

tests/script/unique/dnode/lossdata.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+print ========== step2
+
+sql create database d1 replica 2  
+sql create table d1.t1 (t timestamp, i int) 
+
+print ========== step2.1
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+
+if $data2_1 != 0 then
+	return -1
+endi
+if $data2_2 != 1 then
+	return -1
+endi
+if $data2_3 != 1 then
+	return -1
+endi
+
+print ========== step3
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+
+$x = 0
+show3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then 
+	  return -1
+	endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+
+if $data2_2 != 1 then
+	goto show3
+endi
+if $data2_3 != 1 then
+	goto show3
+endi
+if $data2_4 != 0 then
+	goto show3
+endi
+
+sql show d1.vgroups;
+print d1.vgroups $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+print ========== step4
+sql drop dnode $hostname3
+
+$i = 0
+$rowNum = 10000
+
+while $i < $rowNum
+  $ts = 1500000000000 + $i
+  sql insert into d1.t1 values( $ts , $i )
+  
+  $i = $i + 1
+endw 
+
+print insert $rowNum finished
+
+$x = 0
+show4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then 
+	  return -1
+	endi
+
+sql show dnodes
+print dnode1 openVnodes $data2_1
+print dnode2 openVnodes $data2_2
+print dnode3 openVnodes $data2_3
+print dnode4 openVnodes $data2_4
+print dnode5 openVnodes $data2_5
+
+if $data2_2 != 1 then
+	goto show4
+endi
+if $data2_3 != null then
+	goto show4
+endi
+if $data2_4 != 1 then
+	goto show4
+endi
+
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+
+print ========== step5
+sql select count(*) from d1.t1 
+print select count(*) from d1.t1 ==> $data00 
+if $data00 != $rowNum then
+  return -1
+endi
+
+#system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+#system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+#system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+#system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+#system sh/exec.sh -n dnode5 -s stop  -x SIGINT
\ No newline at end of file