Merge pull request #14131 from taosdata/fix/dnode

feat: balance vgroup

source/dnode/mnode/impl/src/mndDnode.c

@@ -580,6 +580,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
     mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
     if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
   }
+
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1553,10 +1553,11 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
 static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
   void   *pIter = NULL;
   int32_t code = -1;
+  SSdb   *pSdb = pMnode->pSdb;
 
   while (1) {
     SVgObj *pVgroup = NULL;
-    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
 
     bool existInSrc = false;
@@ -1568,13 +1569,15 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
     }
 
     if (!existInSrc || existInDst) {
-      sdbRelease(pMnode->pSdb, pVgroup);
+      sdbRelease(pSdb, pVgroup);
+      continue;
     }
 
     SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
     code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
     mndReleaseDb(pMnode, pDb);
-    sdbRelease(pMnode->pSdb, pVgroup);
+    sdbRelease(pSdb, pVgroup);
+    sdbCancelFetch(pSdb, pIter);
     break;
   }
 
@@ -1593,15 +1596,25 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
 
   while (1) {
     taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
-    SDnodeObj *pSrc = taosArrayGet(pArray, 0);
-    SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
+    for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+      SDnodeObj *pDnode = taosArrayGet(pArray, i);
+      mDebug("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes,
+             pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes);
+    }
+
+    SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
+    SDnodeObj *pDst = taosArrayGet(pArray, 0);
 
     float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
     float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
-    if (srcScore + 0.0001 < dstScore) {
-      mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
+    mDebug("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore,
+           pDst->id, dstScore);
+
+    if (srcScore > dstScore - 0.000001) {
       code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
       if (code == 0) {
+        pSrc->numOfVnodes--;
+        pDst->numOfVnodes++;
         numOfVgroups++;
         continue;
       } else {
@@ -1635,7 +1648,13 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
   void   *pIter = NULL;
   int64_t curMs = taosGetTimestampMs();
 
-  mDebug("start to balance vgroup");
+  SBalanceVgroupReq req = {0};
+  if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _OVER;
+  }
+
+  mInfo("start to balance vgroup");
 
   if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER;
 

tests/script/jenkins/basic.txt

@@ -12,6 +12,8 @@
 ## ---- db
 ./test.sh -f tsim/db/create_all_options.sim
 ./test.sh -f tsim/db/alter_option.sim
+./test.sh -f tsim/db/alter_replica_13.sim
+#./test.sh -f tsim/db/alter_replica_31.sim
 ./test.sh -f tsim/db/basic1.sim
 ./test.sh -f tsim/db/basic2.sim
 ./test.sh -f tsim/db/basic3.sim
@@ -21,6 +23,7 @@
 ./test.sh -f tsim/db/taosdlog.sim
 
 # ---- dnode
+./test.sh -f tsim/dnode/balance_replica1.sim
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@@ -28,6 +31,8 @@
 #./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
 #./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim

tests/script/tsim/dnode/balance_replica1.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+
+print =============== step1 create dnode2
+# no enough vnodes
+sql balance vgroup
+
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ---> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ---> $data00 $data01 $data02 $data03 $data04 $data05
+print ---> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != offline then
+  goto step1
+endi
+
+print =============== step2 create database
+sql create database d1 vgroups 2
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql create table d1.c2 using st tags(1)
+sql create table d1.c3 using st tags(1)
+sql create table d1.c4 using st tags(1)
+sql create table d1.c5 using st tags(1)
+sql create table d1.c6 using st tags(1)
+sql show d1.tables
+if $rows != 6 then
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 2 then
+  return -1
+endi
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(3)[3] != 2 then
+  return -1
+endi
+
+print =============== step3: balance vgroup
+# has offline dnode
+sql_error balance vgroup
+
+system sh/exec.sh -n dnode3 -s start
+$x = 0
+step3:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ---> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ---> $data00 $data01 $data02 $data03 $data04 $data05
+print ---> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step3
+endi
+if $data(2)[4] != ready then
+  goto step3
+endi
+if $data(3)[4] != ready then
+  goto step3
+endi
+
+print =============== step4: balance
+sql balance vgroup
+
+print show d1.vgroups
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 2 then
+  return -1
+endi
+if $data(2)[3] != 3 then
+  return -1
+endi
+if $data(3)[3] != 2 then
+  return -1
+endi
+
+print =============== step7: select data
+sql show d1.tables
+print rows $rows
+if $rows != 6 then
+  return -1
+endi
+
+return
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim

@@ -28,7 +28,7 @@ step1:
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -72,6 +72,13 @@ if $data(2)[7] != 4 then
   return -1
 endi
 
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+
+
+return
+
+
 print =============== step4: drop dnode 2
 system sh/exec.sh -n dnode5 -s start
 $x = 0

tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim

@@ -176,6 +176,8 @@ if $rows != 1 then
   return -1
 endi
 
+return
+
 print =============== step33: move follower1
 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 
 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5