Merge pull request #14113 from taosdata/fix/dnode

enh: drop qnode and snode on dnode

source/dnode/mnode/impl/inc/mndQnode.h

@@ -24,12 +24,12 @@ extern "C" {
 
 #define QNODE_LOAD_VALUE(pQnode) (pQnode ? (pQnode->load.numOfQueryInQueue + pQnode->load.numOfFetchInQueue) : 0)
 
-int32_t mndInitQnode(SMnode *pMnode);
-void    mndCleanupQnode(SMnode *pMnode);
-
+int32_t    mndInitQnode(SMnode *pMnode);
+void       mndCleanupQnode(SMnode *pMnode);
 SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId);
-void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj);
-int32_t mndCreateQnodeList(SMnode       *pMnode, SArray** pList, int32_t limit);
+void       mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj);
+int32_t    mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit);
+int32_t    mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndSnode.h

@@ -22,9 +22,12 @@
 extern "C" {
 #endif
 
-int32_t mndInitSnode(SMnode *pMnode);
-void    mndCleanupSnode(SMnode *pMnode);
-SEpSet  mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode);
+int32_t    mndInitSnode(SMnode *pMnode);
+void       mndCleanupSnode(SMnode *pMnode);
+SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t qnodeId);
+void       mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj);
+SEpSet     mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode);
+int32_t    mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDnode.c

@@ -19,6 +19,7 @@
 #include "mndMnode.h"
 #include "mndQnode.h"
 #include "mndShow.h"
+#include "mndSnode.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
@@ -370,7 +371,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
   SMnodeObj *pObj = mndAcquireMnode(pMnode, pDnode->id);
   if (pObj != NULL) {
     if (pObj->state != statusReq.mload.syncState) {
-      mInfo("dnode:%d, mnode syncstate from %s to %s", pObj->id, syncStr(pObj->state), syncStr(statusReq.mload.syncState));
+      mInfo("dnode:%d, mnode syncstate from %s to %s", pObj->id, syncStr(pObj->state),
+            syncStr(statusReq.mload.syncState));
       pObj->state = statusReq.mload.syncState;
       pObj->stateStartTime = taosGetTimestampMs();
     }
@@ -538,7 +540,8 @@ _OVER:
   return code;
 }
 
-static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, int32_t numOfVnodes) {
+static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, SQnodeObj *pQObj,
+                            SSnodeObj *pSObj, int32_t numOfVnodes) {
   int32_t  code = -1;
   SSdbRaw *pRaw = NULL;
   STrans  *pTrans = NULL;
@@ -562,6 +565,17 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
     mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
     if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER;
   }
+
+  if (pQObj != NULL) {
+    mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id);
+    if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj) != 0) goto _OVER;
+  }
+
+  if (pSObj != NULL) {
+    mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id);
+    if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj) != 0) goto _OVER;
+  }
+
   if (numOfVnodes > 0) {
     mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id);
     if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
@@ -581,6 +595,8 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
   int32_t        code = -1;
   SDnodeObj     *pDnode = NULL;
   SMnodeObj     *pMObj = NULL;
+  SQnodeObj     *pQObj = NULL;
+  SSnodeObj     *pSObj = NULL;
   SMDropMnodeReq dropReq = {0};
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
@@ -601,6 +617,8 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  pQObj = mndAcquireQnode(pMnode, dropReq.dnodeId);
+  pSObj = mndAcquireSnode(pMnode, dropReq.dnodeId);
   pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
   if (pMObj != NULL) {
     if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
@@ -627,7 +645,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  code = mndDropDnode(pMnode, pReq, pDnode, pMObj, numOfVnodes);
+  code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
 _OVER:
@@ -637,6 +655,8 @@ _OVER:
 
   mndReleaseDnode(pMnode, pDnode);
   mndReleaseMnode(pMnode, pMObj);
+  mndReleaseQnode(pMnode, pQObj);
+  mndReleaseSnode(pMnode, pSObj);
   return code;
 }
 

source/dnode/mnode/impl/src/mndQnode.c

@@ -354,6 +354,14 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn
   return 0;
 }
 
+int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj) {
+  if (pObj == NULL) return 0;
+  if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1;
+  if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1;
+  if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  return 0;
+}
+
 static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
   int32_t code = -1;
 
@@ -361,9 +369,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
-  if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) goto _OVER;
-  if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) goto _OVER;
-  if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER;
+  if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndSnode.c

@@ -65,7 +65,7 @@ SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode) {
   return epSet;
 }
 
-static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) {
+SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) {
   SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId);
   if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
     terrno = TSDB_CODE_MND_SNODE_NOT_EXIST;
@@ -73,7 +73,7 @@ static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) {
   return pObj;
 }
 
-static void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) {
+void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) {
   SSdb *pSdb = pMnode->pSdb;
   sdbRelease(pSdb, pObj);
 }
@@ -361,6 +361,14 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn
   return 0;
 }
 
+int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj) {
+  if (pObj == NULL) return 0;
+  if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1;
+  if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1;
+  if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1;
+  return 0;
+}
+
 static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
   int32_t code = -1;
 
@@ -368,10 +376,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
   if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
-
-  if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) goto _OVER;
-  if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) goto _OVER;
-  if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER;
+  if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

tests/script/jenkins/basic.txt

@@ -23,7 +23,11 @@
 # ---- dnode
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
+./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
+#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
+#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
+#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim

tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != offline then
+  goto step1
+endi
+
+print =============== step2 create database
+sql create database d1 vgroups 4
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 4 then
+  return -1
+endi
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(3)[3] != 2 then
+  return -1
+endi
+if $data(4)[3] != 2 then
+  return -1
+endi
+if $data(5)[3] != 2 then
+  return -1
+endi
+
+print =============== step3: start dnode 3
+system sh/exec.sh -n dnode3 -s start
+$x = 0
+step4:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step4
+endi
+if $data(2)[4] != ready then
+  goto step4
+endi
+if $data(3)[4] != ready then
+  goto step4
+endi
+
+print =============== step3: drop dnode2
+sql drop dnode 2
+
+print show dnodes; 
+sql show dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 2 then 
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 4 then
+  return -1
+endi
+if $data(2)[3] != 3 then
+  return -1
+endi
+if $data(3)[3] != 3 then
+  return -1
+endi
+if $data(4)[3] != 3 then
+  return -1
+endi
+if $data(5)[3] != 3 then
+  return -1
+endi
+
+print =============== step4: select data
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+
+print =============== step1 create dnode2 dnode3 dnode4 dnode 5
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+if $data(5)[4] != offline then
+  goto step1
+endi
+
+print =============== step3 create database
+sql create database d1 vgroups 4 replica 3
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(2)[5] != 3 then
+  return -1
+endi
+if $data(2)[7] != 4 then
+  return -1
+endi
+
+print =============== step4: drop dnode 2
+system sh/exec.sh -n dnode5 -s start
+$x = 0
+step4:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step4
+endi
+if $data(2)[4] != ready then
+  goto step4
+endi
+if $data(3)[4] != ready then
+  goto step4
+endi
+if $data(4)[4] != ready then
+  goto step4
+endi
+if $data(5)[4] != ready then
+  goto step4
+endi
+
+print =============== step5: drop dnode2
+sql drop dnode 2
+
+print show dnodes; 
+sql show dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 4 then 
+  return -1
+endi
+
+print show d1.vgroups
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 1 then
+  return -1
+endi
+
+
+print =============== step6: select data
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+return
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_qnode_snode.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create qnode snode on dnode 2
+sql create qnode on dnode 2
+sql create snode on dnode 2
+
+sql show qnodes
+if $rows != 1 then
+  return -1
+endi
+
+sql show snodes
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step3: drop dnode 2
+sql drop dnode 2
+
+print show dnodes; 
+sql show dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 1 then 
+  return -1
+endi
+if $data00 != 1 then 
+  return -1
+endi
+
+sql show qnodes
+if $rows != 0 then
+  return -1
+endi
+
+sql show snodes
+if $rows != 0 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_vnode_replica1.sim

@@ -148,4 +148,4 @@ endi
 return
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+
+print =============== step1 create dnode2 dnode3 dnode4 dnode 5
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+
+$x = 0
+step1:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+if $data(5)[4] != offline then
+  goto step1
+endi
+
+print =============== step3 create database
+sql create database d1 vgroups 1 replica 3
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[3] != 2 then
+  return -1
+endi
+if $data(2)[5] != 3 then
+  return -1
+endi
+if $data(2)[7] != 4 then
+  return -1
+endi
+
+print =============== step4: drop dnode 2
+system sh/exec.sh -n dnode5 -s start
+$x = 0
+step4:
+	$ = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not online!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 3 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step4
+endi
+if $data(2)[4] != ready then
+  goto step4
+endi
+if $data(3)[4] != ready then
+  goto step4
+endi
+if $data(4)[4] != ready then
+  goto step4
+endi
+if $data(5)[4] != ready then
+  goto step4
+endi
+
+print =============== step5: drop dnode2
+sql drop dnode 2
+
+print show dnodes; 
+sql show dnodes;
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+if $rows != 4 then 
+  return -1
+endi
+
+print show d1.vgroups
+sql show d1.vgroups
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+if $rows != 1 then
+  return -1
+endi
+#if $data(2)[3] != 3 then
+#  return -1
+#endi
+
+print =============== step6: select data
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+return
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT