diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 8410ec3757a5b9de5ecf16b0dacaf1ebf191cc65..6b2ee03524e17030dad72da01aa41d66077c396d 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1193,37 +1193,37 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId) { newDnodeId[++newIndex] = req.dnodeId1; - mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex); } if (req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId) { newDnodeId[++newIndex] = req.dnodeId2; - mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex); } if (req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) { newDnodeId[++newIndex] = req.dnodeId3; - mInfo("vgId:2, dnode:%d will be added", newDnodeId[newIndex]); + mInfo("vgId:%d, dnode:%d will be added, index:%d", pVgroup->vgId, newDnodeId[newIndex], newIndex); } if (req.dnodeId1 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[0].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[0].dnodeId) { oldDnodeId[++oldIndex] = pVgroup->vnodeGid[0].dnodeId; - mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex); } if (req.dnodeId1 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[1].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[1].dnodeId) { oldDnodeId[++oldIndex] = pVgroup->vnodeGid[1].dnodeId; - mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex); } if (req.dnodeId1 != pVgroup->vnodeGid[2].dnodeId && req.dnodeId2 != pVgroup->vnodeGid[2].dnodeId && req.dnodeId3 != pVgroup->vnodeGid[2].dnodeId) { oldDnodeId[++oldIndex] = pVgroup->vnodeGid[2].dnodeId; - mInfo("vgId:2, dnode:%d will be removed", oldDnodeId[oldIndex]); + mInfo("vgId:%d, dnode:%d will be removed, index:%d", pVgroup->vgId, oldDnodeId[oldIndex], oldIndex); } if (newDnodeId[0] != 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b4021ec73d4679d266ab52b3ca53c46d17160d4a..087bdb2b49c572f3090883eb0aa929e274cf5750 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -50,14 +50,14 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) { } } -static int32_t vnodeProcessSyncReconfigReq(SVnode *pVnode, SRpcMsg *pMsg) { +static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { SAlterVnodeReq req = {0}; if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) { terrno = TSDB_CODE_INVALID_MSG; return TSDB_CODE_INVALID_MSG; } - vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica); + vInfo("vgId:%d, start to alter vnode replica to %d, handle:%p", TD_VID(pVnode), req.replica, pMsg->info.handle); SSyncCfg cfg = {.replicaNum = req.replica, .myIndex = req.selfIndex}; for (int32_t r = 0; r < req.replica; ++r) { SNodeInfo *pNode = &cfg.nodeInfo[r]; @@ -80,7 +80,7 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { vTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle); if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) { - code = vnodeProcessSyncReconfigReq(pVnode, pMsg); + code = vnodeProcessAlterReplicaReq(pVnode, pMsg); } else { code = vnodePreprocessReq(pVnode, pMsg); if (code != 0) { @@ -92,11 +92,6 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { if (code == 0) { vnodeAccumBlockMsg(pVnode, pMsg->msgType); - if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) { - // todo refactor - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); - } } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { SEpSet newEpSet = {0}; syncGetEpSet(pVnode->sync, &newEpSet); @@ -136,7 +131,8 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { for (int32_t i = 0; i < numOfMsgs; ++i) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - vTrace("vgId:%d, msg:%p get from vnode-apply queue, handle:%p", vgId, pMsg, pMsg->info.handle); + vTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p", vgId, pMsg, TMSG_INFO(pMsg->msgType), + pMsg->info.handle); SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { @@ -182,19 +178,16 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode)); -#if 0 -// send response SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); -#endif - // todo rpc response here - // build rpc msg - // put into apply queue + vInfo("vgId:%d, alter vnode replica is confirmed, type:%s contLen:%d seq:%" PRIu64 " handle:%p", TD_VID(pVnode), + TMSG_INFO(pMsg->msgType), pMsg->contLen, cbMeta.seqNum, rpcMsg.info.handle); + if (rpcMsg.info.handle != NULL) { + tmsgSendRsp(&rpcMsg); + } + vnodePostBlockMsg(pVnode, TDMT_VND_ALTER_REPLICA); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4d410644bb85da1e1e42ab3e982d8d8eb62d6e7e..bd4d865ed2a756b3f84a8d2f5832284d913643b7 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -398,6 +398,7 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) *pInfo = stub.rpcMsg.info; } + sTrace("vgId:%d, get seq:%" PRIu64 " rpc handle:%p", pSyncNode->vgId, index, pInfo->handle); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -470,13 +471,14 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return TAOS_SYNC_PROPOSE_OTHER_ERROR; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; stub.createTime = taosGetTimestampMs(); stub.rpcMsg = *pMsg; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); + sDebug("vgId:%d, sync event propose, type:%s seq:%" PRIu64 " handle:%p", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), + seqNum, pMsg->info.handle); SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak, pSyncNode->vgId); SRpcMsg rpcMsg; @@ -489,7 +491,8 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { } syncClientRequestDestroy(pSyncMsg); } else { - sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); + sDebug("vgId:%d, failed to propose since not leader, type:%s handle:%p %s", pSyncNode->vgId, + TMSG_INFO(pMsg->msgType), pMsg->info.handle, syncUtilState2String(pSyncNode->state)); ret = TAOS_SYNC_PROPOSE_NOT_LEADER; } diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim new file mode 100644 index 0000000000000000000000000000000000000000..a73c140a0c5cd729366e88d42632e66e49a780cc --- /dev/null +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -0,0 +1,255 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +#system sh/exec.sh -n dnode5 -s start +sql connect +sql create user u1 pass 'taosdata' + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi +#if $data(5)[4] != ready then +# goto step1 +#endi + +print =============== step2: create db +sql create database d1 vgroups 1 replica 3 + +# dnode not exist +sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 +# vgroup not exist +sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 +# un changed +sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 +# no enought vnodes +sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 +# offline vnodes +sql_error redistribute vgroup 2 dnode 5 dnode 3 dnode 4 +# Invalid replica +sql_error redistribute vgroup 2 dnode 5 +sql_error redistribute vgroup 2 dnode 5 dnode 3 +sql_error redistribute vgroup 2 dnode 2 dnode 3 +sql_error redistribute vgroup 2 dnode 2 dnode 2 +sql_error redistribute vgroup 3 dnode 2 dnode 2 +sql_error redistribute vgroup 2 dnode 2 dnode 2 dnode 3 + +system sh/exec.sh -n dnode5 -s start +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 +print ===> $data40 $data41 $data42 $data43 $data44 $data45 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi +if $data(3)[4] != ready then + goto step2 +endi +if $data(4)[4] != ready then + goto step2 +endi +if $data(5)[4] != ready then + goto step2 +endi + +print =============== step31: move follower +$leaderExist = 0 +$leaderVnode = 0 +$follower1 = 0 +$follower2 = 0 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> db not ready! + return -1 + endi +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +if $rows != 1 then + return -1 +endi +if $data(2)[4] == leader then + $leaderExist = 1 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 +endi +if $data(2)[6] == leader then + $leaderExist = 1 + $leaderVnode = 3 + $follower1 = 2 + $follower2 = 4 +endi +if $data(2)[8] == leader then + $leaderExist = 1 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 +endi +if $leaderExist != 1 then + goto step3 +endi + +print leader $leaderVnode +print follower1 $follower1 +print follower2 $follower2 + +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step32: move follower2 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step33: move follower1 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step34: move follower2 +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step35: move follower1 +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step36: move follower2 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step37: move follower1 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step38: move follower2 +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step39: move follower1 +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index c5cc765b880ef66e3d8b5b1c06b5f0bc7aa3c761..6ce6fb4187ddd34e750daacd034ed1b00a602fc5 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -55,7 +55,9 @@ endi print =============== step2: create db sql create database d1 vgroups 1 replica 3 -# Invalid vgroup +# dnode not exist +sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4 +# vgroup not exist sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 # un changed sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4 @@ -158,42 +160,93 @@ if $rows != 1 then return -1 endi -print =============== step32: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +print =============== step32: move leader +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + sql show d1.tables if $rows != 1 then return -1 endi -print =============== step33: move follower1 +print =============== step33: move follower2 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step34: move follower1 +print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step35: move 5 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + sql show d1.tables if $rows != 1 then return -1 endi -print =============== step34: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +print =============== step36: move follower1 +print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + sql show d1.tables if $rows != 1 then return -1 endi -print =============== step35: move follower1 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 +print =============== step37: move follower2 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + sql show d1.tables if $rows != 1 then return -1 endi -print =============== step4: move leader +print =============== step38: move leader +print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +sql show d1.tables +if $rows != 1 then + return -1 +endi +print =============== step39: move 5 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +sql show d1.tables +if $rows != 1 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT