Merge pull request #13391 from taosdata/fix/tsim

enh: add creating and dropping states to mnode

include/util/taoserror.h

@@ -85,6 +85,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RPC_NETWORK_UNAVAIL           TAOS_DEF_ERROR_CODE(0, 0x0102)
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0103)
 #define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0104)
+#define TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL  TAOS_DEF_ERROR_CODE(0, 0x0105)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)

source/common/src/systable.c

@@ -36,6 +36,7 @@ static const SSysDbTableSchema mnodesSchema[] = {
     {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
 

source/dnode/mnode/impl/src/mndMnode.c

@@ -392,11 +392,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
 
   mDebug("mnode:%d, start to create", createReq.dnodeId);
 
-  if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
-    terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
-    goto _OVER;
-  }
-
   pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
     terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
@@ -405,12 +400,22 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) {
+    terrno = TSDB_CODE_MND_TOO_MANY_MNODES;
+    goto _OVER;
+  }
+
   pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
   if (pDnode == NULL) {
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
     goto _OVER;
   }
 
+  if (!mndIsDnodeOnline(pMnode, pDnode, taosGetTimestampMs())) {
+    terrno = TSDB_CODE_NODE_OFFLINE;
+    goto _OVER;
+  }
+
   pUser = mndAcquireUser(pMnode, pReq->conn.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
@@ -632,11 +637,12 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
   int32_t    numOfRows = 0;
   int32_t    cols = 0;
   SMnodeObj *pObj = NULL;
+  ESdbStatus objStatus;
   char      *pWrite;
   int64_t    curMs = taosGetTimestampMs();
 
   while (numOfRows < rows) {
-    pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj);
+    pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus);
     if (pShow->pIter == NULL) break;
 
     cols = 0;
@@ -649,23 +655,26 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, b1, false);
 
-    bool        online = mndIsDnodeOnline(pMnode, pObj->pDnode, curMs);
-    const char *roles = NULL;
+    const char *roles = "OFFLINE";
     if (pObj->id == pMnode->selfDnodeId) {
       roles = syncStr(TAOS_SYNC_STATE_LEADER);
-    } else {
-      if (!online) {
-        roles = "OFFLINE";
-      } else {
-        roles = syncStr(pObj->state);
-      }
     }
-    char *b2 = taosMemoryCalloc(1, 12 + VARSTR_HEADER_SIZE);
+    if (pObj->pDnode && mndIsDnodeOnline(pMnode, pObj->pDnode, curMs)) {
+      roles = syncStr(pObj->state);
+    }
+    char b2[12 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->pMeta->pSchemas[cols].bytes);
-
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
 
+    const char *status = "READY";
+    if (objStatus == SDB_STATUS_CREATING) status = "CREATING";
+    if (objStatus == SDB_STATUS_DROPPING) status = "DROPPING";
+    char b3[9 + VARSTR_HEADER_SIZE] = {0};
+    STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)b3, false);
+
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false);
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -344,7 +344,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
       action.pRaw = taosMemoryMalloc(dataLen);
       if (action.pRaw == NULL) goto _OVER;
-      mTrace("raw:%p, is created", pData);
+      mTrace("raw:%p, is created", action.pRaw);
       SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER);
       if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER;
       action.pRaw = NULL;
@@ -619,9 +619,7 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *
   pTrans->paramLen = paramLen;
 }
 
-void mndTransSetDbName(STrans *pTrans, const char *dbname) {
-  memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN);
-}
+void mndTransSetDbName(STrans *pTrans, const char *dbname) { memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN); }
 
 void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
 
@@ -753,22 +751,30 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
       sendRsp = true;
     }
   } else {
-    if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) {
+    if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) {
       if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
       sendRsp = true;
     }
   }
 
   if (sendRsp && pTrans->rpcInfo.handle != NULL) {
-    void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
-    if (rpcCont != NULL) {
-      memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen);
-    }
-    taosMemoryFree(pTrans->rpcRsp);
-
     mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage),
            pTrans->rpcInfo.ahandle);
-    SRpcMsg rspMsg = {.code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, .info = pTrans->rpcInfo};
+    if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+      code = TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL;
+    }
+    SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo};
+
+    if (pTrans->rpcRspLen != 0) {
+      void *rpcCont = rpcMallocCont(pTrans->rpcRspLen);
+      if (rpcCont != NULL) {
+        memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen);
+        rspMsg.pCont = rpcCont;
+        rspMsg.contLen = pTrans->rpcRspLen;
+      }
+      taosMemoryFree(pTrans->rpcRsp);
+    }
+
     tmsgSendRsp(&rspMsg);
     pTrans->rpcInfo.handle = NULL;
     pTrans->rpcRsp = NULL;
@@ -1025,18 +1031,23 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
     }
 
     if (code == 0) {
+      pTrans->code = 0;
       pTrans->redoActionPos++;
       mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage),
              pAction->id);
       code = mndTransSync(pMnode, pTrans);
       if (code != 0) {
-        mError("trans:%d, failed to sync redoActionPos since %s", pTrans->id, terrstr());
+        pTrans->code = terrno;
+        mError("trans:%d, %s:%d is executed and failed to sync to other mnodes since %s", pTrans->id,
+               mndTransStr(pAction->stage), pAction->id, terrstr());
         break;
       }
     } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
       mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id);
       break;
     } else {
+      terrno = code;
+      pTrans->code = code;
       mError("trans:%d, %s:%d failed to execute since %s", pTrans->id, mndTransStr(pAction->stage), pAction->id,
              terrstr());
       break;
@@ -1239,19 +1250,8 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
     return -1;
   }
 
-  int32_t size = taosArrayGetSize(pArray);
-
-  for (int32_t i = 0; i < size; ++i) {
+  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
     STransAction *pAction = taosArrayGet(pArray, i);
-    if (pAction == NULL) continue;
-
-    if (pAction->msgReceived == 0) {
-      mInfo("trans:%d, %s:%d set processed for kill msg received", pTrans->id, mndTransStr(pAction->stage), i);
-      pAction->msgSent = 1;
-      pAction->msgReceived = 1;
-      pAction->errCode = 0;
-    }
-
     if (pAction->errCode != 0) {
       mInfo("trans:%d, %s:%d set processed for kill msg received, errCode from %s to success", pTrans->id,
             mndTransStr(pAction->stage), i, tstrerror(pAction->errCode));
@@ -1290,9 +1290,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) {
 
   pTrans = mndAcquireTrans(pMnode, killReq.transId);
   if (pTrans == NULL) {
-    terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
-    mError("trans:%d, failed to kill since %s", killReq.transId, terrstr());
-    return -1;
+    goto _OVER;
   }
 
   code = mndKillTrans(pMnode, pTrans);
@@ -1300,9 +1298,9 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) {
 _OVER:
   if (code != 0) {
     mError("trans:%d, failed to kill since %s", killReq.transId, terrstr());
-    return -1;
   }
 
+  mndReleaseUser(pMnode, pUser);
   mndReleaseTrans(pMnode, pTrans);
   return code;
 }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -504,7 +504,7 @@ _OVER:
   taosArrayDestroy(pArray);
   return code;
 }
-//--->
+
 int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
@@ -543,7 +543,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
   terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   return -1;
 }
-//--->
+
 int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) {
   taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
   for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {

source/dnode/mnode/sdb/inc/sdb.h

@@ -301,6 +301,7 @@ void sdbRelease(SSdb *pSdb, void *pObj);
  * @return void* The next iterator of the table.
  */
 void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj);
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) ;
 
 /**
  * @brief Cancel a traversal

source/dnode/mnode/sdb/src/sdbHash.c

@@ -368,6 +368,34 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
   return ppRow;
 }
 
+void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) {
+  *ppObj = NULL;
+
+  SHashObj *hash = sdbGetHash(pSdb, type);
+  if (hash == NULL) return NULL;
+
+  TdThreadRwlock *pLock = &pSdb->locks[type];
+  taosThreadRwlockRdlock(pLock);
+
+  SSdbRow **ppRow = taosHashIterate(hash, pIter);
+  while (ppRow != NULL) {
+    SSdbRow *pRow = *ppRow;
+    if (pRow == NULL) {
+      ppRow = taosHashIterate(hash, ppRow);
+      continue;
+    }
+
+    atomic_add_fetch_32(&pRow->refCount, 1);
+    sdbPrintOper(pSdb, pRow, "fetch");
+    *ppObj = pRow->pObj;
+    *status = pRow->status;
+    break;
+  }
+  taosThreadRwlockUnlock(pLock);
+
+  return ppRow;
+}
+
 void sdbCancelFetch(SSdb *pSdb, void *pIter) {
   if (pIter == NULL) return;
   SSdbRow  *pRow = *(SSdbRow **)pIter;

source/libs/transport/src/transCli.c

@@ -962,7 +962,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
    * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL
    */
   tmsg_t msgType = pCtx->msgType;
-  if ((pTransInst->retry != NULL && (pTransInst->retry(pResp->code))) ||
+  if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) ||
       (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY ||
        pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) {
     pMsg->sent = 0;

source/util/src/terror.c

@@ -90,6 +90,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE,             "Authentication failur
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL,          "Unable to establish connection")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR,               "Unable to resolve FQDN")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE,          "Port already in use")
+TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL, "Unable to establish connection")
 
 //client
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION,        "Invalid operation")
@@ -244,7 +245,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB,       "Database is single st
 // mnode-infoSchema
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME,    "Invalid system table name")
 
-
 // mnode-func
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST,       "Func already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST,           "Func not exists")
@@ -325,9 +325,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         "Invalid table ID")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE,       "Invalid table type")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION,    "Invalid table schema version")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST,      "Table already exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST,      "Table not exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST,      "Stable already exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST,      "Stable not exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST,          "Table not exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST,        "Stable already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST,            "Stable not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG,           "Invalid configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED,              "Tsdb init failed")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE,             "No diskspace for tsdb")

tests/script/tsim/mnode/basic3.sim

@@ -3,6 +3,10 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
@@ -18,7 +22,7 @@ $x = 0
 step1: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 50 then
+	if $x == 10 then
 		return -1
 	endi
 sql show dnodes -x step1
@@ -41,7 +45,7 @@ $x = 0
 step2: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 50 then
+	if $x == 10 then
 		return -1
 	endi
 sql show mnodes -x step2
@@ -72,7 +76,7 @@ $x = 0
 step4: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 50 then
+	if $x == 10 then
 		return -1
 	endi
 sql show mnodes -x step4
@@ -102,7 +106,7 @@ $x = 0
 step5: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 50 then
+	if $x == 10 then
 		return -1
 	endi
 sql show mnodes -x step5
@@ -127,7 +131,7 @@ $x = 0
 step6: 
 	$x = $x + 1
 	sleep 1000
-	if $x == 50 then
+	if $x == 10 then
 		return -1
 	endi
 sql show mnodes -x step6

tests/script/tsim/mnode/basic4.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+
+print =============== step1: create dnodes
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 5 then
+		return -1
+	endi
+sql show dnodes -x step1
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create mnode 2
+sql create mnode on dnode 2
+sql_error create mnode on dnode 3
+return
+system sh/exec.sh -n dnode3 -s start
+
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 5 then
+		return -1
+	endi
+sql show dnodes -x step2
+if $data(1)[4] != ready then
+  goto step2
+endi
+if $data(2)[4] != ready then
+  goto step2
+endi
+
+sleep 2000
+system sh/exec.sh -n dnode3 -s stop
+sql_error create mnode on dnode 3
+
+print =============== step3: show mnodes
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes -x step3
+print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4]
+
+if $data(1)[2] != LEADER then
+  goto step3
+endi
+if $data(2)[2] != FOLLOWER then
+  goto step3
+endi
+if $data(3)[2] != OFFLINE then
+  goto step3
+endi
+if $data(1)[3] != READY then
+  goto step3
+endi
+if $data(2)[3] != READY then
+  goto step3
+endi
+if $data(3)[3] != CREATING then
+  goto step3
+endi
+
+print =============== step4: start dnode3
+system sh/exec.sh -n dnode3 -s start
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show mnodes -x step4
+if $data(1)[2] != LEADER then
+  goto step4
+endi
+if $data(2)[2] != FOLLOWER then
+  goto step4
+endi
+if $data(3)[2] != FOLLOWER then
+  goto step4
+endi
+if $data(1)[3] != READY then
+  goto step4
+endi
+if $data(2)[3] != READY then
+  goto step4
+endi
+if $data(3)[3] != READY then
+  goto step4
+endi
+
+print =============== step5: drop mnode 3 and stop dnode3
+
+return
+
+print =============== step3: create user
+sql create user user1 PASS 'user1'
+sql show users
+if $rows != 2 then 
+  return -1
+endi
+
+# wait mnode2 mnode3 recv data finish
+sleep 10000
+
+print =============== step4: stop dnode1
+system sh/exec.sh -n dnode1 -s stop
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 50 then
+		return -1
+	endi
+sql show mnodes -x step4
+print $data(1)[0] $data(1)[1] $data(1)[2] 
+print $data(2)[0] $data(2)[1] $data(2)[2] 
+print $data(3)[0] $data(3)[1] $data(3)[2] 
+
+sql show users
+if $rows != 2 then 
+  return -1
+endi
+
+sleep 1000
+sql show dnodes
+if $data(2)[4] != ready then
+  return -1
+endi
+if $data(3)[4] != ready then
+  return -1
+endi
+
+print =============== step5: stop dnode1
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s stop
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 50 then
+		return -1
+	endi
+sql show mnodes -x step5
+print $data(1)[0] $data(1)[1] $data(1)[2] 
+print $data(2)[0] $data(2)[1] $data(2)[2] 
+print $data(3)[0] $data(3)[1] $data(3)[2] 
+
+if $data(2)[2] != OFFLINE then
+  goto step5
+endi
+
+sql show users
+if $rows != 2 then 
+  return -1
+endi
+
+print =============== step6: stop dnode1
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s stop
+
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 50 then
+		return -1
+	endi
+sql show mnodes -x step6
+print $data(1)[0] $data(1)[1] $data(1)[2] 
+print $data(2)[0] $data(2)[1] $data(2)[2] 
+print $data(3)[0] $data(3)[1] $data(3)[2] 
+
+sql show users
+if $rows != 2 then 
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop
+system sh/exec.sh -n dnode2 -s stop
+system sh/exec.sh -n dnode3 -s stop
+system sh/exec.sh -n dnode4 -s stop
\ No newline at end of file

tests/script/tsim/stable/column_add.sim

@@ -143,7 +143,7 @@ sql insert into db.ctb values(now+2s, 1, 2, 3, 4)
 sql select * from db.stb
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 3 then
   return -1
@@ -200,7 +200,6 @@ sql insert into db.ctb values(now+3s, 1, 2, 3, 4, 5)
 sql select * from db.stb
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
 print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 4 then

tests/script/tsim/stable/tag_add.sim

@@ -129,7 +129,7 @@ sql select * from db.stb
 sql select * from db.stb
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1
@@ -160,7 +160,7 @@ sql insert into db.ctb2 values(now, 1, "2")
 sql select * from db.stb where tbname = 'ctb2';
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1

tests/script/tsim/stable/tag_drop.sim

@@ -155,7 +155,7 @@ sql select * from db.stb
 sql select * from db.stb
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1
@@ -186,7 +186,7 @@ sql insert into db.ctb2 values(now, 1, "2")
 sql select * from db.stb where tbname = 'ctb2';
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1
@@ -219,7 +219,7 @@ sql select * from db.stb where tbname = 'ctb2';
 sql select * from db.stb where tbname = 'ctb2';
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1
@@ -251,7 +251,7 @@ sql insert into db.ctb3 values(now, 1, "2")
 sql select * from db.stb where tbname = 'ctb3';
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1
@@ -313,7 +313,7 @@ endi
 sql select * from db.stb where tbname = 'ctb3';
 print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
 print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
+print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6]
 
 if $rows != 1 then
   return -1