Merge branch 'develop' into hotfix/crash

src/client/inc/tscUtil.h

@@ -247,6 +247,8 @@ void tscDoQuery(SSqlObj* pSql);
  * @param pPrevSql
  * @return
  */
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd);
+
 SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql);
 void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
 

src/client/inc/tsclient.h

@@ -296,7 +296,7 @@ typedef struct STscObj {
 typedef struct SSqlObj {
   void            *signature;
   STscObj         *pTscObj;
-  void            *SRpcReqContext;
+  void            *pRpcCtx;
   void            (*fp)();
   void            (*fetchFp)();
   void            *param;
@@ -308,8 +308,7 @@ typedef struct SSqlObj {
   char             retry;
   char             maxRetry;
   SRpcIpSet        ipList;
-  char             freed : 4;
-  char             listed : 4;
+  char             listed;
   tsem_t           rspSem;
   SSqlCmd          cmd;
   SSqlRes          res;
@@ -349,7 +348,7 @@ typedef struct SSqlStream {
 int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn);
 void    tscInitMsgsFp();
 
-int tsParseSql(SSqlObj *pSql, bool initialParse);
+int tsParseSql(SSqlObj *pSql, bool initial);
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet);
 int  tscProcessSql(SSqlObj *pSql);

src/client/src/tscParseInsert.c

@@ -1048,7 +1048,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
     str = pCmd->curSql;
   }
   
-  tscTrace("%p create data block list for submit data:%p, curSql:%p, pTableList:%p", pSql, pSql->cmd.pDataBlocks, pCmd->curSql, pCmd->pTableList);
+  tscTrace("%p create data block list for submit data:%p, pTableList:%p", pSql, pCmd->pDataBlocks, pCmd->pTableList);
 
   while (1) {
     int32_t   index = 0;
@@ -1088,22 +1088,16 @@ int tsParseInsertSql(SSqlObj *pSql) {
       goto _error;
     }
 
-    ptrdiff_t pos = pCmd->curSql - pSql->sqlstr;
-    
     if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) {
       /*
-       * For async insert, after get the table meta from server, the sql string will not be
-       * parsed using the new table meta to avoid the overhead cause by get table meta data information.
-       * And during the getMeterMetaCallback function, the sql string will be parsed from the
-       * interrupted position.
+       * After retrieving the table meta from server, the sql string will be parsed from the paused position.
+       * And during the getTableMetaCallback function, the sql string will be parsed from the paused position.
        */
       if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) {
-        tscTrace("%p waiting for get table meta during insert, then resume from offset: %" PRId64 ", %s", pSql,
-            pos, pCmd->curSql);
         return code;
       }
       
-      tscError("%p async insert parse error, code:%d, reason:%s", pSql, code, tstrerror(code));
+      tscError("%p async insert parse error, code:%s", pSql, tstrerror(code));
       pCmd->curSql = NULL;
       goto _error;
     }
@@ -1317,11 +1311,11 @@ int tsInsertInitialCheck(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-int tsParseSql(SSqlObj *pSql, bool initialParse) {
+int tsParseSql(SSqlObj *pSql, bool initial) {
   int32_t ret = TSDB_CODE_SUCCESS;
   SSqlCmd* pCmd = &pSql->cmd;
 
-  if (!pCmd->parseFinished) {
+  if ((!pCmd->parseFinished) && (!initial)) {
     tscTrace("%p resume to parse sql: %s", pSql, pCmd->curSql);
   }
   
@@ -1330,12 +1324,12 @@ int tsParseSql(SSqlObj *pSql, bool initialParse) {
      * Set the fp before parse the sql string, in case of getTableMeta failed, in which
      * the error handle callback function can rightfully restore the user-defined callback function (fp).
      */
-    if (initialParse && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
+    if (initial && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) {
       pSql->fetchFp = pSql->fp;
       pSql->fp = (void(*)())tscHandleMultivnodeInsert;
     }
     
-    if (initialParse && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) {
+    if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) {
       return ret;
     }
     

src/client/src/tscSQLParser.c

@@ -5755,6 +5755,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
   const char* msg7 = "illegal number of tables in from clause";
   const char* msg8 = "too many columns in selection clause";
   const char* msg9 = "TWA query requires both the start and end time";
+  const char* msg10= "too many tables in from clause";
 
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -5790,6 +5791,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
 
   pQueryInfo->command = TSDB_SQL_SELECT;
 
+  if (pQuerySql->from->nExpr > 2) {
+    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
+  }
+
   // set all query tables, which are maybe more than one.
   for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) {
     tVariant* pTableItem = &pQuerySql->from->a[i].pVar;

src/client/src/tscServer.c

@@ -197,29 +197,34 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .code    = 0
   };
 
-  pSql->SRpcReqContext = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
+  pSql->pRpcCtx = rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
 void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   SSqlObj *pSql = (SSqlObj *)rpcMsg->handle;
-  if (pSql == NULL) {
+  if (pSql == NULL || pSql->signature != pSql) {
     tscError("%p sql is already released", pSql->signature);
     return;
   }
 
-  if (pSql->signature != pSql) {
-    tscError("%p sql is already released, signature:%p", pSql, pSql->signature);
-    return;
-  }
-
+  STscObj *pObj = pSql->pTscObj;
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
-  STscObj *pObj = pSql->pTscObj;
 
-  if (pObj->signature != pObj || pSql->freed == 1) {
-    tscTrace("%p sqlObj needs to be released or DB connection is closed, freed:%d pObj:%p signature:%p", pSql, pSql->freed,
+  if (pObj->signature != pObj) {
+    tscTrace("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
+
+    tscFreeSqlObj(pSql);
+    rpcFreeCont(rpcMsg->pCont);
+    return;
+  }
+
+  SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
+  if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
+    tscTrace("%p sqlObj needs to be released or DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command,
              pObj, pObj->signature);
+
     tscFreeSqlObj(pSql);
     rpcFreeCont(rpcMsg->pCont);
     return;
@@ -421,7 +426,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
      * sub-queries not correctly released and master sql object of super table query reaches an abnormal state.
      */
     pSql->pSubs[i]->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    rpcCancelRequest(pSql->pSubs[i]->SRpcReqContext);
+    rpcCancelRequest(pSql->pSubs[i]->pRpcCtx);
   }
 
   /*

src/client/src/tscSql.c

@@ -489,7 +489,6 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
 
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
     tscTrace("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]);
-    pSql->freed = 1;
     tscProcessSql(pSql);
 
     // in case of sync model query, waits for response and then goes on
@@ -631,7 +630,7 @@ void taos_stop_query(TAOS_RES *res) {
     return;
   }
 
-  rpcCancelRequest(pSql->SRpcReqContext);
+  rpcCancelRequest(pSql->pRpcCtx);
   tscTrace("%p query is cancelled", res);
 }
 

src/client/src/tscSubquery.c

@@ -1853,11 +1853,11 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
   SSubqueryState* pState = pSupporter->pState;
 
   // record the total inserted rows
-  if (numOfRows > 0 && tres != pParentObj) {
-      pParentObj->res.numOfRows += numOfRows;
+  if (numOfRows > 0) {
+    pParentObj->res.numOfRows += numOfRows;
   }
 
-  if (taos_errno(tres) != 0) {
+  if (taos_errno(tres) != TSDB_CODE_SUCCESS) {
     SSqlObj* pSql = (SSqlObj*) tres;
     assert(pSql != NULL && pSql->res.code == numOfRows);
     
@@ -1865,13 +1865,9 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
   }
 
   // it is not the initial sqlObj, free it
-  if (tres != pParentObj) {
-    taos_free_result(tres);
-  } else {
-    assert(pParentObj->pSubs[0] == tres);
-  }
-
+  taos_free_result(tres);
   tfree(pSupporter);
+
   if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
     return;
   }
@@ -1904,30 +1900,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   pState->numOfRemain = pSql->numOfSubs;
  
   pRes->code = TSDB_CODE_SUCCESS;
+  int32_t numOfSub = 0;
 
-  SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
-  pSupporter->pSql   = pSql;
-  pSupporter->pState = pState;
-
-  pSql->fp       = multiVnodeInsertFinalize;
-  pSql->param    = pSupporter;
-  pSql->pSubs[0] = pSql;  // the first sub insert points back to itself
-  tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pSql, 0);
-
-  int32_t numOfSub = 1;
-  int32_t code = tscCopyDataBlockToPayload(pSql, pDataBlocks->pData[0]);
-  if (code != TSDB_CODE_SUCCESS) {
-    tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, 0,
-             pDataBlocks->nSize, code);
-    goto _error;
-  }
-
-  for (; numOfSub < pSql->numOfSubs; ++numOfSub) {
-    SInsertSupporter* pSupporter1 = calloc(1, sizeof(SInsertSupporter));
-    pSupporter1->pSql = pSql;
-    pSupporter1->pState = pState;
+  while(numOfSub < pSql->numOfSubs) {
+    SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
+    pSupporter->pSql = pSql;
+    pSupporter->pState = pState;
     
-    SSqlObj *pNew = createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL);
+    SSqlObj *pNew = createSimpleSubObj(pSql, multiVnodeInsertFinalize, pSupporter, TSDB_SQL_INSERT);//createSubqueryObj(pSql, 0, multiVnodeInsertFinalize, pSupporter1, TSDB_SQL_INSERT, NULL);
     if (pNew == NULL) {
       tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, numOfSub, strerror(errno));
       goto _error;
@@ -1940,13 +1920,13 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
     pNew->fetchFp = pNew->fp;
     pSql->pSubs[numOfSub] = pNew;
 
-    code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub]);
-    if (code != TSDB_CODE_SUCCESS) {
-      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%d", pSql, numOfSub,
-               pDataBlocks->nSize, code);
-      goto _error;
-    } else {
+    pRes->code = tscCopyDataBlockToPayload(pNew, pDataBlocks->pData[numOfSub++]);
+    if (pRes->code == TSDB_CODE_SUCCESS) {
       tscTrace("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
+    } else {
+      tscTrace("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
+               pDataBlocks->nSize, tstrerror(pRes->code));
+      goto _error;
     }
   }
   
@@ -1966,18 +1946,12 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 
   _error:
-  // restore the udf fp
-  pSql->fp = pSql->fetchFp;
-  pSql->pSubs[0] = NULL;
-
-  tfree(pState);
-  tfree(pSql->param);
-
-  for(int32_t j = 1; j < numOfSub; ++j) {
+  for(int32_t j = 0; j < numOfSub; ++j) {
     tfree(pSql->pSubs[j]->param);
     taos_free_result(pSql->pSubs[j]);
   }
 
+  tfree(pState);
   return TSDB_CODE_TSC_OUT_OF_MEMORY;
 }
 

src/client/src/tscUtil.c

@@ -364,7 +364,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   
   tfree(pSql->pSubs);
-  pSql->freed = 0;
   pSql->numOfSubs = 0;
   
   tscResetSqlCmdObj(pCmd);
@@ -1653,6 +1652,46 @@ void tscResetForNextRetrieve(SSqlRes* pRes) {
   pRes->numOfRows = 0;
 }
 
+SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) {
+  SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
+  if (pNew == NULL) {
+    tscError("%p new subquery failed, tableIndex:%d", pSql, 0);
+    return NULL;
+  }
+
+  pNew->pTscObj = pSql->pTscObj;
+  pNew->signature = pNew;
+
+  SSqlCmd* pCmd = &pNew->cmd;
+  pCmd->command = cmd;
+
+  if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
+    tscFreeSqlObj(pNew);
+    return NULL;
+  }
+
+  pNew->fp = fp;
+  pNew->param = param;
+  pNew->maxRetry = TSDB_MAX_REPLICA_NUM;
+
+  pNew->sqlstr = strdup(pSql->sqlstr);
+  if (pNew->sqlstr == NULL) {
+    tscError("%p new subquery failed", pSql);
+
+    free(pNew);
+    return NULL;
+  }
+
+  SQueryInfo* pQueryInfo = NULL;
+  tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo);
+
+  assert(pSql->cmd.clauseIndex == 0);
+  STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
+
+  tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
+  return pNew;
+}
+
 SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) {
   SSqlCmd* pCmd = &pSql->cmd;
   SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
@@ -1795,6 +1834,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (pPrevSql == NULL) {
     STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta);  // get by name may failed due to the cache cleanup
     assert(pTableMeta != NULL);
+
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
   } else {  // transfer the ownership of pTableMeta to the newly create sql object.
     STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);

src/mnode/inc/mnodeDef.h

@@ -82,7 +82,8 @@ typedef struct STableObj {
 
 typedef struct SSuperTableObj {
   STableObj  info; 
-  int8_t     reserved0[3]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int16_t    nextColId;
   int32_t    sversion;
   uint64_t   uid;
   int64_t    createdTime;
@@ -95,13 +96,13 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   SSchema *  schema;
   void *     vgHash;
-  int16_t    nextColId;
   int8_t     reserved2[6];
 } SSuperTableObj;
 
 typedef struct {
   STableObj  info;  
-  int8_t     reserved0[3]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int16_t    nextColId;    //used by normal table
   int32_t    sversion;     //used by normal table  
   uint64_t   uid;
   uint64_t   suid;
@@ -112,7 +113,6 @@ typedef struct {
   int32_t    sqlLen;
   int8_t     updateEnd[1];
   int8_t     reserved1[1]; 
-  int16_t    nextColId;    //used by normal table
   int32_t    refCount;
   char*      sql;          //used by normal table
   SSchema*   schema;       //used by normal table

src/mnode/src/mnodeVgroup.c

@@ -258,7 +258,37 @@ void mnodeUpdateVgroup(SVgObj *pVgroup) {
   mnodeSendCreateVgroupMsg(pVgroup, NULL);
 }
 
-void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {}
+/*
+  Traverse all vgroups on mnode, if there no such vgId on a dnode, so send msg to this dnode for re-creating this vgId/vnode 
+*/
+void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes) {
+  SVnodeLoad *pNextV = NULL;
+
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    pNextV = pVloads;
+    int32_t i;
+    for (i = 0; i < openVnodes; ++i) {
+      if ((pVgroup->vnodeGid[i].pDnode == pDnode) && (pVgroup->vgId == pNextV->vgId)) {
+        break;
+      }
+      pNextV++;
+    }
+
+    if (i == openVnodes) {
+      mnodeSendCreateVgroupMsg(pVgroup, NULL);
+    }
+    
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+  return;
+}
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
   bool dnodeExist = false;
@@ -861,4 +891,4 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
   sdbFreeIter(pIter);
 
   mPrint("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
-}
+}
\ No newline at end of file

tests/script/jenkins/basic.txt

@@ -337,7 +337,9 @@ cd ../../../debug; make
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
-#./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
+./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
 ./test.sh -f unique/arbitrator/insert_duplicationTs.sim

tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+    $ts = $tsStart + $x
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) 
+    $x = $x + 60
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step3: stop dnode3, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode3_offline_0
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != offline then
+  sleep 2000
+  goto wait_dnode3_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+if $dnode3Vtatus != offline then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+if $dnode2Vtatus != master then
+  sleep 2000
+  goto wait_dnode3_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step4: restart dnode3, waiting sync end 
+system sh/exec.sh -n dnode3 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode2Vtatus: $dnode3Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+if $dnode2Vtatus != master then
+  sleep 2000
+  goto wait_dnode3_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+print ============== step6: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode3Vtatus = $data4_2
+$dnode2Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+if $dnode2Vtatus != slave then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
\ No newline at end of file

tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sql create dnode $hostname4
+sleep 3000
+
+$totalTableNum = 10
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 3 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+    $ts = $tsStart + $x
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) 
+    $x = $x + 60
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step3: stop dnode4, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode4_offline_0
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode4Status != offline then
+  sleep 2000
+  goto wait_dnode4_offline_0
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+if $dnode4Vtatus != offline then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_offline
+endi
+
+system rm -rf ../../../sim/dnode4/data
+#system rm -rf ../../../sim/dnode4/data/vnode/*
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step4: restart dnode4, waiting sync end 
+system sh/exec.sh -n dnode4 -s start
+#sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode4_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode4_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+if $dnode3Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_slave
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode3/dnode2, and remove its vnode
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+if $dnode3Status != offline then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode23_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data
+system rm -rf ../../../sim/dnode3/data
+#system rm -rf ../../../sim/dnode2/data/vnode/*
+#system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode2/dnode3, and check rows
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode23_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode23_reready
+endi
+
+$loopCnt = 0
+wait_dnode4_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_master
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+print ============== step7: stop dnode3/dnode2, and cluster unable to provide services
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step8: restart dnode2, and cluster Still unable to provide services
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+sql_error select count(*) from $stb
+
+print ============== step9: restart dnode3, and cluster Resume service delivery
+system sh/exec.sh -n dnode3 -s start
+
+$loopCnt = 0
+wait_dnode4_vgroup_master_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode4_vgroup_master_2
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi

tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 system sh/exec.sh -n dnode4 -s start
@@ -96,7 +97,14 @@ endi
 print ============== step3: stop dnode4, and remove its vnodeX subdirector 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -105,21 +113,25 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode4Status != offline then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
 
+
+$loopCnt = 0
 wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -141,14 +153,19 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_offline
 endi
 
-
 system rm -rf ../../../sim/dnode4/data/vnode/*
-sleep 1000
 
 print ============== step4: restart dnode4, waiting sync end 
 system sh/exec.sh -n dnode4 -s start
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -157,21 +174,24 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode4Status != ready then
   sleep 2000
   goto wait_dnode4_reready
 endi
 
+$loopCnt = 0
 wait_dnode4_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -195,47 +215,87 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_slave
 endi
 
-print ============== step5: stop dnode3/dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
-system sh/exec.sh -n dnode3 -s stop
+print ============== step5: stop dnode3, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
 
-wait_dnode23_offline:
+$loopCnt = 0
+wait_dnode3_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
-if $dnode2Status != offline then
+if $dnode2Status != ready then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 if $dnode3Status != offline then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
 endi
 if $dnode4Status != ready then
   sleep 2000
-  goto wait_dnode23_offline
+  goto wait_dnode3_offline
+endi
+
+system rm -rf ../../../sim/dnode3/data/vnode/*
+
+print ============== step6: restart dnode3, and check rows
+system sh/exec.sh -n dnode3 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode3_reready
+endi
+
+$loopCnt = 0
+wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
 endi
 
-wait_dnode4_vgroup_master:
 sql show vgroups
 if $rows != 1 then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
 print show vgroups:
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
@@ -248,11 +308,11 @@ print dnode4Vtatus: $dnode4Vtatus
 print dnode3Vtatus: $dnode3Vtatus 
 if $dnode4Vtatus != master then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
-if $dnode3Vtatus != offline then
+if $dnode3Vtatus != slave then
   sleep 2000
-  goto wait_dnode4_vgroup_master
+  goto wait_dnode3_vgroup_slave
 endi
 
 # check using select
@@ -271,3 +331,128 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+print ============== step7: stop dnode2, and remove its vnodeX subdirector 
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+system rm -rf ../../../sim/dnode2/data/vnode/*
+
+print ============== step8: restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s start
+sleep $sleepTimer
+
+$loopCnt = 0
+wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_reready
+endi
+
+$loopCnt = 0
+wait_dnode2_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
+sql show vgroups
+if $rows != 1 then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+print show vgroups:
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
+$dnode4Vtatus = $data4_2
+$dnode3Vtatus = $data7_2
+
+print dnode4Vtatus: $dnode4Vtatus
+print dnode3Vtatus: $dnode3Vtatus 
+if $dnode4Vtatus != master then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+if $dnode3Vtatus != slave then
+  sleep 2000
+  goto wait_dnode2_vgroup_slave
+endi
+
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+sql insert into $tb values ( now , 20000 ) ( now + 1a, 20001 ) ( now + 2a, 20002 )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+
+
+
+
+
+
+
+

tests/script/unique/arbitrator/testSuite.sim

@@ -12,7 +12,9 @@ run unique/arbitrator/dn3_mn1_vnode_change.sim
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim 
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
 run unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
-####run unique/arbitrator/dn3_mn1_vnode_delDir.sim                  # unsupport
+run unique/arbitrator/dn3_mn1_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim 
+run unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim 
 run unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 run unique/arbitrator/dn3_mn2_killDnode.sim
 run unique/arbitrator/insert_duplicationTs.sim
@@ -34,4 +36,4 @@ run unique/arbitrator/sync_replica2_dropTable.sim
 run unique/arbitrator/sync_replica3_alterTable_add.sim
 run unique/arbitrator/sync_replica3_alterTable_drop.sim
 run unique/arbitrator/sync_replica3_dropDb.sim
-run unique/arbitrator/sync_replica3_dropTable.sim
+run unique/arbitrator/sync_replica3_dropTable.sim
\ No newline at end of file