Merge branch 'develop' into feature/2.0tsdb

packaging/cfg/taos.cfg

@@ -51,18 +51,15 @@
 # number of threads per CPU core
 # numOfThreadsPerCore   1.0
 
-# number of vnodes per core in DNode
-# numOfVnodesPerCore    8
+# number of vgroups per db
+# maxVgroupsPerDb       0
+
+# max number of tables per vnode
+# maxTablesPerVnode     1000000
 
 # the ratio of threads responsible for querying in the total thread
 # ratioOfQueryThreads   0.5
 
-# number of total vnodes in DNode
-# numOfTotalVnodes      0
-
-# max number of tables per vnode
-# maxtablesPerVnode     1000
-
 # interval of check load balance when the management node is in normal operation
 # balanceInterval       300
 

src/client/src/tscAsync.c

@@ -45,6 +45,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
   pSql->pTscObj   = pObj;
   pSql->maxRetry  = TSDB_MAX_REPLICA;
   pSql->fp        = fp;
+  pSql->fetchFp   = fp;
 
   pSql->sqlstr = calloc(1, sqlLen + 1);
   if (pSql->sqlstr == NULL) {
@@ -159,7 +160,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
       pRes->code = numOfRows;
     }
 
-    tscQueueAsyncError(pSql->fetchFp, param, pRes->code);
+    tscQueueAsyncRes(pSql);
     return;
   }
 
@@ -167,6 +168,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
   if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
   }
+
   tscProcessSql(pSql);
 }
 
@@ -346,31 +348,32 @@ void tscProcessFetchRow(SSchedMsg *pMsg) {
 
 void tscProcessAsyncRes(SSchedMsg *pMsg) {
   SSqlObj *pSql = (SSqlObj *)pMsg->ahandle;
-  SSqlCmd *pCmd = &pSql->cmd;
+//  SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 
-  void *taosres = pSql;
+//  void *taosres = pSql;
 
   // pCmd may be released, so cache pCmd->command
-  int cmd = pCmd->command;
-  int code = pRes->code;
+//  int cmd = pCmd->command;
+//  int code = pRes->code;
 
   // in case of async insert, restore the user specified callback function
-  bool shouldFree = tscShouldBeFreed(pSql);
-
-  if (cmd == TSDB_SQL_INSERT) {
-    assert(pSql->fp != NULL);
-    pSql->fp = pSql->fetchFp;
-  }
-
-  if (pSql->fp) {
-    (*pSql->fp)(pSql->param, taosres, code);
-  }
-
-  if (shouldFree) {
-    tscDebug("%p sqlObj is automatically freed in async res", pSql);
-    tscFreeSqlObj(pSql);
-  }
+//  bool shouldFree = tscShouldBeFreed(pSql);
+
+//  if (pCmd->command == TSDB_SQL_INSERT) {
+//    assert(pSql->fp != NULL);
+  assert(pSql->fp != NULL && pSql->fetchFp != NULL);
+//  }
+
+//  if (pSql->fp) {
+  pSql->fp = pSql->fetchFp;
+  (*pSql->fp)(pSql->param, pSql, pRes->code);
+//  }
+
+//  if (shouldFree) {
+//    tscDebug("%p sqlObj is automatically freed in async res", pSql);
+//    tscFreeSqlObj(pSql);
+//  }
 }
 
 static void tscProcessAsyncError(SSchedMsg *pMsg) {
@@ -420,15 +423,15 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
 
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
+  pRes->code = code;
 
   if (code != TSDB_CODE_SUCCESS) {
-    pRes->code = code;
-    tscQueueAsyncRes(pSql);
-    return;
+    tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code));
+    goto _error;
+  } else {
+    tscDebug("%p get tableMeta successfully", pSql);
   }
 
-  tscDebug("%p get tableMeta successfully", pSql);
-
   if (pSql->pStream == NULL) {
     SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
@@ -453,11 +456,9 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex &&
           pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL);
 
-      if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-        return;
-      }
-
-      goto _error;
+      // tscProcessSql can add error into async res
+      tscProcessSql(pSql);
+      return;
     } else {  // continue to process normal async query
       if (pCmd->parseFinished) {
         tscDebug("%p update table meta in local cache, continue to process sql and send corresponding query", pSql);
@@ -481,26 +482,21 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
 
           if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
             return;
+          } else if (code != TSDB_CODE_SUCCESS) {
+            goto _error;
           }
 
-          if (code == TSDB_CODE_SUCCESS) {
-            /*
-             * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
-             * and send the required submit block according to index value in supporter to server.
-             */
-            pSql->fp = pSql->fetchFp;  // restore the fp
-            if ((code = tscHandleInsertRetry(pSql)) == TSDB_CODE_SUCCESS) {
-              return;
-            }
-          }
-
+          /*
+           * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
+           * and send the required submit block according to index value in supporter to server.
+           */
+          pSql->fp = pSql->fetchFp;  // restore the fp
+          tscHandleInsertRetry(pSql);
         } else {// in case of other query type, continue
-          if ((code = tscProcessSql(pSql)) == TSDB_CODE_SUCCESS) {
-            return;
-          }
+          tscProcessSql(pSql);
         }
 
-        goto _error;
+        return;
       } else {
         tscDebug("%p continue parse sql after get table meta", pSql);
 
@@ -538,7 +534,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
       goto _error;
     }
 
-    if (code == TSDB_CODE_SUCCESS && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
+    if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
       code = tscGetSTableVgroupInfo(pSql, pCmd->clauseIndex);
       if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
         return;

src/client/src/tscSQLParser.c

@@ -2471,7 +2471,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
     startIdx++;
   }
 
-  int32_t factor = funcCompatDefList[tscSqlExprGet(pQueryInfo, startIdx)->functionId];
+  int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId];
 
   // diff function cannot be executed with other function
   // arithmetic function can be executed with other arithmetic functions
@@ -2489,7 +2489,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) {
       continue;
     }
 
-    if (funcCompatDefList[functionId] != factor) {
+    if (functionCompatList[functionId] != factor) {
       return false;
     }
   }

src/client/src/tscServer.c

@@ -339,7 +339,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) {
   }
 
   if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
-    rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? pRes->numOfRows: pRes->code;
+    rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS)? pRes->numOfRows: pRes->code;
     
     bool shouldFree = tscShouldBeFreed(pSql);
     (*pSql->fp)(pSql->param, pSql, rpcMsg->code);
@@ -412,7 +412,7 @@ int tscProcessSql(SSqlObj *pSql) {
       return pSql->res.code;
     }
   } else if (pCmd->command < TSDB_SQL_LOCAL) {
-    pSql->ipList = tscMgmtIpSet; //?
+    pSql->ipList = tscMgmtIpSet;
   } else {  // local handler
     return (*tscProcessMsgRsp[pCmd->command])(pSql);
   }
@@ -476,6 +476,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
     int32_t vgIndex = pTableMetaInfo->vgroupIndex;
     
     SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList;
+    assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups);
+
     pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId);
   } else {
     STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@@ -549,6 +551,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
       assert(index >= 0);
   
       if (pTableMetaInfo->vgroupList->numOfVgroups > 0) {
+        assert(index < pTableMetaInfo->vgroupList->numOfVgroups);
         pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index];
       }
       tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups);
@@ -1372,7 +1375,6 @@ static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
   pRes->code = TSDB_CODE_SUCCESS;
-
   if (pRes->rspType == 0) {
     pRes->numOfRows = numOfRes;
     pRes->row = 0;

src/client/src/tscSql.c

@@ -481,25 +481,14 @@ static bool tscFreeQhandleInVnode(SSqlObj* pSql) {
 
   if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) &&
       (pCmd->command == TSDB_SQL_SELECT ||
-          pCmd->command == TSDB_SQL_SHOW ||
-          pCmd->command == TSDB_SQL_RETRIEVE ||
-          pCmd->command == TSDB_SQL_FETCH) &&
-      (pCmd->command == TSDB_SQL_SELECT && pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
+       pCmd->command == TSDB_SQL_SHOW ||
+       pCmd->command == TSDB_SQL_RETRIEVE ||
+       pCmd->command == TSDB_SQL_FETCH) &&
+      (pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
 
     pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
-    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s", pSql, sqlCmd[pCmd->command]);
+    tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
     tscProcessSql(pSql);
-
-    // in case of sync model query, waits for response and then goes on
-//    if (pSql->fp == waitForQueryRsp || pSql->fp == waitForRetrieveRsp) {
-//      sem_wait(&pSql->rspSem);
-
-//      tscFreeSqlObj(pSql);
-//      tscDebug("%p sqlObj is freed by app", pSql);
-//    } else {
-      tscDebug("%p sqlObj will be freed while rsp received", pSql);
-//    }
-
     return true;
   }
 

src/client/src/tscSubquery.c

@@ -1895,9 +1895,11 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
   assert(pSupporter->index < pSupporter->pState->numOfTotal);
 
   STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index);
-  pRes->code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
-  if (pRes->code != TSDB_CODE_SUCCESS) {
-    return pRes->code;
+  int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock);
+
+  if ((pRes->code = code)!= TSDB_CODE_SUCCESS) {
+    tscQueueAsyncRes(pSql);
+    return code;  // here the pSql may have been released already.
   }
 
   return tscProcessSql(pSql);

src/client/src/tscUtil.c

@@ -1648,6 +1648,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   }
 
   pNew->fp = fp;
+  pNew->fetchFp = fp;
   pNew->param = param;
   pNew->maxRetry = TSDB_MAX_REPLICA;
 
@@ -1803,6 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   pNew->fp = fp;
+  pNew->fetchFp = fp;
+
   pNew->param = param;
   pNew->maxRetry = TSDB_MAX_REPLICA;
 
@@ -2005,7 +2008,7 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   
   int32_t totalVgroups = pTableMetaInfo->vgroupList->numOfVgroups;
-  while (++pTableMetaInfo->vgroupIndex < totalVgroups) {
+  if (++pTableMetaInfo->vgroupIndex < totalVgroups) {
     tscDebug("%p results from vgroup index:%d completed, try next:%d. total vgroups:%d. current numOfRes:%" PRId64, pSql,
              pTableMetaInfo->vgroupIndex - 1, pTableMetaInfo->vgroupIndex, totalVgroups, pRes->numOfClauseTotal);
 
@@ -2041,11 +2044,9 @@ void tscTryQueryNextVnode(SSqlObj* pSql, __async_cb_func_t fp) {
 
     // set the callback function
     pSql->fp = fp;
-    int32_t ret = tscProcessSql(pSql);
-    if (ret == TSDB_CODE_SUCCESS) {
-      return;
-    } else {// todo check for failure
-    }
+    tscProcessSql(pSql);
+  } else {
+    tscDebug("%p try all %d vnodes, query complete. current numOfRes:%" PRId64, pSql, totalVgroups, pRes->numOfClauseTotal);
   }
 }
 

src/common/inc/tglobal.h

@@ -68,7 +68,9 @@ extern int64_t tsMaxRetentWindow;
 // db parameters in client
 extern int32_t tsCacheBlockSize;
 extern int32_t tsBlocksPerVnode;
+extern int32_t tsMinTablePerVnode;
 extern int32_t tsMaxTablePerVnode;
+extern int32_t tsTableIncStepPerVnode;
 extern int32_t tsMaxVgroupsPerDb;
 extern int16_t tsDaysPerFile;
 extern int32_t tsDaysToKeep;

src/common/src/tglobal.c

@@ -38,7 +38,7 @@ uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
 uint16_t tsDnodeDnodePort = 6035;  // udp/tcp
 uint16_t tsSyncPort = 6040;
 int32_t  tsStatusInterval = 1;  // second
-int16_t  tsNumOfVnodesPerCore = 8;
+int16_t  tsNumOfVnodesPerCore = 32;
 int16_t  tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
 int32_t  tsNumOfMnodes = 3;
 int32_t  tsEnableVnodeBak = 1;
@@ -112,7 +112,10 @@ int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
 int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
+int32_t tsMinTablePerVnode = 100;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
+int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
+
 // balance
 int32_t tsEnableBalance = 1;
 int32_t tsAlternativeRole = 0;
@@ -389,16 +392,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "numOfVnodesPerCore";
-  cfg.ptr = &tsNumOfVnodesPerCore;
-  cfg.valType = TAOS_CFG_VTYPE_INT16;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 1;
-  cfg.maxValue = 64;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   cfg.option = "numOfTotalVnodes";
   cfg.ptr = &tsNumOfTotalVnodes;
   cfg.valType = TAOS_CFG_VTYPE_INT16;
@@ -622,6 +615,26 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "minTablesPerVnode";
+  cfg.ptr = &tsMinTablePerVnode;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_TABLES;
+  cfg.maxValue = TSDB_MAX_TABLES;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "tableIncStepPerVnode";
+  cfg.ptr = &tsTableIncStepPerVnode;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_TABLES;
+  cfg.maxValue = TSDB_MAX_TABLES;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "cache";
   cfg.ptr = &tsCacheBlockSize;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/dnode/src/dnodePeer.c

@@ -60,7 +60,7 @@ int32_t dnodeInitServer() {
   rpcInit.label        = "DND-S";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp          = dnodeProcessReqMsgFromDnode;
-  rpcInit.sessions     = 100;
+  rpcInit.sessions     = TSDB_MAX_VNODES;
   rpcInit.connType     = TAOS_CONN_SERVER;
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
 
@@ -122,7 +122,7 @@ int32_t dnodeInitClient() {
   rpcInit.label        = "DND-C";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp          = dnodeProcessRspFromDnode;
-  rpcInit.sessions     = 100;
+  rpcInit.sessions     = TSDB_MAX_VNODES;
   rpcInit.connType     = TAOS_CONN_CLIENT;
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.user         = "t";

src/dnode/src/dnodeSystem.c

@@ -70,7 +70,13 @@ int32_t main(int32_t argc, char *argv[]) {
     }
 #endif
 #ifdef TAOS_RANDOM_FILE_FAIL
-    else if (strcmp(argv[i], "--random-file-fail-factor") == 0) {
+    else if (strcmp(argv[i], "--random-file-fail-output") == 0) {
+      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
+        taosSetRandomFileFailOutput(argv[++i]);
+      } else {
+        taosSetRandomFileFailOutput(NULL);
+      }
+    } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) {
       if ( (i+1) < argc ) {
         int factor = atoi(argv[i+1]);
         printf("The factor of random failure is %d\n", factor);

src/inc/taosdef.h

@@ -274,8 +274,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120   // default payload size, greater than PATH_MAX value
 #define TSDB_EXTRA_PAYLOAD_SIZE   128    // extra bytes for auth
 #define TSDB_CQ_SQL_SIZE          1024
-#define TSDB_MAX_VNODES           256
-#define TSDB_MIN_VNODES           50
+#define TSDB_MAX_VNODES           2048
+#define TSDB_MIN_VNODES           256
 #define TSDB_INVALID_VNODE_NUM    0
 
 #define TSDB_DNODE_ROLE_ANY       0
@@ -296,9 +296,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_TOTAL_BLOCKS       4
 
 #define TSDB_MIN_TABLES                 4
-#define TSDB_MAX_TABLES                 5000000
-#define TSDB_DEFAULT_TABLES             200000
-#define TSDB_TABLES_STEP                10000
+#define TSDB_MAX_TABLES                 10000000
+#define TSDB_DEFAULT_TABLES             1000000
+#define TSDB_TABLES_STEP                1000
 
 #define TSDB_MIN_DAYS_PER_FILE          1
 #define TSDB_MAX_DAYS_PER_FILE          3650 

src/inc/taoserror.h

@@ -209,6 +209,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE,             0, 0x0702, "query no d
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "query out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "query app error")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "query duplicated join key")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT,        0, 0x0706, "query tag conditon too many")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "grant expired")

src/mnode/src/mnodeTable.c

@@ -1740,11 +1740,13 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
         return code;
       }
 
-      if (pMsg->pVgroup == NULL) {
-        pMsg->pVgroup = pVgroup;
-        mnodeIncVgroupRef(pVgroup);
+      if (pMsg->pVgroup != NULL) {
+        mnodeDecVgroupRef(pMsg->pVgroup);
       }
 
+      pMsg->pVgroup = pVgroup;
+      mnodeIncVgroupRef(pVgroup);
+
       mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
              pVgroup->vgId, sid);
 

src/mnode/src/mnodeVgroup.c

@@ -323,7 +323,7 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
   if (pDb == NULL) return TSDB_CODE_MND_APP_ERROR;
 
   int32_t minIdPoolSize = TSDB_MAX_TABLES;
-  int32_t maxIdPoolSize = TSDB_MIN_TABLES;
+  int32_t maxIdPoolSize = tsMinTablePerVnode;
   for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
     SVgObj *pVgroup = pDb->vgList[v];
     if (pVgroup == NULL) continue;
@@ -347,10 +347,10 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
 
   // realloc all vgroups in db
   int32_t newIdPoolSize;
-  if (minIdPoolSize * 4 < TSDB_TABLES_STEP) {
+  if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
     newIdPoolSize = minIdPoolSize * 4;
   } else {
-    newIdPoolSize = ((minIdPoolSize / TSDB_TABLES_STEP) + 1) * TSDB_TABLES_STEP;
+    newIdPoolSize = ((minIdPoolSize / tsTableIncStepPerVnode) + 1) * tsTableIncStepPerVnode;
   }
 
   if (newIdPoolSize > tsMaxTablePerVnode) {
@@ -471,6 +471,8 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
   }
 
   pMsg->expected = pVgroup->numOfVnodes;
+  pMsg->successed = 0;
+  pMsg->received = 0;
   mnodeSendCreateVgroupMsg(pVgroup, pMsg);
 
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
@@ -836,6 +838,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
          pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected,
          mnodeMsg->rpcMsg.handle, rpcMsg->ahandle);
 
+  assert(mnodeMsg->received <= mnodeMsg->expected);
+
   if (mnodeMsg->received != mnodeMsg->expected) return;
 
   if (mnodeMsg->received == mnodeMsg->successed) {

src/query/inc/qExecutor.h

@@ -121,6 +121,7 @@ typedef struct SQueryCostInfo {
   uint32_t loadBlockStatis;
   uint32_t discardBlocks;
   uint64_t elapsedTime;
+  uint64_t ioTime;
   uint64_t computTime;
 } SQueryCostInfo;
 

src/query/inc/qUtil.h

@@ -37,7 +37,7 @@ SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot);
 #define curTimeWindow(_winres)  ((_winres)->curIndex)
 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
-void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize);
+int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize);
 
 char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult);
 

src/query/inc/tsqlfunction.h

@@ -125,7 +125,8 @@ typedef struct SArithmeticSupport {
 } SArithmeticSupport;
 
 typedef struct SQLPreAggVal {
-  bool    isSet;
+  bool        isSet;             // statistics info set or not
+  bool        dataBlockLoaded;   // data block is loaded or not
   SDataStatis statis;
 } SQLPreAggVal;
 
@@ -224,25 +225,14 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
 #define IS_SINGLEOUTPUT(x)        (((x)&TSDB_FUNCSTATE_SO) != 0)
 #define IS_OUTER_FORWARD(x)       (((x)&TSDB_FUNCSTATE_OF) != 0)
 
-/*
- * the status of one block, used in metric query. all blocks are mixed together,
- * we need the status to decide if one block is a first/end/inter block of one meter
- */
-enum {
-  BLK_FILE_BLOCK = 0x1,
-  BLK_BLOCK_LOADED = 0x2,
-  BLK_CACHE_BLOCK = 0x4,  // in case of cache block, block must be loaded
-};
-
 /* determine the real data need to calculated the result */
 enum {
-  BLK_DATA_NO_NEEDED = 0x0,
+  BLK_DATA_NO_NEEDED     = 0x0,
   BLK_DATA_STATIS_NEEDED = 0x1,
-  BLK_DATA_ALL_NEEDED = 0x3,
+  BLK_DATA_ALL_NEEDED    = 0x3,
+  BLK_DATA_DISCARD       = 0x4,   // discard current data block since it is not qualified for filter
 };
 
-#define SET_DATA_BLOCK_NOT_LOADED(x) ((x) &= (~BLK_BLOCK_LOADED));
-
 typedef struct STwaInfo {
   TSKEY   lastKey;
   int8_t  hasResult;  // flag to denote has value
@@ -264,12 +254,9 @@ typedef struct STwaInfo {
 /* global sql function array */
 extern struct SQLAggFuncElem aAggs[];
 
-/* compatible check array list */
-extern int32_t funcCompatDefList[];
-
-bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *minval, char *maxval);
+extern int32_t functionCompatList[]; // compatible check array list
 
-bool stableQueryFunctChanged(int32_t funcId);
+bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval);
 
 void resetResultInfo(SResultInfo *pResInfo);
 void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);

src/query/src/qUtil.c

@@ -50,9 +50,15 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
   
   // use the pointer arraylist
   pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
+  if (pWindowResInfo->pResult == NULL) {
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
   for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
     SPosInfo posInfo = {-1, -1};
-    createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize);
+    int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
   }
   
   return TSDB_CODE_SUCCESS;
@@ -76,9 +82,11 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo, int32_t numOfCols) {
     return;
   }
   
-  for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
-    SWindowResult *pResult = &pWindowResInfo->pResult[i];
-    destroyTimeWindowRes(pResult, numOfCols);
+  if (pWindowResInfo->pResult != NULL) {
+    for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
+      SWindowResult *pResult = &pWindowResInfo->pResult[i];
+      destroyTimeWindowRes(pResult, numOfCols);
+    }
   }
   
   taosHashCleanup(pWindowResInfo->hashList);

src/query/src/qast.c

@@ -962,10 +962,13 @@ static UNUSED_FUNC char* exception_strdup(const char* str) {
 
 static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
   int32_t anchor = CLEANUP_GET_ANCHOR();
+  if (CLEANUP_EXCEED_LIMIT()) {
+    THROW(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT);
+    return NULL;
+  }
 
   tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode));
   CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL);
-
   pExpr->nodeType = tbufReadUint8(br);
   
   if (pExpr->nodeType == TSQL_NODE_VALUE) {
@@ -995,7 +998,6 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
     pExpr->_node.hasPK = tbufReadUint8(br);
     pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
     pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
-    
     assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
   }
   

src/tsdb/src/tsdbRead.c

@@ -1801,7 +1801,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   }
   
   tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
-  
+
+  // todo opt perf
   size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
   for(int32_t i = 0; i < numOfCols; ++i) {
     SDataStatis* st = &pHandle->statis[i];
@@ -1820,6 +1821,13 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
     if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL
       pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows;
     }
+
+    // todo opt perf
+    SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i);
+    if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) {
+      pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst;
+      pHandle->statis[i].max = pBlockInfo->compBlock->keyLast;
+    }
   }
   
   return TSDB_CODE_SUCCESS;
@@ -2193,7 +2201,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   int32_t ret = TSDB_CODE_SUCCESS;
   tExprNode* expr = NULL;
 
-  TRY(32) {
+  TRY(TSDB_MAX_TAGS) {
     expr = exprTreeFromTableName(tbnameCond);
     if (expr == NULL) {
       expr = exprTreeFromBinary(pTagCond, len);
@@ -2217,7 +2225,8 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
   } CATCH( code ) {
     CLEANUP_EXECUTE();
-    ret = code;
+    terrno = code;
+    goto _error; 
     // TODO: more error handling
   } END_TRY
 

src/util/inc/exception.h

@@ -78,6 +78,7 @@ void cleanupPush_int_ptr        ( bool failOnly, void* func, void* arg );
 int32_t cleanupGetActionCount();
 void cleanupExecuteTo( int32_t anchor, bool failed );
 void cleanupExecute( SExceptionNode* node, bool failed );
+bool cleanupExceedLimit();
 
 #define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 )  cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) )
 #define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) )
@@ -91,7 +92,7 @@ void cleanupExecute( SExceptionNode* node, bool failed );
 
 #define CLEANUP_GET_ANCHOR()          cleanupGetActionCount()
 #define CLEANUP_EXECUTE_TO( anchor, failed )  cleanupExecuteTo( (anchor), (failed) )
-
+#define CLEANUP_EXCEED_LIMIT()        cleanupExceedLimit() 
 
 ////////////////////////////////////////////////////////////////////////////////
 // functions & macros for exception handling

src/util/inc/tfile.h

@@ -19,13 +19,14 @@
 #ifdef TAOS_RANDOM_FILE_FAIL
 
 void taosSetRandomFileFailFactor(int factor);
-ssize_t taos_tread(int fd, void *buf, size_t count);
-ssize_t taos_twrite(int fd, void *buf, size_t count);
-off_t taos_lseek(int fd, off_t offset, int whence);
+void taosSetRandomFileFailOutput(const char *path);
+ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line);
+ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line);
+off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line);
 
-#define tread(fd, buf, count)  taos_tread(fd, buf, count)
-#define twrite(fd, buf, count)  taos_twrite(fd, buf, count)
-#define lseek(fd, offset, whence)  taos_lseek(fd, offset, whence)
+#define tread(fd, buf, count)  taos_tread(fd, buf, count, __FILE__, __LINE__)
+#define twrite(fd, buf, count)  taos_twrite(fd, buf, count, __FILE__, __LINE__)
+#define lseek(fd, offset, whence)  taos_lseek(fd, offset, whence, __FILE__, __LINE__)
 
 #endif  // TAOS_RANDOM_FILE_FAIL
 

src/util/src/exception.c

@@ -147,3 +147,6 @@ void cleanupExecuteTo( int32_t anchor, bool failed ) {
 void cleanupExecute( SExceptionNode* node, bool failed ) {
     doExecuteCleanup( node, 0, failed );
 }
+bool cleanupExceedLimit() {
+  return expList->numCleanupAction >= expList->maxCleanupAction;
+}

src/util/src/tcache.c

@@ -413,57 +413,90 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   *data = NULL;
 
   // note: extend lifespan before dec ref count
-  if (pCacheObj->extendLifespan) {
+  bool inTrashCan = pNode->inTrashCan;
+
+  if (pCacheObj->extendLifespan && (!inTrashCan)) {
     atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs());
     uDebug("cache:%s data:%p extend life time to %"PRId64 "  before release", pCacheObj->name, pNode->data, pNode->expireTime);
   }
 
-  bool inTrashCan = pNode->inTrashCan;
-  uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1);
-
-  // NOTE: once refcount is decrease, pNode may be free by other thread immediately.
-  int32_t ref = T_REF_DEC(pNode);
-
-  if (inTrashCan) {
-    // Remove it if the ref count is 0.
-    // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when
-    // the node is in trashcan.
-    if (ref == 0) {
-      __cache_wr_lock(pCacheObj);
-      assert(pNode->pTNodeHeader->pData == pNode);
-      taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
-      __cache_unlock(pCacheObj);
+  if (_remove) {
+    __cache_wr_lock(pCacheObj);
+
+    // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
+    int32_t ref = T_REF_DEC(pNode);
+    uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref);
+
+    /*
+     * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users
+     * releasing this resources.
+     *
+     * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread
+     * that tries to do the same thing.
+     */
+    if (pNode->inTrashCan) {
+      if (ref == 0) {
+        assert(pNode->pTNodeHeader->pData == pNode);
+        taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
+      }
+    } else {
+      if (ref > 0) {
+        assert(pNode->pTNodeHeader == NULL);
+        taosCacheMoveToTrash(pCacheObj, pNode);
+      } else {
+        taosCacheReleaseNode(pCacheObj, pNode);
+      }
     }
 
+    __cache_unlock(pCacheObj);
+
   } else {
-    assert(pNode->pTNodeHeader == NULL);
-
-    if (_remove) { // not in trash can, but need to remove it
-      __cache_wr_lock(pCacheObj);
-
-      /*
-       * If not referenced by other users. Otherwise move this node to trashcan wait for all users
-       * releasing this resources.
-       *
-       * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread
-       * that tries to do the same thing.
-       */
+    uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, T_REF_VAL_GET(pNode) - 1);
+
+    // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
+    int32_t ref = T_REF_DEC(pNode);
+
+    if (inTrashCan) {
+      // Remove it if the ref count is 0.
+      // The ref count does not need to load and check again after lock acquired, since ref count can not be increased when
+      // the node is in trashcan.
       if (ref == 0) {
-        if (T_REF_VAL_GET(pNode) == 0) {
-          taosCacheReleaseNode(pCacheObj, pNode);
-        } else {
-          taosCacheMoveToTrash(pCacheObj, pNode);
-        }
+        __cache_wr_lock(pCacheObj);
+        assert(pNode->pTNodeHeader->pData == pNode);
+        taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
+        __cache_unlock(pCacheObj);
       }
 
-      __cache_unlock(pCacheObj);
-//    } else { // extend its life time
-//      if (pCacheObj->extendLifespan) {
-//        atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs());
-//        uDebug("cache:%s data:%p extend life time to %"PRId64 " after release", pCacheObj->name, pNode->data, pNode->expireTime);
-//      }
     }
   }
+
+//   else {
+//    if (_remove) { // not in trash can, but need to remove it
+//      __cache_wr_lock(pCacheObj);
+//
+//      /*
+//       * If not referenced by other users. Otherwise move this node to trashcan wait for all users
+//       * releasing this resources.
+//       *
+//       * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread
+//       * that tries to do the same thing.
+//       */
+//      if (ref == 0) {
+//        if (T_REF_VAL_GET(pNode) == 0) {
+//          taosCacheReleaseNode(pCacheObj, pNode);
+//        } else {
+//          taosCacheMoveToTrash(pCacheObj, pNode);
+//        }
+//      } else if (ref > 0) {
+//        if (!pNode->inTrashCan) {
+//          assert(pNode->pTNodeHeader == NULL);
+//          taosCacheMoveToTrash(pCacheObj, pNode);
+//        }
+//      }
+//
+//      __cache_unlock(pCacheObj);
+//    }
+//  }
 }
 
 void taosCacheEmpty(SCacheObj *pCacheObj) {

src/util/src/tfile.c

@@ -29,14 +29,55 @@
 #ifdef TAOS_RANDOM_FILE_FAIL
 
 static int random_file_fail_factor = 20;
+static FILE *fpRandomFileFailOutput = NULL;
 
 void taosSetRandomFileFailFactor(int factor)
 {
   random_file_fail_factor = factor;
 }
+
+static void close_random_file_fail_output()
+{
+  if (fpRandomFileFailOutput != NULL) {
+    if (fpRandomFileFailOutput != stdout) {
+      fclose(fpRandomFileFailOutput);
+    }
+    fpRandomFileFailOutput = NULL;
+  }
+}
+
+static void random_file_fail_output_sig(int sig)
+{
+  fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
+
+  struct sigaction act = {0};
+  act.sa_handler = SIG_DFL;
+  sigaction(sig, &act, NULL);
+
+  close_random_file_fail_output();
+  exit(EXIT_FAILURE);
+}
+
+void taosSetRandomFileFailOutput(const char *path)
+{
+  if (path == NULL) {
+    fpRandomFileFailOutput = stdout;
+  } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) {
+    atexit(close_random_file_fail_output);
+  } else {
+    printf("failed to open random file fail log file '%s', errno=%d\n", path, errno);
+    return;
+  }
+
+  struct sigaction act = {0};
+  act.sa_handler = random_file_fail_output_sig;
+  sigaction(SIGFPE, &act, NULL);
+  sigaction(SIGSEGV, &act, NULL);
+  sigaction(SIGILL, &act, NULL);
+}
 #endif
 
-ssize_t taos_tread(int fd, void *buf, size_t count)
+ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
   if (random_file_fail_factor > 0) {
@@ -49,7 +90,7 @@ ssize_t taos_tread(int fd, void *buf, size_t count)
   return tread(fd, buf, count);
 }
 
-ssize_t taos_twrite(int fd, void *buf, size_t count)
+ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
   if (random_file_fail_factor > 0) {
@@ -62,7 +103,7 @@ ssize_t taos_twrite(int fd, void *buf, size_t count)
   return twrite(fd, buf, count);
 }
 
-off_t taos_lseek(int fd, off_t offset, int whence)
+off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
   if (random_file_fail_factor > 0) {

src/vnode/src/vnodeMain.c

@@ -59,7 +59,7 @@ int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
-  tsDnodeVnodesHash = taosHashInit(TSDB_MAX_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
+  tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
   if (tsDnodeVnodesHash == NULL) {
     vError("failed to init vnode list");
     return TSDB_CODE_VND_OUT_OF_MEMORY;

src/vnode/src/vnodeRead.c

@@ -110,6 +110,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
       if (handle == NULL) {  // failed to register qhandle
         pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE;
         qDestroyQueryInfo(pQInfo);  // destroy it directly
+        vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void*) pQInfo, tstrerror(pRsp->code));
       } else {
         assert(*handle == pQInfo);
         pRsp->qhandle = htobe64((uint64_t) pQInfo);
@@ -125,12 +126,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     } else {
       assert(pQInfo == NULL);
     }
+
     if (handle != NULL) {
+      vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle);
+
       dnodePutItemIntoReadQueue(pVnode, *handle);
       qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
     }
 
-    vDebug("vgId:%d, QInfo:%p, dnode query msg disposed", vgId, pQInfo);
   } else {
     assert(pCont != NULL);
     handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont);
@@ -138,12 +141,13 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
       vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle);
       code = TSDB_CODE_QRY_INVALID_QHANDLE;
     } else {
-      vDebug("vgId:%d, QInfo:%p, dnode query msg in progress", pVnode->vgId, (void*) pCont);
+      vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont);
       code = TSDB_CODE_VND_ACTION_IN_PROGRESS;
       qTableQuery(*handle); // do execute query
     }
     qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
   }
+
   return code;
 }
 
@@ -155,7 +159,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
   pRetrieve->free = htons(pRetrieve->free);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, *(void**) pRetrieve->qhandle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle);
 
   memset(pRet, 0, sizeof(SRspRet));
 

tests/CMakeLists.txt

@@ -12,3 +12,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON)
 ADD_SUBDIRECTORY(examples/c)
 ADD_SUBDIRECTORY(tsim)
 ADD_SUBDIRECTORY(test/c)
+ADD_SUBDIRECTORY(comparisonTest/tdengine)

tests/comparisonTest/tdengine/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  add_executable(tdengineTest tdengineTest.c)
+  target_link_libraries(tdengineTest taos_static tutil common pthread)
+ENDIF()

tests/comparisonTest/tdengine/tdengineTest.c

@@ -7,6 +7,7 @@
 #include <time.h>
 #include <pthread.h>
 #include <sys/time.h>
+#include <inttypes.h>
 
 typedef struct {
   char sql[256];
@@ -123,19 +124,21 @@ void writeDataImp(void *param) {
   if (taos == NULL)
     taos_error(taos);
 
-  int code = taos_query(taos, "use db");
+  TAOS_RES* result = taos_query(taos, "use db");
+  int32_t code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
-  char sql[65000];
+  char *sql = calloc(1, 8*1024*1024);
   int sqlLen = 0;
   int lastMachineid = 0;
   int counter = 0;
   int totalRecords = 0;
 
   for (int j = pThread->sID; j <= pThread->eID; j++) {
-    char fileName[256];
+    char fileName[300];
     sprintf(fileName, "%s/testdata%d.csv", arguments.dataDir, j);
 
     FILE *fp = fopen(fileName, "r");
@@ -162,7 +165,7 @@ void writeDataImp(void *param) {
       int64_t timestamp;
       int temperature;
       float humidity;
-      sscanf(line, "%d%s%d%lld%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
+      sscanf(line, "%d%s%d%" PRId64 "%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
 
       if (counter == 0) {
         sqlLen = sprintf(sql, "insert into");
@@ -174,14 +177,16 @@ void writeDataImp(void *param) {
                           machineid, machineid, machinename, machinegroup);
       }
 
-      sqlLen += sprintf(sql + sqlLen, "(%lld,%d,%f)", timestamp, temperature, humidity);
+      sqlLen += sprintf(sql + sqlLen, "(%" PRId64 ",%d,%f)", timestamp, temperature, humidity);
       counter++;
 
       if (counter >= arguments.rowsPerRequest) {
-        int code = taos_query(taos, sql);
+        TAOS_RES *result = taos_query(taos, sql);
+        int32_t   code = taos_errno(result);
         if (code != 0) {
-          printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
+          printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
         }
+        taos_free_result(result);
 
         totalRecords += counter;
         counter = 0;
@@ -194,15 +199,18 @@ void writeDataImp(void *param) {
   }
 
   if (counter > 0) {
-    int code = taos_query(taos, sql);
+    TAOS_RES *result = taos_query(taos, sql);
+    int32_t   code = taos_errno(result);
     if (code != 0) {
-      printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
+      printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
     }
+    taos_free_result(result);
 
     totalRecords += counter;
   }
 
   __sync_fetch_and_add(&statis.totalRows, totalRecords);
+  free(sql);
 }
 
 void writeData() {
@@ -215,19 +223,23 @@ void writeData() {
   taos_init();
 
   void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
-  if (taos == NULL)
-    taos_error(taos);
+  if (taos == NULL) taos_error(taos);
 
-  int code = taos_query(taos, "create database if not exists db");
+  TAOS_RES *result = taos_query(taos, "create database if not exists db");
+  int32_t   code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
-  code = taos_query(taos, "create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
-    "tags(devid int, devname binary(16), devgroup int)");
+  result = taos_query(taos,
+                      "create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
+                      "tags(devid int, devname binary(16), devgroup int)");
+  code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
   int64_t st = getTimeStampMs();
 
@@ -292,17 +304,12 @@ void readData() {
 
     int64_t st = getTimeStampMs();
 
-    int code = taos_query(taos, line);
+    TAOS_RES *result = taos_query(taos, line);
+    int32_t   code = taos_errno(result);
     if (code != 0) {
       taos_error(taos);
     }
 
-    void *result = taos_use_result(taos);
-    if (result == NULL) {
-      printf("failed to get result, reason:%s\n", taos_errstr(taos));
-      exit(1);
-    }
-
     TAOS_ROW row;
     int rows = 0;
     //int num_fields = taos_field_count(taos);

tests/pytest/util/dnodes.py

@@ -28,7 +28,9 @@ class TDSimClient:
             "locale": "en_US.UTF-8",
             "charset": "UTF-8",
             "asyncLog": "0",
-            "maxTablesPerVnode": "4",
+            "minTablesPerVnode": "4",
+            "maxTablesPerVnode": "1000",
+            "tableIncStepPerVnode": "10000",
             "maxVgroupsPerDb": "1000",
             "sdbDebugFlag": "143",
             "rpcDebugFlag": "135",

tests/script/sh/deploy.sh

@@ -139,7 +139,9 @@ echo "clog                2"              >> $TAOS_CFG
 echo "statusInterval      1"              >> $TAOS_CFG
 echo "numOfTotalVnodes    4"              >> $TAOS_CFG
 echo "maxVgroupsPerDb     4"              >> $TAOS_CFG
+echo "minTablesPerVnode   4"              >> $TAOS_CFG
 echo "maxTablesPerVnode   1000"           >> $TAOS_CFG
+echo "tableIncStepPerVnode 10000"         >> $TAOS_CFG
 echo "asyncLog            0"              >> $TAOS_CFG
 echo "numOfMnodes         1"              >> $TAOS_CFG
 echo "locale    en_US.UTF-8"              >> $TAOS_CFG

tests/script/tmp/prepare.sim

@@ -25,10 +25,15 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20
 
-system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 2
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
 
 system sh/cfg.sh -n dnode1 -c http -v 1
 system sh/cfg.sh -n dnode2 -c http -v 1

tests/script/unique/cluster/client3.sim

+$tblStart = 0
+$tblEnd = 10000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+loop_lable:
+print ====================== client3 start loop: dynamic create table and insert data, select, drop table ............
+
+$totalRows = 0
+
+#sql create table $stb ( ts timestamp, c1 int) tags ( t1 int, t2 binary(16) )
+$tagBinary = ' . client3
+$tagBinary = $tagBinary . '
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb using $stb tags ( $i , $tagBinary ) values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x  
+#  print ====== app insert totalRows: $totalRows  
+  $i = $i + 1
+endw 
+   
+sql select count(*) from $stb where t2 == $tagBinary
+if $data00 != $totalRows then
+  print data00 $data00 totalRows $totalRows
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client3   insert loss: $deltaRows *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+endi
+
+print ====================== client3 drop table
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  sql drop table if exists $tb 
+  $i = $i + 1
+endw 
+goto loop_lable
\ No newline at end of file

tests/script/unique/cluster/client4.sim

+$tblStart = 0
+$tblEnd = 2000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+loop_lable:
+print ====================== client4 start loop: create db2, tables and insert data, select, drop db2 ............
+
+$db = db2
+$stb = stb2
+
+print create database if not exists $db replica 2
+sql create database if not exists $db replica 2
+sql use $db
+
+print ==== client4start create table
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  sql create table $tb (ts timestamp, c1 int)
+  $i = $i + 1
+endw 
+
+
+print ==== client4start insert, include multi table data in one insert sql
+$totalRows = 0
+$totalRowsPerTbl = 0
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb0 = dtb . $i
+  $i = $i + 1
+  $tb1 = dtb . $i
+  $i = $i + 1
+  $tb2 = dtb . $i
+  $i = $i + 1
+  $tb3 = dtb . $i
+  $i = $i + 1
+  $tb4 = dtb . $i  
+  $i = $i + 1  
+  
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  
+  $totalRowsPerTbl = $totalRowsPerTbl + $x
+  $x = $x * 5
+  $totalRows = $totalRows + $x  
+endw 
+   
+sql select count(*) from tb10
+if $data00 != $totalRowsPerTbl then
+  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client4   insert loss: $deltaRows *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+endi
+
+
+print ====client4 start alter table
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  sql alter table $tb add c2 float
+  $i = $i + 1
+endw 
+
+print ====client4 continue insert, include multi table data in one insert sql
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb0 = dtb . $i
+  $i = $i + 1
+  $tb1 = dtb . $i
+  $i = $i + 1
+  $tb2 = dtb . $i
+  $i = $i + 1
+  $tb3 = dtb . $i
+  $i = $i + 1
+  $tb4 = dtb . $i  
+  $i = $i + 1  
+  
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  
+  $totalRowsPerTbl = $totalRowsPerTbl + $x
+  $x = $x * 5
+  $totalRows = $totalRows + $x  
+endw 
+   
+sql select count(*) from tb10
+if $data00 != $totalRowsPerTbl then
+  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client4   insert loss: $deltaRows *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+endi
+
+print ====================== client4 drop database
+sql drop if exists database $db 
+goto loop_lable
\ No newline at end of file

tests/script/unique/cluster/client5.sim

+$tblStart = 0
+$tblEnd = 2000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+loop_lable:
+print ====================== client5 start loop query
+
+$db = db2
+$stb = stb2
+
+print create database if not exists $db replica 2
+sql create database if not exists $db replica 2
+sql use $db
+
+print ==== client4start create table
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  sql create table $tb (ts timestamp, c1 int)
+  $i = $i + 1
+endw 
+
+
+print ==== client4start insert, include multi table data in one insert sql
+$totalRows = 0
+$totalRowsPerTbl = 0
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb0 = dtb . $i
+  $i = $i + 1
+  $tb1 = dtb . $i
+  $i = $i + 1
+  $tb2 = dtb . $i
+  $i = $i + 1
+  $tb3 = dtb . $i
+  $i = $i + 1
+  $tb4 = dtb . $i  
+  $i = $i + 1  
+  
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  
+  $totalRowsPerTbl = $totalRowsPerTbl + $x
+  $x = $x * 5
+  $totalRows = $totalRows + $x  
+endw 
+   
+sql select count(*) from tb10
+if $data00 != $totalRowsPerTbl then
+  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client4   insert loss: $deltaRows *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+endi
+
+
+print ====client4 start alter table
+$i = $tblStart
+while $i < $tblEnd
+  $tb = dtb . $i
+  sql alter table $tb add c2 float
+  $i = $i + 1
+endw 
+
+print ====client4 continue insert, include multi table data in one insert sql
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb0 = dtb . $i
+  $i = $i + 1
+  $tb1 = dtb . $i
+  $i = $i + 1
+  $tb2 = dtb . $i
+  $i = $i + 1
+  $tb3 = dtb . $i
+  $i = $i + 1
+  $tb4 = dtb . $i  
+  $i = $i + 1  
+  
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb0 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb1 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb2 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb3 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) $tb4 values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) 
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+  
+  $totalRowsPerTbl = $totalRowsPerTbl + $x
+  $x = $x * 5
+  $totalRows = $totalRows + $x  
+endw 
+   
+sql select count(*) from tb10
+if $data00 != $totalRowsPerTbl then
+  print data00 $data00 totalRowsPerTbl $totalRowsPerTbl
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  print ************ client4   insert loss: $deltaRows *****
+  print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+endi
+
+print ====================== client4 drop database
+sql drop if exists database $db 
+goto loop_lable
\ No newline at end of file

tests/script/unique/cluster/cluster_main.sim

@@ -71,7 +71,7 @@ print ============== step3: create stable stb1
 $stb = stb1
 sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
 
-print ============== step4: start 10 client1/ 10 client2/ 10 client3/ 10 client4/ 1 client5
+print ============== step4: start back client1_0.sim
 run_back unique/cluster/client1_0.sim
 #run_back unique/cluster/client1_1.sim
 #run_back unique/cluster/client1_2.sim
@@ -106,26 +106,26 @@ endi
 print wait for a while to let clients start insert data
 sleep 5000    
 
+print ============== step4-1: add dnode4/dnode5 into cluster
+sql create dnode $hostname4
+sql create dnode $hostname5
+
 $loop_cnt = 0
 loop_cluster_do:
 print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
-print ============== step5: start dnode4/dnode5 and add into cluster, then wait ready
+print ============== step5: start dnode4/dnode5
 system sh/exec.sh -n dnode4 -s start
 system sh/exec.sh -n dnode5 -s start
-sql create dnode $hostname4
-sql create dnode $hostname5
-
-sleep 5000
-
+sleep 20000
 
-print ============== step6: stop and drop dnode1, then remove data dir of dnode1 
+print ============== step6: stop dnode1
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
-sql drop dnode $hostname1
-sleep 5000
+sleep 10000
+#sql drop dnode $hostname1
+#sleep 5000
 
-system rm -rf ../../../sim/dnode1/data
-sleep 20000
+#system rm -rf ../../../sim/dnode1/data
+#sleep 20000
 
 sql show mnodes
 print show mnodes
@@ -139,7 +139,22 @@ print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
 print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
 print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
-return -1
+
+print ============== step6-1: restart dnode1
+system sh/exec.sh -n dnode1 -s start
+sleep 10000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
 
 print ============== step7: stop dnode2 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
@@ -236,14 +251,17 @@ if $data04 != 2 then
   return -1
 endi
 
-print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 
+print ============== step14: stop dnode4/dnode5
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 system sh/exec.sh -n dnode5 -s stop -x SIGINT
 sleep 20000
-sql drop dnode $hostname4
-sql drop dnode $hostname5
-system rm -rf ../../../sim/dnode4/data
-system rm -rf ../../../sim/dnode5/data
+#system sh/exec.sh -n dnode4 -s start
+#system sh/exec.sh -n dnode5 -s start
+#sleep 10000
+#sql drop dnode $hostname4
+#sql drop dnode $hostname5
+#system rm -rf ../../../sim/dnode4/data
+#system rm -rf ../../../sim/dnode5/data
 
 print ============== step15: alter replica 1
 sql alter database $db replica 1
@@ -263,13 +281,13 @@ if $data04 != 2 then
   return -1
 endi
 
-print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready
-system sh/cfg.sh -n dnode1 -c first -v $hostname2
-system sh/cfg.sh -n dnode1 -c second -v $hostname3
-
-system sh/exec.sh -n dnode1 -s start
-sql create dnode $hostname1
-sleep 20000
+#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready
+#system sh/cfg.sh -n dnode1 -c first -v $hostname2
+#system sh/cfg.sh -n dnode1 -c second -v $hostname3
+#
+#system sh/exec.sh -n dnode1 -s start
+#sql create dnode $hostname1
+#sleep 20000
 
 print ============== step18: alter replica 3
 sql alter database $db replica 3

tests/script/unique/cluster/cluster_main0.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
+system sh/cfg.sh -n dnode5 -c walLevel -v 1
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode5 -c alternativeRole -v 0
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1/dnode2/dnode3
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep 3000
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+print ============== step2: create db1 with replica 3
+$db = db1
+print create database $db replica 3
+#sql create database $db replica 3 maxTables $totalTableNum
+sql create database $db replica 3
+sql use $db
+
+print ============== step3: create stable stb1
+$stb = stb1
+print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+
+print ============== step4: start others client
+run_back unique/cluster/client1_0.sim
+run_back unique/cluster/client1_1.sim
+run_back unique/cluster/client1_2.sim
+run_back unique/cluster/client1_3.sim
+#run_back unique/cluster/client2_0.sim
+#run_back unique/cluster/client2_1.sim
+#run_back unique/cluster/client2_2.sim
+#run_back unique/cluster/client2_3.sim
+#run_back unique/cluster/client3.sim
+#run_back unique/cluster/client4.sim
+
+sleep 20000
+
+wait_subsim_insert_complete_create_tables:
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_complete_create_tables
+endi
+
+wait_subsim_insert_data:
+print select count(*) from $stb
+sql select count(*) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_data
+endi
+
+print wait for a while to let clients start insert data
+sleep 5000    
+
+$loop_cnt = 0
+loop_cluster_do:
+print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
+print ============== step5: start dnode4/dnode5 and add into cluster, then wait ready
+system sh/exec.sh -n dnode4 -s start
+system sh/exec.sh -n dnode5 -s start
+sql create dnode $hostname4
+sql create dnode $hostname5
+
+sleep 5000
+
+
+print ============== step6: stop dnode1
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 10000
+#sql drop dnode $hostname1
+#sleep 5000
+#system rm -rf ../../../sim/dnode1/data
+#sleep 20000
+print ============== step6-1: restart dnode1
+system sh/exec.sh -n dnode1 -s start
+sleep 10000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step7: stop dnode2 
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep 5000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step8: restart dnode2, then wait sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step9: stop dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step10: restart dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step11: stop dnode4, then wait sync end
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep 20000
+
+print ============== step12: restart dnode4, then wait sync end
+system sh/exec.sh -n dnode4 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step13: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+
+if $data04 != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
+sleep 3000
+sql drop dnode $hostname4
+sql drop dnode $hostname5
+system rm -rf ../../../sim/dnode4/data
+system rm -rf ../../../sim/dnode5/data
+sleep 20000
+
+print ============== step15: alter replica 1
+sql alter database $db replica 1
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09
+if $data04 != 1 then
+  print rplica is not modify to 1, error!!!!!!
+  return -1
+endi
+
+print ============== step16: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09
+if $data04 != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready
+#system sh/cfg.sh -n dnode1 -c first -v $hostname2
+#system sh/cfg.sh -n dnode1 -c second -v $hostname3
+#
+#system sh/exec.sh -n dnode1 -s start
+#sql create dnode $hostname1
+#sleep 20000
+
+print ============== step18: alter replica 3
+sql alter database $db replica 3
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09
+if $data04 != 3 then
+  print rplica is not modify to 3, error!!!!!!
+  return -1
+endi
+
+print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** ****
+$loop_cnt = $loop_cnt + 1
+goto loop_cluster_do

tests/script/unique/cluster/cluster_main1.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
+system sh/cfg.sh -n dnode5 -c walLevel -v 1
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode5 -c alternativeRole -v 0
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1/dnode2/dnode3
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep 3000
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+print ============== step2: create db1 with replica 3
+$replica = 3
+$db = db1
+print create database $db replica $replica
+#sql create database $db replica 3 maxTables $totalTableNum
+sql create database $db replica $replica
+sql use $db
+
+print ============== step3: create stable stb1
+$stb = stb1
+print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+
+print ============== step4: start others client
+run_back unique/cluster/main1_client1_0.sim
+run_back unique/cluster/main1_client1_1.sim
+run_back unique/cluster/main1_client1_2.sim
+run_back unique/cluster/main1_client1_3.sim
+run_back unique/cluster/client3.sim
+run_back unique/cluster/client4.sim
+
+sleep 20000
+
+wait_subsim_insert_complete_create_tables:
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_complete_create_tables
+endi
+
+wait_subsim_insert_data:
+print select count(*) from $stb
+sql select count(*) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_data
+endi
+
+print wait for a while to let clients start insert data
+sleep 5000    
+
+print ============== step4-1: add dnode4/dnode5 into cluster
+sql create dnode $hostname4
+sql create dnode $hostname5
+sleep 5000
+
+$loop_cnt = 0
+loop_cluster_do:
+print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
+print ============== step5: start dnode4/dnode5
+system sh/exec.sh -n dnode4 -s start
+system sh/exec.sh -n dnode5 -s start
+sleep 20000
+
+print ============== step6: stop dnode1
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 10000
+#sql drop dnode $hostname1
+#sleep 5000
+#system rm -rf ../../../sim/dnode1/data
+#sleep 20000
+print ============== step6-1: restart dnode1
+system sh/exec.sh -n dnode1 -s start
+sleep 10000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step7: stop dnode2 
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep 5000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step8: restart dnode2, then wait sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step9: stop dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step10: restart dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step11: stop dnode4, then wait sync end
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep 20000
+
+#print ============== step12: restart dnode4, then wait sync end
+#system sh/exec.sh -n dnode4 -s start
+#sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step13: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+print ============== step14: stop dnode5
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
+sleep 20000
+#system rm -rf ../../../sim/dnode5/data
+
+#print ============== step14-1: drop dnode5, then remove data dir
+#sql drop dnode $hostname5
+#sleep 20000
+#system rm -rf ../../../sim/dnode5/data
+
+print ============== step15: alter replica 1
+sql alter database $db replica 1
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 1 then
+  print rplica is not modify to 1, error!!!!!!
+  return -1
+endi
+
+print ============== step16: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready
+#system sh/cfg.sh -n dnode1 -c first -v $hostname2
+#system sh/cfg.sh -n dnode1 -c second -v $hostname3
+#
+#system sh/exec.sh -n dnode1 -s start
+#sql create dnode $hostname1
+#sleep 20000
+
+print ============== step18: alter replica 3
+sql alter database $db replica 3
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 3 then
+  print rplica is not modify to 3, error!!!!!!
+  return -1
+endi
+
+print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** ****
+$loop_cnt = $loop_cnt + 1
+goto loop_cluster_do

tests/script/unique/cluster/cluster_main2.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode2 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode4 -c walLevel -v 1
+system sh/cfg.sh -n dnode5 -c walLevel -v 1
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256
+system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 256
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode5 -c alternativeRole -v 0
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 5000
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1/dnode2/dnode3
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sleep 3000
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+print ============== step2: create db1 with replica 3
+$replica = 3
+$db = db1
+print create database $db replica $replica
+#sql create database $db replica 3 maxTables $totalTableNum
+sql create database $db replica $replica
+sql use $db
+
+print ============== step3: create stable stb1
+$stb = stb1
+print create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(16))
+
+print ============== step4: start others client
+run_back unique/cluster/main2_client1_0.sim
+run_back unique/cluster/main2_client1_1.sim
+run_back unique/cluster/main2_client1_2.sim
+run_back unique/cluster/main2_client1_3.sim
+run_back unique/cluster/main2_client2_0.sim
+run_back unique/cluster/main2_client2_1.sim
+run_back unique/cluster/main2_client2_2.sim
+run_back unique/cluster/main2_client2_3.sim
+run_back unique/cluster/client3.sim
+run_back unique/cluster/client4.sim
+
+sleep 20000
+
+wait_subsim_insert_complete_create_tables:
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_complete_create_tables
+endi
+
+wait_subsim_insert_data:
+print select count(*) from $stb
+sql select count(*) from $stb
+print data00 $data00
+if $data00 < 1000 then
+  sleep 3000
+  goto wait_subsim_insert_data
+endi
+
+print wait for a while to let clients start insert data
+sleep 5000    
+
+print ============== step4-1: add dnode4/dnode5 into cluster
+sql create dnode $hostname4
+sql create dnode $hostname5
+sleep 5000
+
+
+$loop_cnt = 0
+loop_cluster_do:
+print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** ****
+print ============== step5: start dnode4/dnode5
+system sh/exec.sh -n dnode4 -s start
+system sh/exec.sh -n dnode5 -s start
+
+print ============== step6: stop dnode1
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 10000
+#sql drop dnode $hostname1
+#sleep 5000
+#system rm -rf ../../../sim/dnode1/data
+#sleep 20000
+print ============== step6-1: restart dnode1
+system sh/exec.sh -n dnode1 -s start
+sleep 10000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step7: stop dnode2 
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep 5000
+
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step8: restart dnode2, then wait sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step9: stop dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step10: restart dnode3, then wait sync end
+system sh/exec.sh -n dnode3 -s start
+sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step11: stop dnode4, then wait sync end
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+sleep 20000
+
+#print ============== step12: restart dnode4, then wait sync end
+#system sh/exec.sh -n dnode4 -s start
+#sleep 20000
+sql show mnodes
+print show mnodes
+print rows: $rows
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
+print $data0_8  $data1_8  $data2_8  $data3_8  $data4_8
+print $data0_9  $data1_9  $data2_9  $data3_9  $data4_9
+
+print ============== step13: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+print ============== step14: drop dnode5, then remove data dir
+system sh/exec.sh -n dnode5 -s stop -x SIGINT
+sleep 20000
+#system rm -rf ../../../sim/dnode5/data
+
+#print ============== step14-1: drop dnode5, then remove data dir
+#sql drop dnode $hostname5
+#sleep 20000
+#system rm -rf ../../../sim/dnode5/data
+
+print ============== step15: alter replica 1
+sql alter database $db replica 1
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 1 then
+  print rplica is not modify to 1, error!!!!!!
+  return -1
+endi
+
+print ============== step16: alter replica 2
+sql alter database $db replica 2
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 2 then
+  print rplica is not modify to 2, error!!!!!!
+  return -1
+endi
+
+#print ============== step17: start dnode1 and add into cluster, then wait dnode1 ready
+#system sh/cfg.sh -n dnode1 -c first -v $hostname2
+#system sh/cfg.sh -n dnode1 -c second -v $hostname3
+#
+#system sh/exec.sh -n dnode1 -s start
+#sql create dnode $hostname1
+#sleep 20000
+
+print ============== step18: alter replica 3
+sql alter database $db replica 3
+sql show databases
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07   $data08  $data09 
+print $data10  $data11  $data12  $data13  $data14  $data15  $data16  $data17   $data18  $data19 
+
+if $data00 == db1 then
+  $replica = $data04
+elif $data10 == db1 then
+  $replica = $data14
+else then
+  print ==== db1 already not exists!!!!!
+  return -1
+endi
+
+if $replica != 3 then
+  print rplica is not modify to 3, error!!!!!!
+  return -1
+endi
+
+print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** ****
+$loop_cnt = $loop_cnt + 1
+goto loop_cluster_do

tests/script/unique/cluster/main1_client1_0.sim

+#system sh/stop_dnodes.sh
+#system sh/deploy.sh -n dnode1 -i 1
+#system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 10000
+#system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+#system sh/exec.sh -n dnode1 -s start
+#sql connect
+#$db = db1
+#sql create database $db
+#sql use $db
+#$stb = stb1
+#sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+
+
+$tblStart = 0
+$tblEnd = 1000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_0
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_0 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop
+    $x = $x + 20
+    $ts = $ts + 40a
+    goto continue_next_0
+    insert_error_loop:
+    print ============== main1_client1_0 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ...
+    continue_next_0:
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    if $data00 != $totalRows then
+      print data00 $data00 totalRows $totalRows
+      $deltaRows = $totalRows - $data00
+      if $lastLossRows == 0 then 
+        $lastLossRows = $deltaRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_0 insert loss: $deltaRows *****
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      elif $deltaRows != $lastLossRows
+        $tmp = $deltaRows - $lastLossRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_0 insert loss: $tmp ***********
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        $lastLossRows = $deltaRows
+      endi
+#	    return -1
+    endi
+    goto continue_next_1
+    query_error_loop:
+    print ============== main1_client1_0 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    continue_next_1:
+    print ====================== client1_0  insert data complete once ............
+  endi
+endw 
+print ====================== client1_0  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main1_client1_1.sim

+$tblStart = 1000
+$tblEnd = 2000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_1
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_1 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop
+    $x = $x + 20
+    $ts = $ts + 40a
+    goto continue_next_0
+    insert_error_loop:
+    print ============== main1_client1_1 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ...
+    continue_next_0:
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    if $data00 != $totalRows then
+      print data00 $data00 totalRows $totalRows
+      $deltaRows = $totalRows - $data00
+      if $lastLossRows == 0 then 
+        $lastLossRows = $deltaRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_1 insert loss: $deltaRows *****
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      elif $deltaRows != $lastLossRows
+        $tmp = $deltaRows - $lastLossRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_1 insert loss: $tmp ***********
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        $lastLossRows = $deltaRows
+      endi
+#	    return -1
+    endi
+    goto continue_next_1
+    query_error_loop:
+    print ============== main1_client1_1 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    continue_next_1:
+    print ====================== client1_2  insert data complete once ............
+  endi
+endw 
+print ====================== client1_1  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main1_client1_2.sim

+$tblStart = 2000
+$tblEnd = 3000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_2
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_2 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop
+    $x = $x + 20
+    $ts = $ts + 40a
+    goto continue_next_0
+    insert_error_loop:
+    print ============== main1_client1_2 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ...
+    continue_next_0:
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    if $data00 != $totalRows then
+      print data00 $data00 totalRows $totalRows
+      $deltaRows = $totalRows - $data00
+      if $lastLossRows == 0 then 
+        $lastLossRows = $deltaRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_2 insert loss: $deltaRows *****
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      elif $deltaRows != $lastLossRows
+        $tmp = $deltaRows - $lastLossRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_2 insert loss: $tmp ***********
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        $lastLossRows = $deltaRows
+      endi
+#	    return -1
+    endi
+    goto continue_next_1
+    query_error_loop:
+    print ============== main1_client1_2 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    continue_next_1:
+    print ====================== client1_2  insert data complete once ............
+  endi
+endw 
+print ====================== client1_2  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main1_client1_3.sim

+$tblStart = 3000
+$tblEnd = 4000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_3
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_3 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x ) -x insert_error_loop
+    $x = $x + 20
+    $ts = $ts + 40a
+    goto continue_next_0
+    insert_error_loop:
+    print ============== main1_client1_3 run error: sql insert into $tb values ( $ts + 0a , $x ) ... ...
+    continue_next_0:
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    if $data00 != $totalRows then
+      print data00 $data00 totalRows $totalRows
+      $deltaRows = $totalRows - $data00
+      if $lastLossRows == 0 then 
+        $lastLossRows = $deltaRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_3 insert loss: $deltaRows *****
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+      elif $deltaRows != $lastLossRows
+        $tmp = $deltaRows - $lastLossRows
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        print ************ client1_3 insert loss: $tmp ***********
+        print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+        $lastLossRows = $deltaRows
+      endi
+#	    return -1
+    endi
+    goto continue_next_1
+    query_error_loop:
+    print ============== main1_client1_3 run error: sql select count(*) from $stb where t2 == $tagBinary -x query_error_loop
+    continue_next_1:
+    print ====================== client1_3  insert data complete once ............
+  endi
+endw 
+print ====================== client1_3  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main1_client4.sim

+$tblStart = 0
+$tblEnd = 10000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db2
+$stb = stb2
+
+loop_lable:
+print ========= start loop create db, table, inset data, alter column/tag, select, drop db
+
+sql create database if not exists $db
+sql use $db
+sql create table if not exists $stb ( ts timestamp, c1 int, c2 float ) tags ( t1 int , t2 binary(16) )
+$tagBinary = ' . client4
+$tagBinary = $tagBinary . '
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  sql create table if not exists  $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb -x continue_loop
+    print data00 $data00 totalRows $totalRows
+    if $data00 != $totalRows then
+	    print **********************  select error **********************
+    endi
+    continue_loop:
+    print ====================== client4  insert data complete once ............
+  endi
+endw 
+
+##################### alter column
+sql alter table $stb add column c3 double
+sql alter table $stb drop column c2
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb -x continue_loop
+    print data00 $data00 totalRows $totalRows
+    if $data00 != $totalRows then
+	    print **********************  select error **********************
+    endi
+    continue_loop:
+    print ====================== client4  insert data complete once ............
+  endi
+endw 
+
+
+##################### alter tag
+sql alter table $stb add tag t3 int
+sql alter table $stb drop tag t2
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = ttb . $i
+  sql create table if not exists  $tb using $stb tags ($i, $i)
+  $i = $i + 1
+endw 
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x , $x ) ( $ts + 2a , $x , $x ) ( $ts + 4a , $x , $x ) ( $ts + 6a , $x , $x ) ( $ts + 8a , $x , $x ) ( $ts + 10a , $x , $x ) ( $ts + 12a , $x , $x ) ( $ts + 14a , $x , $x ) ( $ts + 16a , $x , $x ) ( $ts + 18a , $x , $x ) ( $ts + 20a , $x , $x ) ( $ts + 22a , $x , $x ) ( $ts + 24a , $x , $x ) ( $ts + 26a , $x , $x ) ( $ts + 28a , $x , $x ) ( $ts + 30a , $x , $x ) ( $ts + 32a , $x , $x ) ( $ts + 34a , $x , $x ) ( $ts + 36a , $x , $x ) ( $ts + 38a , $x , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb -x continue_loop
+    print data00 $data00 totalRows $totalRows
+    if $data00 != $totalRows then
+	    print **********************  select error **********************
+    endi
+    continue_loop:
+    print ====================== client4  insert data complete once ............
+  endi
+endw 
+
+goto loop_lable
\ No newline at end of file

tests/script/unique/cluster/main2_client1_0.sim

+#system sh/stop_dnodes.sh
+#system sh/deploy.sh -n dnode1 -i 1
+#system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 10000
+#system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256
+#system sh/exec.sh -n dnode1 -s start
+#sql connect
+#$db = db1
+#sql create database $db
+#sql use $db
+#$stb = stb1
+#sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+
+
+$tblStart = 0
+$tblEnd = 1000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_0
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_0 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client1_0  insert data complete once ............
+  endi
+endw 
+print ====================== client1_0  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client1_1.sim

+$tblStart = 1000
+$tblEnd = 2000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_1
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_1 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client1_1  insert data complete once ............
+  endi
+endw 
+print ====================== client1_1  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client1_2.sim

+$tblStart = 2000
+$tblEnd = 3000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_2
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_2 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client1_2  insert data complete once ............
+  endi
+endw 
+print ====================== client1_2  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client1_3.sim

+$tblStart = 3000
+$tblEnd = 4000
+$tsStart = 1325347200000  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client1_3
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client1_3 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client1_3  insert data complete once ............
+  endi
+endw 
+print ====================== client1_3  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client2_0.sim

+$tblStart = 0
+$tblEnd = 1000
+$tsStart = 1325347200001  # 2012-01-01 00:00:00.001
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client2_0
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client2_0 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lastLossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client2_0  insert data complete once ............
+  endi
+endw 
+print ====================== client2_0  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client2_1.sim

+$tblStart = 1000
+$tblEnd = 2000
+$tsStart = 1325347200001  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client2_1
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client2_1 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client2_1  insert data complete once ............
+  endi
+endw 
+print ====================== client2_1  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client2_2.sim

+$tblStart = 2000
+$tblEnd = 3000
+$tsStart = 1325347200001  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client2_2
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client2_2 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client2_2  insert data complete once ............
+  endi
+endw 
+print ====================== client2_2  success and auto end =====================
\ No newline at end of file

tests/script/unique/cluster/main2_client2_3.sim

+$tblStart = 3000
+$tblEnd = 4000
+$tsStart = 1325347200001  # 2012-01-01 00:00:00.000
+###############################################################
+
+sql connect
+
+$db = db1
+$stb = stb1
+
+sql use $db
+
+
+######sql create table $stb (ts timestamp, c1 int) tags(t1 int, t2 binary(8))
+$tagBinary = ' . client2_3
+$tagBinary = $tagBinary . '
+#print ======= tag: $tagBinary
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+#  print create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  sql create table if not exists $tb using $stb tags ( $i , $tagBinary )
+  $i = $i + 1
+endw 
+
+print ====================== client2_3 create table end, start insert data ............
+sql select count(tbname) from $stb
+print select count(tbname) from $stb
+print data00 $data00
+
+$rowsPerLoop = 100
+$ts = $tsStart
+
+$lossRows = 0
+
+$i = $tblStart
+while $i < $tblEnd
+  $tb = tb . $i
+  $x = 0
+  while $x < $rowsPerLoop
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 2a , $x ) ( $ts + 4a , $x ) ( $ts + 6a , $x ) ( $ts + 8a , $x ) ( $ts + 10a , $x ) ( $ts + 12a , $x ) ( $ts + 14a , $x ) ( $ts + 16a , $x ) ( $ts + 18a , $x ) ( $ts + 20a , $x ) ( $ts + 22a , $x ) ( $ts + 24a , $x ) ( $ts + 26a , $x ) ( $ts + 28a , $x ) ( $ts + 30a , $x ) ( $ts + 32a , $x ) ( $ts + 34a , $x ) ( $ts + 36a , $x ) ( $ts + 38a , $x )
+    $x = $x + 20
+    $ts = $ts + 40a
+  endw 
+
+  $totalRows = $totalRows + $x
+  $i = $i + 1
+  
+  if $i == $tblEnd then
+    $i = $tblStart
+    
+    sql select count(*) from $stb where t2 == $tagBinary
+    print ====================== client2_3  insert data complete once ............
+  endi
+endw 
+print ====================== client2_3  success and auto end =====================
\ No newline at end of file