Merge branch 'develop' into feature/query

src/rpc/src/rpcCache.c

@@ -127,7 +127,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in
 
   hash = rpcHashConn(pCache, fqdn, port, connType);
   pNode = (SConnHash *)taosMemPoolMalloc(pCache->connHashMemPool);
-  strcpy(pNode->fqdn, fqdn);
+  tstrncpy(pNode->fqdn, fqdn, sizeof(pNode->fqdn));
   pNode->port = port;
   pNode->connType = connType;
   pNode->data = data;

src/rpc/src/rpcMain.c

@@ -60,7 +60,7 @@ typedef struct {
 
   void     *idPool;   // handle to ID pool
   void     *tmrCtrl;  // handle to timer
-  void     *hash;     // handle returned by hash utility
+  SHashObj *hash;     // handle returned by hash utility
   void     *tcphandle;// returned handle from TCP initialization
   void     *udphandle;// returned handle from UDP initialization
   void     *pCache;   // connection cache
@@ -211,7 +211,7 @@ void *rpcOpen(const SRpcInit *pInit) {
   pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
   if (pRpc == NULL) return NULL;
 
-  if(pInit->label) strcpy(pRpc->label, pInit->label);
+  if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label));
   pRpc->connType = pInit->connType;
   pRpc->idleTime = pInit->idleTime;
   pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads;
@@ -228,7 +228,7 @@ void *rpcOpen(const SRpcInit *pInit) {
   size_t size = sizeof(SRpcConn) * pRpc->sessions;
   pRpc->connList = (SRpcConn *)calloc(1, size);
   if (pRpc->connList == NULL) {
-    tError("%s failed to allocate memory for taos connections, size:%d", pRpc->label, size);
+    tError("%s failed to allocate memory for taos connections, size:%ld", pRpc->label, size);
     rpcClose(pRpc);
     return NULL;
   }
@@ -459,7 +459,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) {
   pInfo->clientPort = pConn->peerPort;
   // pInfo->serverIp = pConn->destIp;
 
-  strcpy(pInfo->user, pConn->user);
+  strncpy(pInfo->user, pConn->user, sizeof(pInfo->user));
   return 0;
 }
 
@@ -503,10 +503,10 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort,
   pConn = rpcAllocateClientConn(pRpc); 
 
   if (pConn) { 
-    strcpy(pConn->peerFqdn, peerFqdn);
+    tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn));
     pConn->peerIp = peerIp;
     pConn->peerPort = peerPort;
-    strcpy(pConn->user, pRpc->user);
+    tstrncpy(pConn->user, pRpc->user, sizeof(pConn->user));
     pConn->connType = connType;
 
     if (taosOpenConn[connType]) {
@@ -804,7 +804,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
 
   pConn = rpcGetConnObj(pRpc, sid, pRecv);
   if (pConn == NULL) {
-    tTrace("%s %p, failed to get connection obj(%s)", pRpc->label, pHead->ahandle, tstrerror(terrno)); 
+    tTrace("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); 
     return NULL;
   } else {
     if (rpcIsReq(pHead->msgType)) {

src/rpc/src/rpcTcp.c

@@ -73,7 +73,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1);
   pServerObj->ip = ip;
   pServerObj->port = port;
-  strcpy(pServerObj->label, label);
+  tstrncpy(pServerObj->label, label, sizeof(pServerObj->label));
   pServerObj->numOfThreads = numOfThreads;
 
   pServerObj->pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), numOfThreads);
@@ -87,7 +87,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
   pThreadObj = pServerObj->pThreadObj;
   for (int i = 0; i < numOfThreads; ++i) {
     pThreadObj->processData = fp;
-    strcpy(pThreadObj->label, label);
+    tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
     pThreadObj->shandle = shandle;
 
     code = pthread_mutex_init(&(pThreadObj->mutex), NULL);
@@ -247,7 +247,7 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *
 
   pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj));
   memset(pThreadObj, 0, sizeof(SThreadObj));
-  strcpy(pThreadObj->label, label);
+  tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
   pThreadObj->ip = ip;
   pThreadObj->shandle = shandle;
 

src/rpc/src/rpcUdp.c

@@ -72,7 +72,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
   pSet->port = port;
   pSet->shandle = shandle;
   pSet->fp = fp;
-  strcpy(pSet->label, label);
+  tstrncpy(pSet->label, label, sizeof(pSet->label));
 
   uint16_t ownPort;
   for (int i = 0; i < threads; ++i) {
@@ -99,7 +99,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
       pConn->localPort = (uint16_t)ntohs(sin.sin_port);
     }
 
-    strcpy(pConn->label, label);
+    tstrncpy(pConn->label, label, sizeof(pConn->label));
     pConn->shandle = shandle;
     pConn->processData = fp;
     pConn->index = i;

src/rpc/test/rclient.c

@@ -76,6 +76,7 @@ int main(int argc, char *argv[]) {
   int      numOfReqs = 0;
   int      appThreads = 1;
   char     serverIp[40] = "127.0.0.1";
+  char     secret[TSDB_KEY_LEN] = "mypassword";
   struct   timeval systemTime;
   int64_t  startTime, endTime;
   pthread_attr_t thattr;
@@ -97,7 +98,7 @@ int main(int argc, char *argv[]) {
   rpcInit.sessions     = 100;
   rpcInit.idleTime     = tsShellActivityTimer*1000;
   rpcInit.user         = "michael";
-  rpcInit.secret       = "mypassword";
+  rpcInit.secret       = secret;
   rpcInit.ckey         = "key";
   rpcInit.spi          = 1;
   rpcInit.connType     = TAOS_CONN_CLIENT;
@@ -106,7 +107,7 @@ int main(int argc, char *argv[]) {
     if (strcmp(argv[i], "-p")==0 && i < argc-1) {
       ipSet.port[0] = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
-      tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn)); 
+      tstrncpy(ipSet.fqdn[0], argv[++i], sizeof(ipSet.fqdn[0])); 
     } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
       rpcInit.numOfThreads = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {

src/rpc/test/rsclient.c

@@ -77,6 +77,7 @@ int main(int argc, char *argv[]) {
   int      numOfReqs = 0;
   int      appThreads = 1;
   char     serverIp[40] = "127.0.0.1";
+  char     secret[TSDB_KEY_LEN] = "mypassword";
   struct   timeval systemTime;
   int64_t  startTime, endTime;
   pthread_attr_t thattr;
@@ -98,7 +99,7 @@ int main(int argc, char *argv[]) {
   rpcInit.sessions     = 100;
   rpcInit.idleTime     = tsShellActivityTimer*1000;
   rpcInit.user         = "michael";
-  rpcInit.secret       = "mypassword";
+  rpcInit.secret       = secret;
   rpcInit.ckey         = "key";
   rpcInit.spi          = 1;
   rpcInit.connType     = TAOS_CONN_CLIENT;

src/util/src/tsocket.c

@@ -19,6 +19,7 @@
 #include "tutil.h"
 
 int taosGetFqdn(char *fqdn) {
+  int  code = 0;
   char hostname[1024];
   hostname[1023] = '\0';
   gethostname(hostname, 1023);
@@ -27,13 +28,15 @@ int taosGetFqdn(char *fqdn) {
   h = gethostbyname(hostname);
   if (h != NULL) {
     strcpy(fqdn, h->h_name);
-    return 0;
   } else {
-    uError("failed to get host name");
-    return -1;
+    uError("failed to get host name(%s)", strerror(errno));
+    code = -1;
   }
 
-  free(h);
+  // to do: free the resources
+  // free(h);
+
+  return code;
 }
 
 uint32_t taosGetIpFromFqdn(const char *fqdn) {
@@ -47,7 +50,7 @@ uint32_t ip2uint(const char *const ip_addr) {
   char ip_addr_cpy[20];
   char ip[5];
 
-  strcpy(ip_addr_cpy, ip_addr);
+  tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy));
 
   char *s_start, *s_end;
   s_start = ip_addr_cpy;
@@ -206,7 +209,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
   int                reuse, nocheck;
   int                bufSize = 8192000;
 
-  uTrace("open udp socket:%s:%hu", ip, port);
+  uTrace("open udp socket:0x%x:%hu", ip, port);
 
   memset((char *)&localAddr, 0, sizeof(localAddr));
   localAddr.sin_family = AF_INET;
@@ -257,7 +260,7 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) {
 
   /* bind socket to local address */
   if (bind(sockFd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
-    uError("failed to bind udp socket: %d (%s), %s:%hu", errno, strerror(errno), ip, port);
+    uError("failed to bind udp socket: %d (%s), 0x%x:%hu", errno, strerror(errno), ip, port);
     taosCloseSocket(sockFd);
     return -1;
   }
@@ -363,7 +366,7 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
   int                sockFd;
   int                reuse;
 
-  uTrace("open tcp server socket:%s:%hu", ip, port);
+  uTrace("open tcp server socket:0x%x:%hu", ip, port);
 
   bzero((char *)&serverAdd, sizeof(serverAdd));
   serverAdd.sin_family = AF_INET;

src/wal/src/walMain.c

@@ -68,11 +68,19 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
   pWal->num = 0;
   pWal->level = pCfg->walLevel;
   pWal->keep = pCfg->keep;
-  strcpy(pWal->path, path);
+  tstrncpy(pWal->path, path, sizeof(pWal->path));
   pthread_mutex_init(&pWal->mutex, NULL);
 
-  if (access(path, F_OK) != 0) mkdir(path, 0755);
-  
+  if (access(path, F_OK) != 0) {
+    if (mkdir(path, 0755) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
+      pthread_mutex_destroy(&pWal->mutex);
+      free(pWal);
+      pWal = NULL;
+    }
+  }
+     
   if (pCfg->keep == 1) return pWal;
 
   if (walHandleExistingFiles(path) == 0) 
@@ -80,7 +88,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
 
   if (pWal->fd <0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    wError("wal:%s, failed to open", path);
+    wError("wal:%s, failed to open(%s)", path, strerror(errno));
     pthread_mutex_destroy(&pWal->mutex);
     free(pWal);
     pWal = NULL;
@@ -119,7 +127,8 @@ void walClose(void *handle) {
 int walRenew(void *handle) {
   if (handle == NULL) return 0;
   SWal *pWal = handle;
-  int   code = 0;
+
+  terrno = 0;
 
   pthread_mutex_lock(&pWal->mutex);
 
@@ -135,8 +144,8 @@ int walRenew(void *handle) {
   pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
   if (pWal->fd < 0) {
-    wError("wal:%d, failed to open(%s)", pWal->name, strerror(errno));
-    code = -1;
+    wError("wal:%s, failed to open(%s)", pWal->name, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
     wTrace("wal:%s, it is created", pWal->name);
 
@@ -156,14 +165,15 @@ int walRenew(void *handle) {
   
   pthread_mutex_unlock(&pWal->mutex);
 
-  return code;
+  return terrno;
 }
 
 int walWrite(void *handle, SWalHead *pHead) {
   SWal *pWal = handle;
-  int   code = 0;
   if (pWal == NULL) return -1;
 
+  terrno = 0;
+
   // no wal  
   if (pWal->level == TAOS_WAL_NOLOG) return 0;
   if (pHead->version <= pWal->version) return 0;
@@ -174,12 +184,12 @@ int walWrite(void *handle, SWalHead *pHead) {
 
   if(write(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
-    code = -1;
+    terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
     pWal->version = pHead->version;
   }
 
-  return code;
+  return terrno;
 }
 
 void walFsync(void *handle) {
@@ -196,11 +206,11 @@ void walFsync(void *handle) {
 
 int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int)) {
   SWal    *pWal = handle;
-  int      code = 0;
   struct   dirent *ent;
   int      count = 0;
   uint32_t maxId = 0, minId = -1, index =0;
 
+  terrno = 0;
   int   plen = strlen(walPrefix);
   char  opath[TSDB_FILENAME_LEN+5];
    
@@ -224,30 +234,30 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
   closedir(dir);
 
   if (count == 0) {
-    if (pWal->keep) code = walRenew(pWal);
-    return code;
+    if (pWal->keep) terrno = walRenew(pWal);
+    return terrno;
   }
 
   if ( count != (maxId-minId+1) ) {
     wError("wal:%s, messed up, count:%d max:%d min:%d", opath, count, maxId, minId);
-    code = -1;
+    terrno = TAOS_SYSTEM_ERROR(TSDB_CODE_APP_ERROR);
   } else {
     wTrace("wal:%s, %d files will be restored", opath, count);
 
     for (index = minId; index<=maxId; ++index) {
       sprintf(pWal->name, "%s/%s%d", opath, walPrefix, index);
-      code = walRestoreWalFile(pWal, pVnode, writeFp);
-      if (code < 0) break;
+      terrno = walRestoreWalFile(pWal, pVnode, writeFp);
+      if (terrno < 0) break;
     }
   }
 
-  if (code == 0) {
+  if (terrno == 0) {
     if (pWal->keep == 0) {
-      code = walRemoveWalFiles(opath);
-      if (code == 0) {
+      terrno = walRemoveWalFiles(opath);
+      if (terrno == 0) {
         if (remove(opath) < 0) {
           wError("wal:%s, failed to remove directory(%s)", opath, strerror(errno));
-          code = -1;
+          terrno = TAOS_SYSTEM_ERROR(errno);
         }
       }
     } else { 
@@ -258,12 +268,12 @@ int walRestore(void *handle, void *pVnode, int (*writeFp)(void *, void *, int))
       pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
       if (pWal->fd < 0) {
         wError("wal:%s, failed to open file(%s)", pWal->name, strerror(errno));
-        code = -1;
+        terrno = TAOS_SYSTEM_ERROR(errno);
       }
     }
   }
 
-  return code;
+  return terrno;
 }
 
 int walGetWalFile(void *handle, char *name, uint32_t *index) {
@@ -292,40 +302,47 @@ int walGetWalFile(void *handle, char *name, uint32_t *index) {
 }  
 
 static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
-  int   code = 0;
   char *name = pWal->name;
 
+  terrno = 0;
   char *buffer = malloc(1024000);  // size for one record
-  if (buffer == NULL) return -1;
+  if (buffer == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);   
+    return terrno;
+  }
 
   SWalHead *pHead = (SWalHead *)buffer;
 
   int fd = open(name, O_RDONLY);
   if (fd < 0) {
     wError("wal:%s, failed to open for restore(%s)", name, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
     free(buffer);
-    return -1;
+    return terrno;
   }
 
   wTrace("wal:%s, start to restore", name);
 
   while (1) {
     int ret = read(fd, pHead, sizeof(SWalHead));
-    if ( ret == 0) { code = 0; break;}  
+    if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
       wWarn("wal:%s, failed to read head, skip, ret:%d(%s)", name, ret, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
       break;
     }
 
     if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
       wWarn("wal:%s, cksum is messed up, skip the rest of file", name);
+      terrno = TAOS_SYSTEM_ERROR(errno);
       break;
     } 
 
     ret = read(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
+      terrno = TAOS_SYSTEM_ERROR(errno);
       break;
     }
 
@@ -336,11 +353,10 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   close(fd);
   free(buffer);
 
-  return code;
+  return terrno;
 }
 
 int walHandleExistingFiles(const char *path) {
-  int    code = 0;
   char   oname[TSDB_FILENAME_LEN * 3];
   char   nname[TSDB_FILENAME_LEN * 3];
   char   opath[TSDB_FILENAME_LEN];
@@ -350,6 +366,7 @@ int walHandleExistingFiles(const char *path) {
   struct dirent *ent;
   DIR   *dir = opendir(path);
   int    plen = strlen(walPrefix);
+  terrno = 0;
 
   if (access(opath, F_OK) == 0) {
     // old directory is there, it means restore process is not finished
@@ -360,13 +377,19 @@ int walHandleExistingFiles(const char *path) {
     int count = 0;
     while ((ent = readdir(dir))!= NULL) {  
       if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
-        if (access(opath, F_OK) != 0) mkdir(opath, 0755);
-
         sprintf(oname, "%s/%s", path, ent->d_name);
         sprintf(nname, "%s/old/%s", path, ent->d_name);
+        if (access(opath, F_OK) != 0) {
+          if (mkdir(opath, 0755) != 0) {
+            wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
+            terrno = TAOS_SYSTEM_ERROR(errno);
+            break;
+          } 
+        }
+
         if (rename(oname, nname) < 0) {
           wError("wal:%s, failed to move to new:%s", oname, nname);
-          code = -1;
+          terrno = TAOS_SYSTEM_ERROR(errno);
           break;
         } 
 
@@ -378,14 +401,14 @@ int walHandleExistingFiles(const char *path) {
   }
   
   closedir(dir);
-  return code;
+  return terrno;
 }
 
 static int walRemoveWalFiles(const char *path) {
   int    plen = strlen(walPrefix);
   char   name[TSDB_FILENAME_LEN * 3];
-  int    code = 0;
-
+ 
+  terrno = 0;
   if (access(path, F_OK) != 0) return 0;
 
   struct dirent *ent;
@@ -396,13 +419,13 @@ static int walRemoveWalFiles(const char *path) {
       sprintf(name, "%s/%s", path, ent->d_name);
       if (remove(name) <0) {
         wError("wal:%s, failed to remove(%s)", name, strerror(errno));
-        code = -1; break;
+        terrno = TAOS_SYSTEM_ERROR(errno);
       }
     }
   } 
 
   closedir(dir);
 
-  return code;
+  return terrno;
 }
 

tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim

@@ -2,11 +2,11 @@
 # step 1: start dnode1
 # step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
 # step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
-# step 4: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc 
+# step 4: insert old data(now-20d) and new data(now-40d), control data rows  in order to save in cache, not falling disc 
 # step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
-# step 6: insert two data rows: now-16d, now+16d
+# step 6: insert two data rows: now-21d, now-41d
 # step 7: restart dnode2, waiting sync end
-# expect: in dnode2, the files 1840 and 1842 will be removed
+# expect: in dnode2, the files 1837 and 1839 will be removed
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
@@ -14,10 +14,10 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -39,11 +39,6 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
 
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
@@ -64,16 +59,16 @@ sql connect
 
 print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
-#system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname2
-#sql create dnode $hostname3
+sql create dnode $hostname3
 sleep 3000
 
 $totalTableNum = 1
 $sleepTimer = 3000
 
 $db = db
-sql create database $db replica 1 cache 1
+sql create database $db replica 2 cache 1
 sql use $db
 
 # create table , insert data
@@ -82,7 +77,7 @@ sql create table $stb (ts timestamp, c1 double) tags(t1 int)
 $rowNum = 130000
 $tblNum = $totalTableNum
 $totalRows = 0
-#$tsStart = 1420041600000
+$tsStart = 1577808000000  # 2020-01-01 00:00:00.000
 
 # insert over 2M data in order to falling disc, generate one file
 $i = 0
@@ -102,19 +97,24 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-sleep 1000
-print data00 $data00
-if $data00 != $totalRows then
-	return -1
+print rows:$rows data00:$data00
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 == 0 then
+  return -1
 endi
 
+$totalRows = $data00
+
 print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc
-sql insert into $tb values ( now - 15d , -15 )
-sql insert into $tb values ( now + 15d ,  15 )
+sql insert into $tb values ( now - 20d , -20 )
+sql insert into $tb values ( now - 40d , -40 )
 $totalRows = $totalRows + 2
 
 print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
-system sh/exec.sh -n dnode2 -s stop
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 
 wait_dnode2_offline:
@@ -153,48 +153,14 @@ if $data00 != $totalRows then
 endi
 
 print ============== step5: insert two data rows: now-16d, now+16d, 
-sql insert into $tb values ( now - 16d , -16 )
-sql insert into $tb values ( now + 16d ,  16 )
+sql insert into $tb values ( now - 21d , -21 )
+sql insert into $tb values ( now - 41d , -41 )
 $totalRows = $totalRows + 2
 
-return 1
-
-
-
-print ============== step5: restart dnode2, waiting sync end
-system sh/exec.sh -n dnode2 -s start
-sleep 3000
-
-wait_dnode2_ready:
-sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode2_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode2_ready
-endi
-
-sleep $sleepTimer
-# check using select
 sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step6: in dnode2, the files 1840 and 1842 will be removed
-# how check in script ???
\ No newline at end of file
+print ============== step6: please check there should be 3 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/

tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim

+# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
+# step 1: start dnode1
+# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
+# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
+# step 4: insert old data(now-20d) and new data(now-40d), control data rows  in order to save in cache, not falling disc 
+# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
+# step 6: insert two data rows: now-21d, now-41d
+# step 7: restart dnode2, waiting sync end
+# expect: in dnode2, the files 1837 and 1839 will be removed
+
+sql connect
+sleep 3000
+
+print ============== step7: restart dnode2, waiting sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+
+wait_dnode2_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+
+sleep 1000
+# check using select
+$db = db
+$stb = stb
+sql use $db
+sql select count(*) from $stb
+print data00 $data00, should equal to dn2_mn1_cache_file_sync.sim output
+#if $data00 != $totalRows then
+#	return -1
+#endi
+
+print ============== step8: please check there should be 1 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/

tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim

@@ -23,28 +23,39 @@ system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
 
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 200
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
 
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
+
 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
 
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
 
-system sh/cfg.sh -n dnode1 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode2 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode3 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode4 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 20
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
 
 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
@@ -63,9 +74,9 @@ sql connect
 print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 1000
 sql create dnode $hostname2
 sql create dnode $hostname3
+sleep 3000
 
 $rowNum = 100
 $tblNum = 16
@@ -151,7 +162,7 @@ endi
 #  return -1
 #endi
 
-sleep 30000
+sleep 15000
 
 wait_drop:
 sql show dnodes
@@ -174,10 +185,69 @@ endi
 if $dnode2Status != ready then
   return -1
 endi
+if $dnode3Status != null then
+  return -1
+endi
 if $dnode4Status != ready then
   return -1
 endi
 
+print ============== step4-1: restart dnode3, adn add into cluster
+system rm -rf ../../sim/dnode3
+sleep 3000
+
+system sh/deploy.sh -n dnode3 -i 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
+
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname3
+sleep 3000
+
+wait_dnode3_ready:
+sql show dnodes
+print rows: $rows
+if $rows != 4 then
+  sleep 3000
+  goto wait_dnode3_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4  
+$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  return -1
+endi
+if $dnode2Status != ready then
+  return -1
+endi
+if $dnode3Status != null then
+  return -1
+endi
+if $dnode4Status != ready then
+  return -1
+endi
+if $dnode5Status != ready then
+  return -1
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
 
 print ============== step5: start dnode5 and add into cluster , drop database
 sql drop database $db
@@ -187,7 +257,7 @@ sql create dnode $hostname5
 sleep 3000
 wait_dnode5:
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 3000
   goto wait_dnode5
 endi
@@ -196,10 +266,12 @@ print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
 print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 $dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode4Status = $data4_4  
-$dnode5Status = $data4_5
+$dnode5Status = $data4_5 
+$dnode6Status = $data4_6
 
 if $dnode1Status != ready then
   return -1
@@ -207,17 +279,22 @@ endi
 if $dnode2Status != ready then
   return -1
 endi
+if $dnode3Status != null then
+  return -1
+endi
 if $dnode4Status != ready then
   return -1
 endi
 if $dnode5Status != ready then
   return -1
 endi
-
+if $dnode6Status != ready then
+  return -1
+endi
 
 print ============== step6: create database and table until not free vnodes 
 $rowNum = 100
-$tblNum = 24
+$tblNum = 32
 $totalRows = 0
 $tsStart = 1420041600000
 
@@ -259,8 +336,9 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step7: drop dnode3, and system should prompt cannot drop dnodes 
-sql_error drop dnode $hostname3
-print ============== step8: add one new table, and system should prompt 'need more dnode' 
+print ============== step7: drop dnode $hostname5, system should prompt "DB error: no enough dnodes" 
+sql_error drop dnode $hostname5
+print error: $error 
+print ============== step8: create table tb_more using $stb tags( 1000 ), system should prompt 'DB error: no enough dnodes' 
 sql_error create table tb_more using $stb tags( 1000 )
-
+print error: $error 

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -89,11 +89,8 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-
-
-
-print ============== step2-1: start dnode2 for falling disc, then restart dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
+print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 wait_dnode2_offline_0:
 sql show dnodes
@@ -151,10 +148,6 @@ if $data00 != $totalRows then
 endi
 
 
-
-
-
-
 print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3

tests/script/unique/arbitrator/insert_duplicationTs.sim

+# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
+# step 1: start dnode1
+# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
+# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
+# step 4: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc 
+# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
+# step 6: insert two data rows: now-16d, now+16d
+# step 7: restart dnode2, waiting sync end
+# expect: in dnode2, the files 1840 and 1842 will be removed
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+$totalTableNum = 1
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 cache 1
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 double) tags(t1 int)
+$rowNum = 130000
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1577808000000  # 2020-01-01 00:00:00.000
+
+# insert over 2M data in order to falling disc, generate one file
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+#    $ts = $tsStart + $x
+    sql insert into $tb values ( now + 0s , $x ) ( now + 1s , $x ) ( now + 2s , $x ) ( now + 3s , $x ) ( now + 4s , $x ) ( now + 5s , $x ) ( now + 6s , $x ) ( now + 7s , $x ) ( now + 8s , $x ) ( now + 9s , $x )
+    $x = $x + 10
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print rows:$rows data00:$data00
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 == 0 then
+  return -1
+endi
+
+$totalRows = $data00
+
+print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc
+sql insert into $tb values ( now - 20d , -20 )
+sql insert into $tb values ( now - 40d , -40 )
+$totalRows = $totalRows + 2
+
+print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+wait_dnode2_offline:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+sleep $sleepTimer   # waitting for move master vnode of dnode2 to dnode3
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: insert two data rows: now-16d, now+16d, 
+sql insert into $tb values ( now - 21d , -21 )
+sql insert into $tb values ( now - 41d , -41 )
+$totalRows = $totalRows + 2
+
+print ============== step5: restart dnode2, waiting sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+
+wait_dnode2_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+
+sleep $sleepTimer
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+

tests/script/unique/arbitrator/replica_changeWithArbitrator.sim

@@ -2,18 +2,42 @@ system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
-system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
-system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
 
 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 8
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
@@ -22,21 +46,23 @@ system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
 
-
-print ============== step1: replica is 1, and start 1 dnode
+print ============== step1: replica is 1, and start 1 dnode, then create tables and insert data
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-$db = replica_db1
-sql create database $db replica 1 maxTables 4
+$totalTableNum = 12
+
+$db = db
+sql create database $db replica 1 maxTables  $totalTableNum
 sql use $db
 
 # create table , insert data
-$stb = repl_stb
+$stb = stb
 sql create table $stb (ts timestamp, c1 int) tags(t1 int)
 $rowNum = 10
-$tblNum = 12
+$tblNum = $totalTableNum
+$totalRows = $rowNum * $tblNum
 
 $ts0 = 1420041600000
 $ts = $ts0
@@ -55,46 +81,136 @@ while $i < $tblNum
     $x = $x + 1
   endw 
   $i = $i + 1
+  print       $tb inserted rows: $x
 endw 
 
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
 print ============== step2: add 1 new dnode, expect balanced
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
+sleep 3000
 
 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2
-$x = 0
-show2: 
-	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
-		return -1
-	endi
-	
+wait_dnode2_ready:
 sql show dnodes
-print dnode1 openVnodes $data2_1
-print dnode2 openVnodes $data2_2
-if $data2_1 != 2 then
-	goto show2
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
 endi
-if $data2_2 != 1 then
-	goto show2
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
 endi
 
-print ============== step4: stop dnode1, and wait dnode2 master
+print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2
 system sh/exec.sh -n dnode1 -s stop
+system sh/exec.sh -n dnode2 -s stop
+sleep 3000
 
-$x = 0
-loop_wait:
-	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
-	  print ERROR: after dnode1 stop, dnode2 didn't become a master! 
-		return -1
-	endi
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sleep 5000
+
+
+print ============= step4: wait dnode ready
+wait_dnode_ready:
+sql show dnodes
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+#print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode1
+system sh/exec.sh -n dnode1 -s stop
+sleep 3000
+
+wait_dnode2_master:
 sql show mnodes
-$dnodeRole = $data2_1
-print dnodeRole ==> $dnodeRole
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_master
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+#print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $data2_1 != offline then
+  sleep 2000
+  goto wait_dnode2_master
+endi
+if $data2_2 != master then
+  sleep 2000
+  goto wait_dnode2_master
+endi
 
-if $dnodeRole != master then
-  goto loop_wait
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
 endi
+
+