Merge pull request #2124 from taosdata/hotfix/fileinfo

Hotfix/fileinfo

src/inc/tsdb.h

@@ -117,7 +117,7 @@ int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
 int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
 TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid);
 
-uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size);
+uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {

src/inc/tsync.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #define TAOS_SYNC_MAX_REPLICA 5
+#define TAOS_SYNC_MAX_INDEX   0x7FFFFFFF
 
 typedef enum _TAOS_SYNC_ROLE {
   TAOS_SYNC_ROLE_OFFLINE,
@@ -53,11 +54,16 @@ typedef struct {
   uint32_t  nodeId[TAOS_SYNC_MAX_REPLICA];
   int       role[TAOS_SYNC_MAX_REPLICA];  
 } SNodesRole;
- 
-// if name is empty(name[0] is zero), get the file from index or after, used by master
-// if name is provided(name[0] is not zero), get the named file at the specified index, used by unsynced node
-// it returns the file magic number and size, if file not there, magic shall be 0.
-typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion); 
+
+/* 
+  if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file
+  is found between index and eindex, index shall be updated, name shall be set, size shall be set to 
+  file size, and file magic number shall be returned. 
+
+  if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
+  zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
+*/
+typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); 
 
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file

src/mnode/src/mnodeSdb.c

@@ -200,7 +200,7 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeIpSet();
 }
 
-static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
+static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) {
   sdbUpdateMnodeRoles();
   return 0;
 }

src/tsdb/inc/tsdbMain.h

@@ -508,6 +508,8 @@ void    tsdbAdjustCacheBlocks(STsdbCache *pCache);
 int32_t tsdbGetMetaFileName(char *rootDir, char *fname);
 int     tsdbUpdateFileHeader(SFile *pFile, uint32_t version);
 
+int compFGroupKey(const void *key, const void *fgroup);
+
 #ifdef __cplusplus
 }
 #endif

src/tsdb/src/tsdbFile.c

@@ -35,7 +35,6 @@ const char *tsdbFileSuffix[] = {
     ".last"   // TSDB_FILE_TYPE_LAST
 };
 
-static int compFGroupKey(const void *key, const void *fgroup);
 static int compFGroup(const void *arg1, const void *arg2);
 static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid);
 
@@ -285,7 +284,7 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf
   return 0;
 }
 
-static int compFGroupKey(const void *key, const void *fgroup) {
+int compFGroupKey(const void *key, const void *fgroup) {
   int         fid = *(int *)key;
   SFileGroup *pFGroup = (SFileGroup *)fgroup;
   if (fid == pFGroup->fileId) {

src/tsdb/src/tsdbMain.c

@@ -1199,46 +1199,71 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) {
   tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables);
 }
 
-uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size) {
-  // TODO: need to refactor this function
-
+#define TSDB_META_FILE_INDEX 10000000
+uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   // STsdbMeta *pMeta = pRepo->tsdbMeta;
   STsdbFileH *pFileH = pRepo->tsdbFileH;
-  uint32_t   magic = 0;
-  char       fname[256] = "\0";
+  uint32_t    magic = 0;
+  char        fname[256] = "\0";
 
   struct stat fState;
-  char *spath = strdup(pRepo->rootDir);
-  char *prefixDir = dirname(spath);
 
-  if (name[0] == 0) {
-    // Map index to the file name
+  tsdbTrace("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex);
+  ASSERT(*index <= eindex);
+
+  char *sdup = strdup(pRepo->rootDir);
+  char *prefix = dirname(sdup);
+
+  if (name[0] == 0) {  // get the file from index or after, but not larger than eindex
     int fid = (*index) / 3;
 
-    if (fid >= pFileH->numOfFGroups) {
-      // return meta data file
-      if ((*index) % 3 > 0) { // it is finished
-        tfree(spath);
+    if (pFileH->numOfFGroups == 0 || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) {
+      if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
+        tsdbGetMetaFileName(pRepo->rootDir, fname);
+        *index = TSDB_META_FILE_INDEX;
+      } else {
+        tfree(sdup);
         return 0;
+      }
+    } else {
+      SFileGroup *pFGroup =
+          taosbsearch(&fid, pFileH->fGroup, pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE);
+      if (pFGroup->fileId == fid) {
+        strcpy(fname, pFGroup->files[(*index) % 3].fname);
       } else {
-        tsdbGetMetaFileName(pRepo->rootDir, fname);
+        if (pFGroup->fileId * 3 + 2 < eindex) {
+          strcpy(fname, pFGroup->files[0].fname);
+          *index = pFGroup->fileId * 3;
+        } else {
+          tfree(sdup);
+          return 0;
+        }
       }
+    }
+    strcpy(name, fname + strlen(prefix));
+  } else {                                 // get the named file at the specified index. If not there, return 0
+    if (*index == TSDB_META_FILE_INDEX) {  // get meta file
+      tsdbGetMetaFileName(pRepo->rootDir, fname);
     } else {
-      // return data file name
-      strcpy(fname, pFileH->fGroup[fid].files[(*index) % 3].fname);
+      int         fid = (*index) / 3;
+      SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid);
+      if (pFGroup == NULL) {  // not found
+        tfree(sdup);
+        return 0;
+      }
+
+      SFile *pFile = &pFGroup->files[(*index) % 3];
+      strcpy(fname, pFile->fname);
     }
-    strcpy(name, fname + strlen(spath));
-  } else {
-    // Name is provided, need to get the file info
-    sprintf(fname, "%s/%s", prefixDir, name);
   }
 
   if (stat(fname, &fState) < 0) {
-    tfree(spath);
+    tfree(sdup);
     return 0;
   }
 
+  tfree(sdup);
   *size = fState.st_size;
   magic = *size;
 

src/vnode/src/vnodeMain.c

@@ -39,7 +39,7 @@ static int32_t  vnodeReadCfg(SVnodeObj *pVnode);
 static int32_t  vnodeSaveVersion(SVnodeObj *pVnode);
 static int32_t  vnodeReadVersion(SVnodeObj *pVnode);
 static int      vnodeProcessTsdbStatus(void *arg, int status);
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion);
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion);
 static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
 static void     vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
@@ -433,10 +433,10 @@ static int vnodeProcessTsdbStatus(void *arg, int status) {
   return 0; 
 }
 
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, int32_t *size, uint64_t *fversion) {
+static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) {
   SVnodeObj *pVnode = ahandle;
   *fversion = pVnode->fversion;
-  return tsdbGetFileInfo(pVnode->tsdb, name, index, size);
+  return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
 }
 
 static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) {

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+sql create dnode $hostname2
+sleep 3000
+
+$totalTableNum = 10000
+$sleepTimer = 10000
+
+$db = db
+sql create database $db replica 1 maxTables $totalTableNum
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 int) tags(t1 int)
+$rowNum = 100
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1420041600000
+
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+    $ts = $tsStart + $x
+    sql insert into $tb values ( $ts + 0a , $x ) ( $ts + 1a , $x ) ( $ts + 2a , $x ) ( $ts + 3a , $x ) ( $ts + 4a , $x ) ( $ts + 5a , $x ) ( $ts + 6a , $x ) ( $ts + 7a , $x ) ( $ts + 8a , $x ) ( $ts + 9a , $x ) ( $ts + 10a , $x ) ( $ts + 11a , $x ) ( $ts + 12a , $x ) ( $ts + 13a , $x ) ( $ts + 14a , $x ) ( $ts + 15a , $x ) ( $ts + 16a , $x ) ( $ts + 17a , $x ) ( $ts + 18a , $x ) ( $ts + 19a , $x ) ( $ts + 20a , $x ) ( $ts + 21a , $x ) ( $ts + 22a , $x ) ( $ts + 23a , $x ) ( $ts + 24a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 25a , $x ) ( $ts + 26a , $x ) ( $ts + 27a , $x ) ( $ts + 28a , $x ) ( $ts + 29a , $x ) ( $ts + 30a , $x ) ( $ts + 31a , $x ) ( $ts + 32a , $x ) ( $ts + 33a , $x ) ( $ts + 34a , $x ) ( $ts + 35a , $x ) ( $ts + 36a , $x ) ( $ts + 37a , $x ) ( $ts + 38a , $x ) ( $ts + 39a , $x ) ( $ts + 40a , $x ) ( $ts + 41a , $x ) ( $ts + 42a , $x ) ( $ts + 43a , $x ) ( $ts + 44a , $x ) ( $ts + 45a , $x ) ( $ts + 46a , $x ) ( $ts + 47a , $x ) ( $ts + 48a , $x ) ( $ts + 49a , $x ) ( $ts + 50a , $x ) ( $ts + 51a , $x ) ( $ts + 52a , $x ) ( $ts + 53a , $x ) ( $ts + 54a , $x ) ( $ts + 55a , $x ) ( $ts + 56a , $x ) ( $ts + 57a , $x ) ( $ts + 58a , $x ) ( $ts + 59a , $x ) 
+    $x = $x + 60
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+sleep 1000
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname3
+sleep 3000
+
+sql alter database $db replica 2
+sleep $sleepTimer
+
+wait_dnode3_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode3_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode3_ready
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode3_ready
+endi
+
+sleep $sleepTimer
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+
+print ============== step4: stop dnode2 for checking if sync success
+system sh/exec.sh -n dnode2 -s stop
+sleep $sleepTimer
+
+wait_dnode2_offline:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+sleep $sleepTimer   # waitting for move master vnode of dnode2 to dnode3
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: restart dnode2
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+
+wait_dnode2_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+
+sleep $sleepTimer
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync
+system sh/exec.sh -n dnode4 -s start
+sql create dnode $hostname4
+sleep 3000
+
+sql alter database $db replica 3
+sleep $sleepTimer
+
+wait_dnode4_ready:
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto wait_dnode4_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+#$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode4Status != ready then
+  sleep 2000
+  goto wait_dnode4_ready
+endi
+
+sleep $sleepTimer
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step7: alter replica from 3 to 2, and waiting sync
+sql alter database $db replica 2
+sleep $sleepTimer
+
+wait_vgroups_replic_to_2:
+sql show vgroups
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$thirdDnode_2 = $data8_1 
+$thirdDnode_3 = $data8_2 
+$thirdDnode_4 = $data8_3 
+$thirdDnode_5 = $data8_4 
+
+if $thirdDnode_2 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_2
+endi
+if $thirdDnode_3 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_2
+endi
+if $thirdDnode_4 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_2
+endi
+if $thirdDnode_5 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_2
+endi
+
+sleep $sleepTimer  #waiting del one replica data
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step8: alter replica from 2 to 1, and waiting sync
+sql alter database $db replica 1
+sleep $sleepTimer
+
+wait_vgroups_replic_to_1:
+sql show vgroups
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$sencodDnode_2 = $data5_1 
+$sencodDnode_3 = $data5_2 
+$sencodDnode_4 = $data5_3 
+$sencodDnode_5 = $data5_4 
+
+if $sencodDnode_2 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_1
+endi
+if $sencodDnode_3 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_1
+endi
+if $sencodDnode_4 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_1
+endi
+if $sencodDnode_5 != null then
+  sleep 2000
+  goto wait_vgroups_replic_to_1
+endi
+
+all_dnodes_ready:
+sql show dnodes
+if $rows != 4 then
+  sleep 2000
+  goto all_dnodes_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  sleep 2000
+  goto all_dnodes_ready
+endi
+if $dnode2Status != ready then
+  sleep 2000
+  goto all_dnodes_ready
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto all_dnodes_ready
+endi
+if $dnode4Status != ready then
+  sleep 2000
+  goto all_dnodes_ready
+endi
+
+sleep $sleepTimer  #waiting del one replica data
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step9: drop dnode2/dnode3
+sql drop dnode $hostname2
+sql drop dnode $hostname3
+sleep $sleepTimer
+
+wait_dnode23_dropped:
+sql show dnodes
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode23_dropped
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4 
+
+if $dnode2Status != null then
+  sleep 2000
+  goto wait_dnode23_dropped
+endi
+if $dnode3Status != null then
+  sleep 2000
+  goto wait_dnode23_dropped
+endi
+if $dnode4Status != ready then
+  return -1
+endi
+
+sleep $sleepTimer  #waiting move vnode from dnode3/dnode3 to dnode4
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi