Merge pull request #21317 from taosdata/test/TS-3333-3.0

test: add cluster global tdDnodes

tests/parallel_test/cases.task

@@ -479,6 +479,11 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4
 
+,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3
+
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py

tests/pytest/util/autogen.py

@@ -93,9 +93,9 @@ class AutoGen:
         return ''.join(random.choice(letters) for i in range(count))
 
     # create db
-    def create_db(self, dbname):
+    def create_db(self, dbname, vgroups = 2, replica = 1):
         self.dbname  = dbname
-        tdSql.execute(f'create database {dbname}')
+        tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
         tdSql.execute(f'use {dbname}')
         
     # create table or stable

tests/pytest/util/dnodes.py

@@ -859,5 +859,4 @@ class TDDnodes:
     def getAsan(self):
         return self.asan
 
-
 tdDnodes = TDDnodes()
\ No newline at end of file

tests/system-test/3-enterprise/restore/restoreBasic.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+from util.dnodes import *
+from util.autogen import *
+from util.cluster import *
+
+import random
+import os
+import subprocess
+import shutil
+import time
+    
+
+class RestoreBasic:
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdSql.init(conn.cursor())
+        self.dnodes_num = 5
+
+        # get from global
+        # test
+        self.dnodes = cluster.dnodes
+        num = len(self.dnodes)
+
+        if num < self.dnodes_num :
+            tdLog.exit(f" cluster dnode is less than {self.dnodes_num}. num={num}")
+
+        # create data
+        self.dbname = "db"
+        self.stable = "st"
+        self.child_count = 100
+        self.insert_rows = 10000
+        self.create_data()
+
+    #  create data
+    def create_data(self):
+        gen = AutoGen()
+        gen.create_db(self.dbname, 8, 3)
+        gen.create_stable(self.stable, 5, 10, 8, 8)
+        gen.create_child(self.stable, "d", self.child_count)
+        gen.set_batch_size(1000)
+        gen.insert_data(self.insert_rows)
+        
+        tdSql.execute(f"flush database {self.dbname}")
+        # put some duplicate ts on wal
+        gen.insert_data(self.insert_rows%100)
+
+        for i in range(self.dnodes_num):
+            sql = f"create qnode on dnode {i+1}"
+            tdSql.execute(sql)
+
+
+    # status
+    def check_status_corrent(self):
+        # query
+        tdSql.query(f" show {self.dbname}.vgroups")
+
+        # check 8 vgroups
+        tdSql.checkRows(8)
+
+        # check data corrent
+        for i in range(8):
+            leader = False
+            for j in range(3):
+                status = tdSql.getData(i, 4 + j*2)
+                if status == "leader":
+                    leader = True
+                elif status == "follower":
+                    pass
+                else:
+                    tdLog.info(f" check vgroups status not leader or follower. i={i} j={j} status={status}")
+                    return False
+            
+            # check leader
+            if  leader == False:
+                tdLog.info(f" check vgroups not found leader i={i} ")
+                return False
+
+        # info
+        tdLog.info("check vgroups status successfully.")
+        return True
+
+    # check data corrent
+    def check_corrent(self):
+        # check status
+        status = False
+        for i in range(100): 
+            if self.check_status_corrent():
+                 status = True
+                 break
+            else:
+                time.sleep(0.5)
+                tdLog.info(f"sleep 500ms retry {i} to check status again...") 
+
+        if status == False:
+            tdLog.exit("check vgroups status failed, exit.")             
+            
+        # check rows count
+        sql = f"select count(ts) from {self.dbname}.{self.stable}"
+        tdSql.query(sql)
+        tdSql.checkData(0, 0, self.child_count* self.insert_rows)
+
+
+    # restore dnode
+    def restore_dnode(self, index):
+        tdLog.info(f"start restore dnode {index}")
+        dnode = self.dnodes[index - 1]
+        
+        # stop dnode
+        tdLog.info(f"stop dnode {index}")
+        dnode.stoptaosd()
+
+        # remove dnode folder
+        try:
+            shutil.rmtree(dnode.dataDir)
+            tdLog.info(f"delete dir {dnode.dataDir} successful")
+        except OSError as x:
+            tdLog.exit(f"remove path {dnode.dataDir} error : {x.strerror}")
+
+        dnode.starttaosd()
+        
+        # exec restore
+        sql = f"restore dnode {index}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+        self.check_corrent()
+
+    # restore vnode
+    def restore_vnode(self, index):
+        tdLog.info(f"start restore vnode on dnode {index}")
+        dnode = self.dnodes[index - 1]
+        del_dir = f"{dnode.dataDir}/vnode"
+
+        # stop dnode
+        tdLog.info(f"stop dnode {index}")
+        dnode.stoptaosd()
+        
+        # remove dnode folder
+        try:
+            shutil.rmtree(del_dir)
+            tdLog.info(f"delete dir {del_dir} successful")
+        except OSError as x:
+            tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
+
+        dnode.starttaosd()
+        
+        # exec restore
+        sql = f"restore vnode on dnode {index}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+
+        # check result
+        self.check_corrent()
+
+        
+    # restore mnode
+    def restore_mnode(self, index):
+        tdLog.info(f"start restore mnode {index}")
+        dnode = self.dnodes[index - 1]
+        del_dir = f"{dnode.dataDir}/mnode"
+        
+        # stop dnode
+        tdLog.info(f"stop dnode {index}")
+        dnode.stoptaosd()
+
+        # remove dnode folder
+        try:
+            shutil.rmtree(del_dir)
+            tdLog.info(f"delete dir {del_dir} successful")
+        except OSError as x:
+            tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
+
+        dnode.starttaosd()
+        
+        # exec restore
+        sql = f"restore mnode on dnode {index}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+        self.check_corrent()
+
+
+    # restore qnode
+    def restore_qnode(self, index):
+        tdLog.info(f"start restore qnode on dnode {index}")
+        dnode = self.dnodes[index - 1]
+        del_dir = f"{dnode.dataDir}/qnode"
+
+        # stop dnode
+        tdLog.info(f"stop dnode {index}")
+        dnode.stoptaosd()
+
+        # remove dnode folder
+        try:
+            shutil.rmtree(del_dir)
+            tdLog.info(f"delete dir {del_dir} successful")
+        except OSError as x:
+            tdLog.exit(f"remove path {del_dir} error : {x.strerror}")
+
+        # start dnode 
+        dnode.starttaosd()
+        
+        # exec restore
+        sql = f"restore qnode on dnode {index}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+        self.check_corrent()
+
+        # path exist
+        qfile = f"{del_dir}/qnode.json"
+        if os.path.exists(qfile) == False:
+            tdLog.exit(f"qnode restore failed. qnode.json is not exist. {qfile}")
+        else:
+            tdLog.info(f"check qnode.json restore ok. {qfile}")
+    
+    # stop
+    def stop(self):
+        tdSql.close()
+
+

tests/system-test/3-enterprise/restore/restoreDnode.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+sys.path.append("./3-enterprise/restore")
+from restoreBasic import *
+
+
+class TDTestCase:
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug("start to execute %s" % __file__)
+        self.basic = RestoreBasic()
+        self.basic.init(conn, logSql, replicaVar)
+    
+    # run
+    def run(self):
+        self.basic.restore_dnode(2)
+
+    # stop
+    def stop(self):
+        self.basic.stop()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/3-enterprise/restore/restoreMnode.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+sys.path.append("./3-enterprise/restore")
+from restoreBasic import *
+
+
+
+class TDTestCase:
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug("start to execute %s" % __file__)
+        self.basic = RestoreBasic()
+        self.basic.init(conn, logSql, replicaVar)
+    
+    # run
+    def run(self):
+        self.basic.restore_mnode(3)
+
+    # stop
+    def stop(self):
+        self.basic.stop()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/3-enterprise/restore/restoreQnode.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+sys.path.append("./3-enterprise/restore")
+from restoreBasic import *
+
+
+class TDTestCase:
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug("start to execute %s" % __file__)
+        self.basic = RestoreBasic()
+        self.basic.init(conn, logSql, replicaVar)
+    
+    # run
+    def run(self):
+        self.basic.restore_qnode(5)
+
+    # stop
+    def stop(self):
+        self.basic.stop()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/3-enterprise/restore/restoreVnode.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+sys.path.append("./3-enterprise/restore")
+from restoreBasic import *
+
+
+class TDTestCase:
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug("start to execute %s" % __file__)
+        self.basic = RestoreBasic()
+        self.basic.init(conn, logSql, replicaVar)
+    
+    # run
+    def run(self):
+        self.basic.restore_vnode(4)
+
+    # stop
+    def stop(self):
+        self.basic.stop()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())