Merge pull request #21229 from taosdata/test/main/TS-3260

test:add testcases of learner in election

tests/pytest/util/cluster.py

@@ -52,8 +52,9 @@ class ConfigureyCluster:
             dnode.addExtraCfg("secondEp", f"{hostname}:{startPort_sec}")
 
             # configure  dnoe of independent mnodes
-            if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True :
-                dnode.addExtraCfg("supportVnodes", 1024)
+            if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == "True" :
+                tdLog.info("set mnode supportVnodes 0")
+                dnode.addExtraCfg("supportVnodes", 0)            
             # print(dnode)
             self.dnodes.append(dnode)
         return self.dnodes
@@ -71,6 +72,7 @@ class ConfigureyCluster:
         tdSql.init(conn.cursor())
         mnodeNums=int(mnodeNums)
         for i in range(2,mnodeNums+1):
+            tdLog.info("create mnode on dnode %d"%i)
             tdSql.execute(" create mnode on  dnode %d;"%i)
 
 

tests/system-test/6-cluster/clusterCommonCheck.py

@@ -207,7 +207,7 @@ class ClusterComCheck:
                 count+=1
         else:
             tdLog.debug(tdSql.queryResult)
-            tdLog.exit("stop mnodes  on dnode %d  failed in 10s ")
+            tdLog.exit(f"stop mnodes  on dnode {offlineDnodeNo}  failed in 10s ")
 
     def check3mnode2off(self,mnodeNums=3):
         count=0
@@ -226,7 +226,45 @@ class ClusterComCheck:
             count+=1
         else:
             tdLog.debug(tdSql.queryResult)
-            tdLog.exit("stop mnodes  on dnode %d  failed in 10s ")
+            tdLog.exit("stop mnodes  on dnode 2 or 3 failed in 10s")
+
+    def check_vgroups_status(self,vgroup_numbers=2,db_replica=3,count_number=10,db_name="db"):
+        """ check vgroups status in 10s after db vgroups status is changed """
+        vgroup_numbers = int(vgroup_numbers)
+        self.db_replica = int(db_replica)
+        tdLog.debug("start to check status of vgroups")
+        count=0
+        last_number=vgroup_numbers-1
+        while count < count_number:
+            time.sleep(1)
+            tdSql.query(f"show  {db_name}.vgroups;")
+            if  count == 0 :
+                if tdSql.checkRows(vgroup_numbers) :
+                    tdLog.success(f"{db_name} has {vgroup_numbers} vgroups" )
+                else:
+                    tdLog.exit(f"vgroup number of {db_name} is not correct")
+            if self.db_replica == 1 :
+                if  tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[1][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader':
+                    ready_time= (count + 1)
+                    tdLog.success(f"all vgroups of {db_name} are leaders in {count + 1} s")
+                    return True
+                count+=1
+            elif self.db_replica == 3 :
+                vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][6],tdSql.queryResult[0][8]]
+                
+                vgroup_status_last=[tdSql.queryResult[last_number][4],tdSql.queryResult[last_number][6],tdSql.queryResult[last_number][8]]
+                if  vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2:
+                    if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2:
+                        ready_time= (count + 1)
+                        tdLog.success(f"all vgroups of {db_name} are ready in {ready_time} s")
+                        return True
+                count+=1
+        else:
+            tdLog.debug(tdSql.queryResult)
+            tdLog.notice(f"all vgroups  leader of {db_name} is selected {count}s ")
+            caller = inspect.getframeinfo(inspect.stack()[1][0])
+            args = (caller.filename, caller.lineno)
+            tdLog.exit("%s(%d) failed " % args)
 
 
 

tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep1to3.py

+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     200,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 1000,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    if i == 0 :
+                        stableName= '%s_%d'%(paraDict['stbName'],0)
+                        newTdSql=tdCom.newTdSql()
+                        # newTdSql.execute('alter database db0_0 replica 3')
+                        clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
+                    tdDnodes[i].stoptaosd()
+                    clusterComCheck.checkDbRows(dbNumbers)
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    if i == 3 :
+                        TdSqlEx=tdCom.newTdSql()
+                        tdLog.info("alter database db0_0 replica 3")
+                        TdSqlEx.execute('alter database db0_0 replica 3')
+ 
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            if i == 0 :
+                tdSql.checkData(0,0,rowsPerStb*2)
+            else:
+                tdSql.checkData(0,0,rowsPerStb)   
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=3,db_name=paraDict["dbName"],count_number=150)     
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDataRebootModifyMetaAlterRep3to1.py

+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    3,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     200,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 1000,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    clusterComCheck.checkDbRows(dbNumbers)
+                    if i == 0 :
+                        stableName= '%s_%d'%(paraDict['stbName'],0)
+                        newTdSql=tdCom.newTdSql()
+                        # newTdSql.execute('alter database db0_0 replica 3')
+                        clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    if i == 3 :
+                        TdSqlEx=tdCom.newTdSql()
+                        tdLog.info("alter database db0_0 replica 1")
+                        TdSqlEx.execute('alter database db0_0 replica 1')
+ 
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            if i == 0 :
+                tdSql.checkData(0,0,rowsPerStb*2)
+            else:
+                tdSql.checkData(0,0,rowsPerStb)   
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=1,db_name=paraDict["dbName"],count_number=150)    
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertDatarRebootAlterRep1-3.py

+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def insertData(self,countstart,countstop):
+        # fisrt add data : db\stable\childtable\general table
+
+        for couti in range(countstart,countstop):
+            tdLog.debug("drop database if exists db%d" %couti)
+            tdSql.execute("drop database if exists db%d" %couti)
+            print("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("use db%d" %couti)
+            tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+            )
+            tdSql.execute(
+                '''
+                create table t1
+                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+                '''
+            )
+            for i in range(4):
+                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 100,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers = int(dnodeNumbers)
+        mnodeNums = int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers = (paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb = paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall = rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+        replica3 = 3
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        TdSqlEx=tdCom.newTdSql()
+        tdLog.info("alter database db0_0 replica 3")
+        TdSqlEx.execute('alter database db0_0 replica 3')
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    # tdLog.info('select  cast(c2 as nchar(10)) from db0_0.stb_1;')
+                    # TdSqlEx.execute('select  cast(c2 as nchar(10)) from db0_0.stb_1;')
+                    # tdLog.info('select  avg(c1)  from db0_0.stb_0 interval(10s);')
+                    # TdSqlEx.execute('select  avg(c1)  from db0_0.stb_0 interval(10s);')
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+        for tr in threads:
+            tr.join()
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        # for i in range(paraDict['stbNumbers']):
+        #     stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+        #     tdSql.query("select count(*) from %s"%stableName)
+        #     tdSql.checkData(0,0,rowsPerStb)
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=240)        
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py

+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def insertData(self,countstart,countstop):
+        # fisrt add data : db\stable\childtable\general table
+
+        for couti in range(countstart,countstop):
+            tdLog.debug("drop database if exists db%d" %couti)
+            tdSql.execute("drop database if exists db%d" %couti)
+            print("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("use db%d" %couti)
+            tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+            )
+            tdSql.execute(
+                '''
+                create table t1
+                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+                '''
+            )
+            for i in range(4):
+                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    3,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 1,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        replica1 = 1
+        replica3 = 3
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        TdSqlEx=tdCom.newTdSql()
+        tdLog.info(f"alter database db0_0 replica {replica1}")
+        TdSqlEx.execute(f'alter database db0_0 replica {replica1}')
+        for tr in threads:
+            tr.join()
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            tdSql.checkData(0,0,rowsPerStb)
+        
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica1,db_name=paraDict["dbName"],count_number=20)
+        sleep(5)
+        tdLog.info(f"show transactions;alter database db0_0 replica {replica3};")
+        TdSqlEx.execute(f'show transactions;')
+        TdSqlEx.execute(f'alter database db0_0 replica {replica3};')
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=replica3,db_name=paraDict["dbName"],count_number=120)
+
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/manually-test/6dnode3mnodeStopDnodeInsertDatatb.py

+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def insertData(self,dbname,tableCount,rowsPerCount):
+        # tableCount : create table number
+        # rowsPerCount :  rows per table
+        # fisrt add data : db\stable\childtable\general table
+        os.system(f"taosBenchmark  -d {dbname} -n {tableCount} -t {rowsPerCount}  -z 1 -k 10000 -y ")
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     10000,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 10000,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        threads.append(threading.Thread(target=self.insertData, args=(paraDict["dbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"])))
+        for tr in threads:
+            tr.start()
+        TdSqlEx=tdCom.newTdSql()
+        tdLog.info("alter database db0_0 replica 3")
+        TdSqlEx.execute('alter database db0_0 replica 3')
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    # tdLog.info('select  cast(c2 as nchar(10)) from db0_0.stb_1;')
+                    # TdSqlEx.execute('select  cast(c2 as nchar(10)) from db0_0.stb_1;')
+                    # tdLog.info('select  avg(c1)  from db0_0.stb_0 interval(10s);')
+                    # TdSqlEx.execute('select  avg(c1)  from db0_0.stb_0 interval(10s);')
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+        for tr in threads:
+            tr.join()
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            tdSql.checkData(0,0,rowsPerStb)
+        clusterComCheck.check_vgroups_status(vgroup_numbers=paraDict["vgroups"],db_replica=3,db_name=paraDict["dbName"],count_number=240)             
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())