Merge pull request #18977 from taosdata/TD-20251

test:add testcase of mnode clusters

Jenkinsfile2

@@ -173,7 +173,7 @@ def pre_test_build_mac() {
     '''
     sh '''
         cd ${WK}/debug
-        cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
+        cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false  -DCMAKE_BUILD_TYPE=Release
         make -j10
         ctest -j10 || exit 7
     '''

tests/parallel_test/cases.task

@@ -648,6 +648,9 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3
+
+,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py  -N 6 -M 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py  -N 6 -M 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3
@@ -658,6 +661,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3
+
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5

tests/pytest/util/cluster.py

@@ -53,7 +53,7 @@ class ConfigureyCluster:
 
             # configure  dnoe of independent mnodes
             if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True :
-                dnode.addExtraCfg("supportVnodes", 0)
+                dnode.addExtraCfg("supportVnodes", 1024)
             # print(dnode)
             self.dnodes.append(dnode)
         return self.dnodes

tests/system-test/0-others/compatibility.py

@@ -68,6 +68,7 @@ class TDTestCase:
         my_file = Path(f"{packagePath}/{packageName}")
         if not  my_file.exists():
             print(f"{packageName} is not exists")
+            tdLog.info(f"cd {packagePath} &&  wget https://www.tdengine.com/assets-download/3.0/{packageName}")
             os.system(f"cd {packagePath} &&  wget https://www.tdengine.com/assets-download/3.0/{packageName}")
         else: 
             print(f"{packageName} has been exists")

tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py

@@ -161,6 +161,19 @@ class TDTestCase:
             stableName= '%s_%d'%(paraDict['stbName'],i)
             newTdSql=tdCom.newTdSql()
             threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        
+        for i in range(5):
+            clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}")
+            userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}")
+            clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}")
+            clusterComCreate.deleteUser(newTdSql,f"user{i}")
+        for j in range(5):
+            i=100
+            clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}")
+            userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}")
+            clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}")
+            clusterComCreate.deleteUser(newTdSql,f"user{i}")
+
         for tr in threads:
             tr.start()
         for tr in threads:

tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py

@@ -67,29 +67,11 @@ class TDTestCase:
         self._async_raise(thread.ident, SystemExit)
 
 
-    def insertData(self,countstart,countstop):
-        # fisrt add data : db\stable\childtable\general table
-
-        for couti in range(countstart,countstop):
-            tdLog.debug("drop database if exists db%d" %couti)
-            tdSql.execute("drop database if exists db%d" %couti)
-            print("create database if not exists db%d replica 1 duration 300" %couti)
-            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
-            tdSql.execute("use db%d" %couti)
-            tdSql.execute(
-            '''create table stb1
-            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-            tags (t1 int)
-            '''
-            )
-            tdSql.execute(
-                '''
-                create table t1
-                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-                '''
-            )
-            for i in range(4):
-                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+    def reCreateUser(self, tdsql, count, user, passwd):
+        clusterComCreate.createUser(tdsql,f"{user}{count}",f"{passwd}{count}")
+        userTdSql=tdCom.newTdSql(user=f"{user}{count}",password=f"{passwd}{count}")
+        clusterComCreate.alterUser(userTdSql,f"{user}{count}",f"{passwd}{count+1}")
+        clusterComCreate.deleteUser(tdsql,f"{user}{count}")
 
 
     def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
@@ -161,6 +143,8 @@ class TDTestCase:
             stableName= '%s_%d'%(paraDict['stbName'],i)
             newTdSql=tdCom.newTdSql()
             threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+            threads.append(threading.Thread(target=self.reCreateUser,args=(newTdSql,i,"user","passwd")))
+
         for tr in threads:
             tr.start()
 

tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py

@@ -147,6 +147,9 @@ class TDTestCase:
             # print(f"==================={dbNameIndex},{a11111}")
             threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])))
 
+            redbNameIndex = '%s%d'%(paraDict["dbName"],i+100)
+            threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])))
+
         for tr in threads:
             tr.start()
 
@@ -199,7 +202,7 @@ class TDTestCase:
 
     def run(self):
         # print(self.master_dnode.cfgDict)
-        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode')
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode')
 
     def stop(self):
         tdSql.close()

tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py

@@ -97,7 +97,7 @@ class TDTestCase:
                     'dropFlag':   1,
                     'event':      '',
                     'vgroups':    4,
-                    'replica':    1,
+                    'replica':    3,
                     'stbName':    'stb',
                     'stbNumbers': 2,
                     'colPrefix':  'c',
@@ -105,9 +105,9 @@ class TDTestCase:
                     'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
                     'ctbPrefix':  'ctb',
-                    'ctbNum':     200,
+                    'ctbNum':     100,
                     'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    "rowsPerTbl": 10000,
+                    "rowsPerTbl": 100000,
                     "batchNum": 5000
                     }
 
@@ -198,16 +198,16 @@ class TDTestCase:
         clusterComCheck.checkDbRows(dbNumbers)
         # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
 
-        tdSql.execute("use %s" %(paraDict["dbName"]))
-        tdSql.query("show stables")
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
         tdSql.checkRows(paraDict["stbNumbers"])
         for i in range(paraDict['stbNumbers']):
-            stableName= '%s_%d'%(paraDict['stbName'],i)
-            tdSql.query("select * from %s"%stableName)
-            tdSql.checkRows(rowsPerStb)
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            tdSql.checkData(0,0,rowsPerStb)
     def run(self):
         # print(self.master_dnode.cfgDict)
-        self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode')
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode')
 
     def stop(self):
         tdSql.close()

tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py

+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+from numpy import row_stack
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    3,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     200,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 1000,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    clusterComCheck.checkDbRows(dbNumbers)
+                    if i == 0 :
+                        stableName= '%s_%d'%(paraDict['stbName'],0)
+                        newTdSql=tdCom.newTdSql()
+                        clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+ 
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            if i == 0 :
+                tdSql.checkData(0,0,rowsPerStb*2)
+            else:
+                tdSql.checkData(0,0,rowsPerStb)   
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py → tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py

@@ -10,9 +10,9 @@ from util.cases import *
 from util.dnodes import TDDnodes
 from util.dnodes import TDDnode
 from util.cluster import *
-from util.common import *
-sys.path.append("./7-tmq")
-from tmqCommon import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
 
 import time
 import socket
@@ -27,8 +27,10 @@ class TDTestCase:
 
     def init(self, conn, logSql, replicaVar=1):
         tdLog.debug(f"start to excute {__file__}")
-        # tdSql.init(conn.cursor())
-        # self.host = socket.gethostname()
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+        self.replicaVar=int(replicaVar)
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -59,11 +61,11 @@ class TDTestCase:
             ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
             raise SystemError("PyThreadState_SetAsyncExc failed")
 
-    def stop_thread(self,thread):
+    def stopThread(self,thread):
         self._async_raise(thread.ident, SystemExit)
 
 
-    def insert_data(self,countstart,countstop):
+    def insertData(self,countstart,countstop):
         # fisrt add data : db\stable\childtable\general table
 
         for couti in range(countstart,countstop):
@@ -87,234 +89,117 @@ class TDTestCase:
             for i in range(4):
                 tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
 
-    def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
-        tdSql.execute("use %s"%dbname)
-        tdSql.query("show stables")
-        tdSql.checkRows(stableCount)
-        tdSql.query("show tables")
-        tdSql.checkRows(CtableCount)
-        for i in range(stableCount):
-            tdSql.query("select count(*) from %s%d"%(stbname,i))
-            tdSql.checkData(0,0,rowsPerSTable)
-        return
-
-    def checkdnodes(self,dnodenumber):
-        count=0
-        while count < 100:
-            time.sleep(1)
-            statusReadyBumber=0
-            tdSql.query("select * from information_schema.ins_dnodes;")
-            if tdSql.checkRows(dnodenumber) :
-                print("dnode is %d nodes"%dnodenumber)
-            for i in range(dnodenumber):
-                if tdSql.queryResult[i][4] !='ready'  :
-                    status=tdSql.queryResult[i][4]
-                    print("dnode:%d status is %s "%(i,status))
-                    break
-                else:
-                    statusReadyBumber+=1
-            print(statusReadyBumber)
-            if statusReadyBumber == dnodenumber :
-                print("all of %d mnodes is ready in 10s "%dnodenumber)
-                return True
-                break
-            count+=1
-        else:
-            print("%d mnodes is not ready in 10s "%dnodenumber)
-            return False
-
-
-    def check3mnode(self):
-        count=0
-        while count < 10:
-            time.sleep(1)
-            tdSql.query("select * from information_schema.ins_mnodes;")
-            if tdSql.checkRows(3) :
-                print("mnode is  three nodes")
-            if  tdSql.queryResult[0][2]=='leader' :
-                if  tdSql.queryResult[1][2]=='follower':
-                    if  tdSql.queryResult[2][2]=='follower':
-                        print("three mnodes is ready in 10s")
-                        break
-            elif tdSql.queryResult[0][2]=='follower' :
-                if  tdSql.queryResult[1][2]=='leader':
-                    if  tdSql.queryResult[2][2]=='follower':
-                        print("three mnodes is ready in 10s")
-                        break
-            elif tdSql.queryResult[0][2]=='follower' :
-                if  tdSql.queryResult[1][2]=='follower':
-                    if  tdSql.queryResult[2][2]=='leader':
-                        print("three mnodes is ready in 10s")
-                        break
-            count+=1
-        else:
-            print("three mnodes is not ready in 10s ")
-            return -1
-
-        tdSql.query("select * from information_schema.ins_mnodes;")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(0,3,'ready')
-        tdSql.checkData(1,1,'%s:6130'%self.host)
-        tdSql.checkData(1,3,'ready')
-        tdSql.checkData(2,1,'%s:6230'%self.host)
-        tdSql.checkData(2,3,'ready')
-
-    def check3mnode1off(self):
-        count=0
-        while count < 10:
-            time.sleep(1)
-            tdSql.query("select * from information_schema.ins_mnodes;")
-            if tdSql.checkRows(3) :
-                print("mnode is  three nodes")
-            if  tdSql.queryResult[0][2]=='offline' :
-                if  tdSql.queryResult[1][2]=='leader':
-                    if  tdSql.queryResult[2][2]=='follower':
-                        print("stop mnodes  on dnode 2 successfully in 10s")
-                        break
-                elif tdSql.queryResult[1][2]=='follower':
-                    if  tdSql.queryResult[2][2]=='leader':
-                        print("stop mnodes  on dnode 2 successfully in 10s")
-                        break
-            count+=1
-        else:
-            print("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
-        tdSql.error("drop mnode on dnode 1;")
-
-        tdSql.query("select * from information_schema.ins_mnodes;")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(0,2,'offline')
-        tdSql.checkData(0,3,'ready')
-        tdSql.checkData(1,1,'%s:6130'%self.host)
-        tdSql.checkData(1,3,'ready')
-        tdSql.checkData(2,1,'%s:6230'%self.host)
-        tdSql.checkData(2,3,'ready')
-
-    def check3mnode2off(self):
-        count=0
-        while count < 40:
-            time.sleep(1)
-            tdSql.query("select * from information_schema.ins_mnodes;")
-            if tdSql.checkRows(3) :
-                print("mnode is  three nodes")
-            if  tdSql.queryResult[0][2]=='leader' :
-                if  tdSql.queryResult[1][2]=='offline':
-                    if  tdSql.queryResult[2][2]=='follower':
-                        print("stop mnodes  on dnode 2 successfully in 10s")
-                        break
-            count+=1
-        else:
-            print("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
-        tdSql.error("drop mnode on dnode 2;")
-
-        tdSql.query("select * from information_schema.ins_mnodes;")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(0,2,'leader')
-        tdSql.checkData(0,3,'ready')
-        tdSql.checkData(1,1,'%s:6130'%self.host)
-        tdSql.checkData(1,2,'offline')
-        tdSql.checkData(1,3,'ready')
-        tdSql.checkData(2,1,'%s:6230'%self.host)
-        tdSql.checkData(2,2,'follower')
-        tdSql.checkData(2,3,'ready')
-
-    def check3mnode3off(self):
-        count=0
-        while count < 10:
-            time.sleep(1)
-            tdSql.query("select * from information_schema.ins_mnodes;")
-            if tdSql.checkRows(3) :
-                print("mnode is  three nodes")
-            if  tdSql.queryResult[0][2]=='leader' :
-                if  tdSql.queryResult[2][2]=='offline':
-                    if  tdSql.queryResult[1][2]=='follower':
-                        print("stop mnodes  on dnode 3 successfully in 10s")
-                        break
-            count+=1
-        else:
-            print("stop mnodes  on dnode 3 failed in 10s")
-            return -1
-        tdSql.error("drop mnode on dnode 3;")
-        tdSql.query("select * from information_schema.ins_mnodes;")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(0,2,'leader')
-        tdSql.checkData(0,3,'ready')
-        tdSql.checkData(1,1,'%s:6130'%self.host)
-        tdSql.checkData(1,2,'follower')
-        tdSql.checkData(1,3,'ready')
-        tdSql.checkData(2,1,'%s:6230'%self.host)
-        tdSql.checkData(2,2,'offline')
-        tdSql.checkData(2,3,'ready')
-
 
-    def check5dnode(self):
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db',
+                    'dbNumbers':   4,
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    3,
+                    'stbName':    'stb',
+                    'stbNumbers': 100,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1,
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        dbNumbers=paraDict['dbNumbers']
+        mnodeNums=int(mnodeNums)
+        repeatNumber = 2
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allDbNumbers=dbNumbers
+        allStbNumbers=(paraDict['stbNumbers']*restartNumbers)
+        paraDict['replica'] = self.replicaVar
+
+        tdLog.info("first check dnode and mnode")
         tdSql.query("select * from information_schema.ins_dnodes;")
         tdSql.checkData(0,1,'%s:6030'%self.host)
         tdSql.checkData(4,1,'%s:6430'%self.host)
-        tdSql.checkData(0,4,'ready')
-        tdSql.checkData(4,4,'ready')
-
-    def five_dnode_three_mnode(self,dnodenumber):
-        tdSql.query("select * from information_schema.ins_dnodes;")
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(4,1,'%s:6430'%self.host)
-        tdSql.checkData(0,4,'ready')
-        tdSql.checkData(4,4,'ready')
-        tdSql.query("select * from information_schema.ins_mnodes;")
-        tdSql.checkRows(1)
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(0,2,'leader')
-        tdSql.checkData(0,3,'ready')
-
-        # fisr add three mnodes;
-        tdSql.execute("create mnode on dnode 2")
-        tdSql.execute("create mnode on dnode 3")
-
-        # fisrt check statut ready
-        self.check3mnode()
-
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
         tdSql.error("create mnode on dnode 2")
         tdSql.query("select * from information_schema.ins_dnodes;")
         print(tdSql.queryResult)
-        tdLog.debug("stop all of mnode ")
+        clusterComCheck.checkDnodes(dnodeNumbers)
 
-        # seperate vnode and mnode in different dnodes.
         # create database and stable
+
+
+        tdDnodes=cluster.dnodes
         stopcount =0
-        while stopcount < 2:
-            for i in range(dnodenumber):
-                # threads=[]
-                # threads = MyThreadFunc(self.insert_data(i*2,i*2+2))
-                threads=threading.Thread(target=self.insert_data, args=(i,i+1))
-                threads.start()
-                self.TDDnodes.stoptaosd(i+1)
-                self.TDDnodes.starttaosd(i+1)
-
-                if self.checkdnodes(5):
-                    print("123")
-                    threads.join()
-                else:
-                    print("456")
-                    threads.join()
-                    self.stop_thread(threads)
-                    assert 1 == 2 ,"some dnode started failed"
-                    return False
+        threads=[]
+        for i in range(dbNumbers):
+            dbNameIndex = '%s%d'%(paraDict["dbName"],0)
+            newTdSql=tdCom.newTdSql()
+            # a11111=paraDict["dbNumbers"]
+            # print(f"==================={dbNameIndex},{a11111}")
+            clusterComCreate.createDeltedatabases(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+
+            redbNameIndex = '%s%d'%(paraDict["dbName"],100)
+            clusterComCreate.createDeltedatabases(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+
+
+        tdLog.info("Take turns stopping Mnodes ")
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("check dnodes status is ready")
+            else:
+                tdLog.info("check dnodes status is not ready")
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
                 # self.check3mnode()
-            self.check3mnode()
+            stopcount+=1
 
 
-            stopcount+=1
-        self.check3mnode()
+        tdLog.info("check dnode number:")
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        tdSql.query("select * from information_schema.ins_databases")
+        tdLog.debug("we find %d databases but exepect to create %d  databases "%(tdSql.queryRows-2,allDbNumbers))
+
+        # tdLog.info("check DB Rows:")
+        # clusterComCheck.checkDbRows(allDbNumbers)
+        # tdLog.info("check DB Status on by on")
+        # for i in range(restartNumbers):
+        #     clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i))
+
 
 
     def run(self):
         # print(self.master_dnode.cfgDict)
-        self.five_dnode_three_mnode(5)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode')
 
     def stop(self):
         tdSql.close()

tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py

+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+from numpy import row_stack
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    3,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     200,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 1000,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert date
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    if i == 0 :
+                        stableName= '%s_%d'%(paraDict['stbName'],0)
+                        newTdSql=tdCom.newTdSql()
+                        clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"])
+                    elif i == 1 :
+                        tdSql.execute("ALTER TABLE db0_0.stb_0_0 SET TAG t1r=10000;")
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    clusterComCheck.checkDbRows(dbNumbers)
+
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+ 
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("123")
+            else:
+                print("456")
+
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        # tdSql.execute("use %s" %(paraDict["dbName"]))
+        tdSql.query("select t1r from db0_0.stb_0_0  limit 1;")
+        tdSql.checkData(0,0,10000)
+        tdSql.query("show %s.stables"%(paraDict["dbName"]))
+        tdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i)
+            tdSql.query("select count(*) from %s"%stableName)
+            if i == 0 :
+                tdSql.checkData(0,0,rowsPerStb*2)
+            else:
+                tdSql.checkData(0,0,rowsPerStb)   
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py

+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+from numpy import row_stack
+import taos
+import sys
+import time
+import os
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+from util.cluster import *
+sys.path.append("./6-cluster")
+from clusterCommonCreate import *
+from clusterCommonCheck import clusterComCheck
+
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading
+import time
+import inspect
+import ctypes
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
+        self.replicaVar =  int(replicaVar)
+
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stopThread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def insertData(self,countstart,countstop):
+        # fisrt add data : db\stable\childtable\general table
+
+        for couti in range(countstart,countstop):
+            tdLog.debug("drop database if exists db%d" %couti)
+            tdSql.execute("drop database if exists db%d" %couti)
+            print("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("use db%d" %couti)
+            tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+            )
+            tdSql.execute(
+                '''
+                create table t1
+                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+                '''
+            )
+            for i in range(4):
+                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+
+    def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db0_0',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'replica':    1,
+                    'stbName':    'stb',
+                    'stbNumbers': 2,
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     200,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    "rowsPerTbl": 100,
+                    "batchNum": 5000
+                    }
+
+        dnodeNumbers=int(dnodeNumbers)
+        mnodeNums=int(mnodeNums)
+        vnodeNumbers = int(dnodeNumbers-mnodeNums)
+        allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"])
+        rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"]
+        rowsall=rowsPerStb*paraDict['stbNumbers']
+        dbNumbers = 1
+        paraDict['replica'] = self.replicaVar
+
+        tdLog.info("first check dnode and mnode")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        
+        #check mnode status
+        tdLog.info("check mnode status")
+        clusterComCheck.checkMnodeStatus(mnodeNums)
+        
+
+        # add some error operations and
+        tdLog.info("Confirm the status of the dnode again")
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("select * from information_schema.ins_dnodes;")
+        print(tdSql.queryResult)
+        clusterComCheck.checkDnodes(dnodeNumbers)
+
+        # create database and stable
+        clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])
+        tdLog.info("Take turns stopping Mnodes ")
+
+        tdDnodes=cluster.dnodes
+        stopcount =0
+        threads=[]
+
+        # create stable:stb_0
+        stableName= paraDict['stbName']
+        newTdSql=tdCom.newTdSql()
+        clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers'])
+        #create child table:ctb_0
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum'])
+        #insert data
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql=tdCom.newTdSql()
+            threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])))
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+
+        while stopcount < restartNumbers:
+            tdLog.info(" restart loop: %d"%stopcount )
+            if stopRole == "mnode":
+                for i in range(mnodeNums):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+            elif stopRole == "vnode":
+                for i in range(vnodeNumbers):
+                    tdDnodes[i+mnodeNums].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i+mnodeNums].starttaosd()
+                    # sleep(10)
+            elif stopRole == "dnode":
+                for i in range(dnodeNumbers):
+                    tdDnodes[i].stoptaosd()
+                    # sleep(10)
+                    tdDnodes[i].starttaosd()
+                    # sleep(10)
+
+            # dnodeNumbers don't include database of schema
+            if clusterComCheck.checkDnodes(dnodeNumbers):
+                tdLog.info("dnode is ready")
+            else:
+                print("dnodes is not  ready")
+                self.stopThread(threads)
+                tdLog.exit("one or more of dnodes failed to start ")
+                # self.check3mnode()
+            stopcount+=1
+
+
+        clusterComCheck.checkDnodes(dnodeNumbers)
+        clusterComCheck.checkDbRows(dbNumbers)
+        # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"])
+
+        newTdSql=tdCom.newTdSql()
+        newTdSql.execute("reset query cache")
+        newTdSql.execute("use %s" %(paraDict["dbName"]))
+        newTdSql.query("show %s.stables"%(paraDict["dbName"]))
+        newTdSql.checkRows(paraDict["stbNumbers"])
+        for i in range(paraDict['stbNumbers']):
+            stableName= '%s_%d'%(paraDict['stbName'],i)
+            newTdSql.query("select * from %s"%stableName)
+            newTdSql.checkRows(rowsPerStb)
+
+    def run(self):
+        # print(self.master_dnode.cfgDict)
+        self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode')
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/6-cluster/clusterCommonCheck.py

@@ -64,7 +64,7 @@ class ClusterComCheck:
         dbNumbers=int(dbNumbers)
         count=0
         while count < 5:
-            tdSql.query("select * from information_schema.ins_databases;")
+            tdSql.query("select * from information_schema.ins_databases where name!='collectd' ;")
             count+=1
             if tdSql.checkRows(dbNumbers+2):
                 tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2))

tests/system-test/6-cluster/clusterCommonCreate.py

@@ -137,12 +137,24 @@ class ClusterComCreate:
         # for i in range(dbNumbers):
         for i in range(dbNumbers):
             if dropFlag == 1:
-                tsql.execute("drop database if exists %s_%d"%(dbNameIndex,1))
-            tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica))
-            tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica))
-            tdLog.debug("complete to create database %s_%d"%(dbNameIndex,1))
+                tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i))
+            tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica))
+            tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica))
+            tdLog.debug("complete to create database %s_%d"%(dbNameIndex,i))
     
+    def createUser(self,tsql,user,password):
+        tdLog.info(f"create new user f{user}")   
+        tsql.execute(f"CREATE USER {user} PASS '{password}';")
 
+    def alterUser(self,tsql,user,password):
+        tdLog.info(f"alter user {user} pass '{password}'")   
+        tsql.execute(f"alter USER {user}  pass '{password}' ;")
+
+    def deleteUser(self,tsql,user):
+        tdLog.info(f"drop user f{user}")   
+        tsql.execute(f"DROP USER {user} ;")
+
+        
     def create_stable(self,tsql, dbName,stbName):
         tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName))
         tdLog.debug("complete to create %s.%s" %(dbName, stbName))
@@ -202,6 +214,52 @@ class ClusterComCreate:
         tdLog.debug("insert data ............ [OK]")
         return
 
+    def alterStbMetaData(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None):
+        tdLog.debug("alter Stb column ............")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP TAG t2;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP TAG t2;")
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32) ;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32);")        
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY TAG t2 binary(34) ;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName}  MODIFY TAG t2 binary(34);")    
+        tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t3 double ;")
+        tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t3 double;")  
+        tsql.error(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 double;")  
+
+        tdLog.debug("start to insert data ............")
+        # tsql.execute("use %s" %dbName)
+        pre_insert = "insert into "
+        sql = pre_insert
+
+        if startTs is None:
+            t = time.time()
+            startTs = int(round(t * 1000))
+        #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+        for i in range(ctbNum):
+            sql += " %s.%s_%d values "%(dbName,stbName,i)
+            for j in range(rowsPerTbl):
+                sql += "(%d, %d,'mnode_%d',  %d ) "%(startTs + j, j, j,j)
+                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                    tsql.execute(sql)
+                    if j < rowsPerTbl - 1:
+                        sql = "insert into %s.%s_%d values " %(dbName,stbName,i)
+                    else:
+                        sql = "insert into "
+        #end sql
+        if sql != pre_insert:
+            #print("insert sql:%s"%sql)
+            tsql.execute(sql)
+        tdLog.debug("insert data ............ [OK]")
+        return
+
     def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)