Merge pull request #14104 from taosdata/test/chr/TD-14699

test: modify testcase of muti-mnodes

tests/pytest/util/cluster.py

+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os 
+import  socket
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+class ClusterDnodes(TDDnodes):
+    """rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
+    def __init__(self ,dnodes_lists):
+
+        super(ClusterDnodes,self).__init__()
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        self.simDeployed = False
+        self.testCluster = False
+        self.valgrind = 0
+        self.killValgrind = 1
+        self.independent = True
+        self.dnodeNums = 5
+
+    # def getTDDnodes(dnodeNums):
+
+    #     return 
+
+
+class ConfigureyCluster:
+    """configure dnodes and return TDDnodes list, it can """
+
+    def __init__(self):
+        self.dnodes = None      
+        self.dnodes_nums = 5
+        self.independent = True
+        self.start_port = 6030
+        self.portStep = 100
+    hostname1= socket.gethostname()
+
+    def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): 
+        self.start_port=int(start_port)
+        self.portStep=int(portStep)
+        self.hostname=hostname
+        self.dnodes_nums = int(dnodes_nums)
+        self.independent = independent
+        self.dnodes = []
+        start_port_sec = 6130
+        for num in range(1, (self.dnodes_nums+1)):
+            dnode = TDDnode(num)
+            dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}")
+            dnode.addExtraCfg("fqdn", f"{hostname}")
+            dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}")
+            # dnode.addExtraCfg("monitorFqdn", hostname)
+            # dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
+            # configure three dnoe don't support vnodes
+            if self.dnodes_nums > 4 :
+                if self.independent and (num < 4):
+                    dnode.addExtraCfg("supportVnodes", 0)
+            # print(dnode)
+            self.dnodes.append(dnode)
+        return self.dnodes
+        
+    def create_dnode(self,conn):
+        tdSql.init(conn.cursor())
+        for dnode in self.dnodes[1:]:
+            # print(dnode.cfgDict)
+            dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
+            print(dnode_id)
+            tdSql.execute(" create dnode '%s';"%dnode_id)
+        # count=0
+        # while count < 10:
+        #     time.sleep(1)
+        #     tdSql.query("show dnodes;")
+        #     if tdSql.checkRows(self.dnodes_nums) :
+        #         print("mnode is  three nodes")
+        #     if  tdSql.queryResult[0][4]=='leader' :
+        #         if  tdSql.queryResult[2][4]=='offline':
+        #             if  tdSql.queryResult[1][2]=='follower':
+        #                 print("stop mnodes  on dnode 3 successfully in 10s")
+        #                 break
+        #     count+=1
+        # else:
+        #     print("stop mnodes  on dnode 3 failed in 10s")
+        #     return -1
+        checkstatus=False
+
+        
+        
+    def check_dnode(self,conn):
+        tdSql.init(conn.cursor())
+        count=0
+        while count < 5:
+            tdSql.query("show dnodes")
+            # tdLog.debug(tdSql.queryResult)
+            status=0
+            for i in range(self.dnodes_nums):
+                if tdSql.queryResult[i][4] == "ready":
+                    status+=1
+            tdLog.debug(status)
+            
+            if status == self.dnodes_nums:
+                tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums)
+                break 
+            count+=1
+            time.sleep(1)
+        else:
+            tdLog.debug("create cluster with %d dnode but  check dnode not ready within 5s ! "%self.dnodes_nums)
+            return -1
+
+
+cluster = ConfigureyCluster()
+
+    # def start(self ,dnodes_nums): 
+
+    #     self.TDDnodes = ClusterDnodes(dnodes)
+    #     self.TDDnodes.init("")
+    #     self.TDDnodes.setTestCluster(testCluster)
+    #     self.TDDnodes.setValgrind(valgrind)
+    #     self.TDDnodes.stopAll()
+    #     for dnode in self.TDDnodes.dnodes:
+    #         self.TDDnodes.deploy(dnode.index,{})
+            
+    #     for dnode in self.TDDnodes.dnodes:
+    #         self.TDDnodes.starttaosd(dnode.index)
+
+    #     # create cluster 
+    #     for dnode in self.TDDnodes.dnodes[1:]:
+    #         # print(dnode.cfgDict)
+    #         dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
+    #         dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
+    #         dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
+    #         cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+    #         print(cmd)
+    #         os.system(cmd)
+        
+    #     time.sleep(2)
+    #     tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
+
+    # def buildcluster(self,dnodenumber):
+    #     self.depoly_cluster(dnodenumber)
+    #     self.master_dnode = self.TDDnodes.dnodes[0]
+    #     self.host=self.master_dnode.cfgDict["fqdn"]
+    #     conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
+    #     tdSql.init(conn1.cursor())
+        
+    # def checkdnodes(self,dnodenumber):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         statusReadyBumber=0
+    #         tdSql.query("show dnodes;")
+    #         if tdSql.checkRows(dnodenumber) :     
+    #             print("dnode is %d nodes"%dnodenumber)   
+    #         for i in range(dnodenumber):
+    #             if tdSql.queryResult[i][4] !='ready'  :
+    #                 status=tdSql.queryResult[i][4]
+    #                 print("dnode:%d status is %s "%(i,status))
+    #                 break
+    #             else:
+    #                 statusReadyBumber+=1
+    #         print(statusReadyBumber)
+    #         if statusReadyBumber == dnodenumber :
+    #             print("all of %d mnodes is ready in 10s "%dnodenumber)
+    #             return True
+    #             break
+    #         count+=1
+    #     else:
+    #         print("%d mnodes is not ready in 10s "%dnodenumber)
+    #         return False
+           
+
+    # def check3mnode(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :     
+    #             print("mnode is  three nodes")           
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("three mnodes is ready in 10s")
+    #                     break
+    #         elif tdSql.queryResult[0][2]=='follower' :
+    #             if  tdSql.queryResult[1][2]=='leader':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("three mnodes is ready in 10s")
+    #                     break      
+    #         elif tdSql.queryResult[0][2]=='follower' :
+    #             if  tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='leader':
+    #                     print("three mnodes is ready in 10s")
+    #                     break                   
+    #         count+=1
+    #     else:
+    #         print("three mnodes is not ready in 10s ")
+    #         return -1
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode1off(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='offline' :
+    #             if  tdSql.queryResult[1][2]=='leader':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #             elif tdSql.queryResult[1][2]=='follower':
+    #                 if  tdSql.queryResult[2][2]=='leader':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 2 failed in 10s ")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 1;")
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'offline')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode2off(self):
+    #     count=0
+    #     while count < 40:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[1][2]=='offline':
+    #                 if  tdSql.queryResult[2][2]=='follower':
+    #                     print("stop mnodes  on dnode 2 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 2 failed in 10s ")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 2;")
+
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'leader')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,2,'offline')
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,2,'follower')
+    #     tdSql.checkData(2,3,'ready')
+
+    # def check3mnode3off(self):
+    #     count=0
+    #     while count < 10:
+    #         time.sleep(1)
+    #         tdSql.query("show mnodes;")
+    #         if tdSql.checkRows(3) :
+    #             print("mnode is  three nodes")
+    #         if  tdSql.queryResult[0][2]=='leader' :
+    #             if  tdSql.queryResult[2][2]=='offline':
+    #                 if  tdSql.queryResult[1][2]=='follower':
+    #                     print("stop mnodes  on dnode 3 successfully in 10s")
+    #                     break
+    #         count+=1
+    #     else:
+    #         print("stop mnodes  on dnode 3 failed in 10s")
+    #         return -1
+    #     tdSql.error("drop mnode on dnode 3;")
+    #     tdSql.query("show mnodes;")       
+    #     tdSql.checkRows(3) 
+    #     tdSql.checkData(0,1,'%s:6030'%self.host)
+    #     tdSql.checkData(0,2,'leader')
+    #     tdSql.checkData(0,3,'ready')
+    #     tdSql.checkData(1,1,'%s:6130'%self.host)
+    #     tdSql.checkData(1,2,'follower')
+    #     tdSql.checkData(1,3,'ready')
+    #     tdSql.checkData(2,1,'%s:6230'%self.host)
+    #     tdSql.checkData(2,2,'offline')
+    #     tdSql.checkData(2,3,'ready')
+

tests/system-test/6-cluster/5dnode3mnodeStop.py

@@ -7,30 +7,24 @@ import os
 from util.log import *
 from util.sql import *
 from util.cases import *
+from util.dnodes import *
 from util.dnodes import TDDnodes
 from util.dnodes import TDDnode
+from util.cluster import *
+from test import tdDnodes
+
 import time
 import socket
 import subprocess
 from multiprocessing import Process
-class MyDnodes(TDDnodes):
-    def __init__(self ,dnodes_lists):
-        super(MyDnodes,self).__init__()
-        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
-        self.simDeployed = False
+
         
 class TDTestCase:
 
     def init(self,conn ,logSql):
         tdLog.debug(f"start to excute {__file__}")
-        self.TDDnodes = None
-
-    def buildcluster(self,dnodenumber):
-        self.depoly_cluster(dnodenumber)
-        self.master_dnode = self.TDDnodes.dnodes[0]
-        self.host=self.master_dnode.cfgDict["fqdn"]
-        conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
-        tdSql.init(conn1.cursor())
+        tdSql.init(conn.cursor())
+        self.host = socket.gethostname()
 
 
     def getBuildPath(self):
@@ -70,45 +64,6 @@ class TDTestCase:
             for i in range(4):
                 tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
 
-    def depoly_cluster(self ,dnodes_nums): 
-
-        testCluster = False
-        valgrind = 0  
-        hostname = socket.gethostname()
-        dnodes = []
-        start_port = 6030
-        for num in range(1, dnodes_nums+1):
-            dnode = TDDnode(num)
-            dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
-            dnode.addExtraCfg("fqdn", f"{hostname}")
-            dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
-            dnode.addExtraCfg("monitorFqdn", hostname)
-            dnode.addExtraCfg("monitorPort", 7043)
-            dnodes.append(dnode)
-        
-        self.TDDnodes = MyDnodes(dnodes)
-        self.TDDnodes.init("")
-        self.TDDnodes.setTestCluster(testCluster)
-        self.TDDnodes.setValgrind(valgrind)
-        self.TDDnodes.stopAll()
-        for dnode in self.TDDnodes.dnodes:
-            self.TDDnodes.deploy(dnode.index,{})
-            
-        for dnode in self.TDDnodes.dnodes:
-            self.TDDnodes.starttaosd(dnode.index)
-
-        # create cluster 
-        for dnode in self.TDDnodes.dnodes[1:]:
-            # tdLog.debug(dnode.cfgDict)
-            dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
-            dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
-            dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
-            cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
-            tdLog.debug(cmd)
-            os.system(cmd)
-        
-        time.sleep(2)
-        tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
     
     def check3mnode(self):
         count=0
@@ -117,25 +72,26 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :     
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break
-            elif tdSql.queryResult[0][2]=='follower' :
-                if  tdSql.queryResult[1][2]=='leader':
+            elif tdSql.queryResult[1][2]=='leader' :
+                if  tdSql.queryResult[0][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break      
-            elif tdSql.queryResult[0][2]=='follower' :
+            elif tdSql.queryResult[2][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
-                    if  tdSql.queryResult[2][2]=='leader':
+                    if  tdSql.queryResult[0][2]=='follower':
                         tdLog.debug("three mnodes is ready in 10s")
                         break                   
             count+=1
         else:
-            tdLog.debug("three mnodes is not ready in 10s ")
-            return -1
+            tdLog.exit("three mnodes is not ready in 10s ")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -153,6 +109,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='offline' :
                 if  tdSql.queryResult[1][2]=='leader':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -164,8 +122,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 2 failed in 10s ")
+            
         tdSql.error("drop mnode on dnode 1;")
 
         tdSql.query("show mnodes;")       
@@ -185,6 +143,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='offline':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -192,8 +152,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 2 failed in 10s ")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 2 failed in 10s ")
+
         tdSql.error("drop mnode on dnode 2;")
 
         tdSql.query("show mnodes;")       
@@ -215,6 +175,8 @@ class TDTestCase:
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
                 tdLog.debug("mnode is  three nodes")
+            else:
+                tdLog.exit("mnode number is correct")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[2][2]=='offline':
                     if  tdSql.queryResult[1][2]=='follower':
@@ -222,8 +184,8 @@ class TDTestCase:
                         break
             count+=1
         else:
-            tdLog.debug("stop mnodes  on dnode 3 failed in 10s")
-            return -1
+            tdLog.exit("stop mnodes  on dnode 3 failed in 10s")
+
         tdSql.error("drop mnode on dnode 3;")
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -238,14 +200,29 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     
+    def check_dnodes_status(self,dnodeNumbers):
+        count=0
+        while count < 5:
+            tdSql.query("show dnodes")
+            # tdLog.debug(tdSql.queryResult)
+            status=0
+            for i in range(dnodeNumbers):
+                if tdSql.queryResult[i][4] == "ready":
+                    status+=1
+            tdLog.debug(status)
+            
+            if status == dnodeNumbers:
+                tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %dnodeNumbers)
+                break 
+            count+=1
+            time.sleep(1)
+        else:
+            tdLog.exit("create cluster with %d dnode but  check dnode not ready within 5s ! "%dnodeNumbers)
 
     def five_dnode_three_mnode(self,dnodenumber):
-        tdSql.query("show dnodes;")
-        tdSql.checkData(0,1,'%s:6030'%self.host)
-        tdSql.checkData(4,1,'%s:6430'%self.host)
-        tdSql.checkData(0,4,'ready')
-        tdSql.checkData(4,4,'ready')
+        self.check_dnodes_status(5)
         tdSql.query("show mnodes;")   
+        tdLog.debug(self.host)
         tdSql.checkRows(1)    
         tdSql.checkData(0,1,'%s:6030'%self.host)
         tdSql.checkData(0,2,'leader')
@@ -262,26 +239,32 @@ class TDTestCase:
         tdSql.error("create mnode on dnode 2")
 
         tdSql.query("show dnodes;")
-        tdLog.debug(tdSql.queryResult)
+        # tdLog.debug(tdSql.queryResult)
 
         tdLog.debug("stop and follower of mnode") 
-        self.TDDnodes.stoptaosd(2)
+        tdDnodes=cluster.dnodes
+        # tdLog.debug(tdDnodes[0])
+
+        tdDnodes[1].stoptaosd()
         self.check3mnode2off()
-        self.TDDnodes.starttaosd(2)
+        tdDnodes[1].starttaosd()
+        self.check3mnode()
 
-        self.TDDnodes.stoptaosd(3)
+        tdDnodes[2].stoptaosd()
         self.check3mnode3off()
-        self.TDDnodes.starttaosd(3)
+        tdDnodes[2].starttaosd()
+        self.check3mnode()
 
-        self.TDDnodes.stoptaosd(1)
+        tdDnodes[0].stoptaosd()
         self.check3mnode1off()
-        self.TDDnodes.starttaosd(1)
+        tdDnodes[0].starttaosd()
+        self.check3mnode()
 
         self.check3mnode()
         stopcount =0 
         while stopcount <= 2:
             for i in range(dnodenumber):
-                self.TDDnodes.stoptaosd(i+1)
+                tdDnodes[i].stoptaosd()
                 # if i == 1 :
                 #     self.check3mnode2off()
                 # elif i == 2 :
@@ -289,22 +272,12 @@ class TDTestCase:
                 # elif i == 0:
                 #     self.check3mnode1off()
 
-                self.TDDnodes.starttaosd(i+1)
+                tdDnodes[i].starttaosd()
                 # self.check3mnode()
             stopcount+=1
         self.check3mnode()
 
-
-    def getConnection(self, dnode):
-        host = dnode.cfgDict["fqdn"]
-        port = dnode.cfgDict["serverPort"]
-        config_dir = dnode.cfgDir
-        return taos.connect(host=host, port=int(port), config=config_dir)
-
-
     def run(self): 
-        # tdLog.debug(self.master_dnode.cfgDict)
-        self.buildcluster(5)
         self.five_dnode_three_mnode(5)
  
     def stop(self):

tests/system-test/fulltest.sh

@@ -112,8 +112,8 @@ python3 ./test.py -f 2-query/twa.py
 
 python3 ./test.py -f 6-cluster/5dnode1mnode.py 
 python3 ./test.py -f 6-cluster/5dnode2mnode.py 
-#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py
-#python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py
+python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5
+# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py 
 
 python3 ./test.py -f 7-tmq/basic5.py

tests/system-test/test.py

@@ -28,6 +28,7 @@ sys.path.append("../pytest")
 from util.log import *
 from util.dnodes import *
 from util.cases import *
+from util.cluster import *
 
 import taos
 
@@ -58,10 +59,11 @@ if __name__ == "__main__":
     logSql = True
     stop = 0
     restart = False
+    dnodeNums = 1
     updateCfgDict = {}
     execCmd = ""
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd'])
+    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [
+        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums'])
     for key, value in opts:
         if key in ['-h', '--help']:
             tdLog.printNoPrefix(
@@ -77,6 +79,8 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-d update cfg dict, base64 json str')
             tdLog.printNoPrefix('-k not kill valgrind processer')
             tdLog.printNoPrefix('-e eval str to run')
+            tdLog.printNoPrefix('-N create dnodes numbers clusters')
+
             sys.exit(0)
 
         if key in ['-r', '--restart']: 
@@ -126,6 +130,9 @@ if __name__ == "__main__":
                 print('updateCfgDict convert fail.')
                 sys.exit(0)
 
+        if key in ['-N', '--dnodeNums']:
+            dnodeNums = value
+
     if not execCmd == "":
         tdDnodes.init(deployPath)
         print(execCmd)
@@ -232,10 +239,35 @@ if __name__ == "__main__":
                     updateCfgDict = ucase.updatecfgDict
             except:
                 pass
+        if dnodeNums == 1 :
             tdDnodes.deploy(1,updateCfgDict)
             tdDnodes.start(1)
+            tdCases.logSql(logSql)
+        else :
+            print("start cluster and  dnodes number")
             
+            dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True)
+            tdDnodes = ClusterDnodes(dnodeslist)
+            tdDnodes.init(deployPath, masterIp)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.deploy(dnode.index,{})
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.starttaosd(dnode.index)
             tdCases.logSql(logSql)
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            print(tdDnodes.getSimCfgPath(),host)
+            cluster.create_dnode(conn)
+            try:
+                if cluster.check_dnode(conn) :
+                    print("check dnode ready")
+            except Exception as r:
+                print(r)
+            
             
         if testCluster:
             tdLog.info("Procedures for testing cluster")
@@ -248,10 +280,12 @@ if __name__ == "__main__":
             conn = taos.connect(
                 host,
                 config=tdDnodes.getSimCfgPath())
+                
             if fileName == "all":
                 tdCases.runAllLinux(conn)
             else:
                 tdCases.runOneLinux(conn, fileName)
+            
         if restart:
             if fileName == "all":
                 tdLog.info("not need to query ")