Merge pull request #13868 from taosdata/test/chr/TD-14699

test:modify  testcase of muti-mnode

tests/pytest/util/dnodes.py

@@ -334,22 +334,22 @@ class TDDnode:
                 bkey2 = bytes(key2, encoding="utf8")
                 logFile = self.logDir + "/taosdlog.0"
                 i = 0
-                while not os.path.exists(logFile):
-                    sleep(0.1)
-                    i += 1
-                    if i > 10:
-                        break
-                tailCmdStr = 'tail -f '
-                if platform.system().lower() == 'windows':
-                    tailCmdStr = 'tail -n +0 -f '
-                popen = subprocess.Popen(
-                    tailCmdStr + logFile,
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.PIPE,
-                    shell=True)
-                pid = popen.pid
-                # print('Popen.pid:' + str(pid))
-                timeout = time.time() + 60 * 2
+                # while not os.path.exists(logFile):
+                #     sleep(0.1)
+                #     i += 1
+                #     if i > 10:
+                #         break
+                # tailCmdStr = 'tail -f '
+                # if platform.system().lower() == 'windows':
+                #     tailCmdStr = 'tail -n +0 -f '
+                # popen = subprocess.Popen(
+                #     tailCmdStr + logFile,
+                #     stdout=subprocess.PIPE,
+                #     stderr=subprocess.PIPE,
+                #     shell=True)
+                # pid = popen.pid
+                # # print('Popen.pid:' + str(pid))
+                # timeout = time.time() + 60 * 2
                 # while True:
                 #     line = popen.stdout.readline().strip()
                 #     print(line)

tests/pytest/util/sql.py

@@ -156,6 +156,7 @@ class TDSql:
     def checkRows(self, expectRows):
         if self.queryRows == expectRows:
             tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows))
+            return True
         else:
             caller = inspect.getframeinfo(inspect.stack()[1][0])
             args = (caller.filename, caller.lineno, self.sql, self.queryRows, expectRows)

tests/system-test/6-cluster/5dnode2mnode.py

@@ -55,6 +55,8 @@ class TDTestCase:
         hostname = socket.gethostname()
         dnodes = []
         start_port = 6030        
+        start_port_sec = 6130
+
         for num in range(1, dnodes_nums+1):
             dnode = TDDnode(num)
             dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
@@ -62,6 +64,8 @@ class TDTestCase:
             dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
             dnode.addExtraCfg("monitorFqdn", hostname)
             dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
+ 
             dnodes.append(dnode)
         
         self.TDDnodes = MyDnodes(dnodes)

tests/system-test/6-cluster/5dnode3mnodeDrop.py

@@ -78,6 +78,7 @@ class TDTestCase:
         print(hostname)
         dnodes = []
         start_port = 6030
+        start_port_sec = 6130
         for num in range(1, dnodes_nums+1):
             dnode = TDDnode(num)
             dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
@@ -85,6 +86,7 @@ class TDTestCase:
             dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
             dnode.addExtraCfg("monitorFqdn", hostname)
             dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
             dnodes.append(dnode)
         
         self.TDDnodes = MyDnodes(dnodes)
@@ -117,6 +119,7 @@ class TDTestCase:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :     
+                print("mnode is  three nodes")           
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -134,7 +137,9 @@ class TDTestCase:
                         break                   
             count+=1
         else:
+            print(tdSql.queryResult)
             print("three mnodes is not ready in 10s ")
+            return -1
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -151,6 +156,7 @@ class TDTestCase:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='offline' :
                 if  tdSql.queryResult[1][2]=='leader':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -163,6 +169,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1 
+        tdSql.error("drop mnode on dnode 1;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -174,12 +182,13 @@ class TDTestCase:
         tdSql.checkData(2,1,'%s:6230'%self.host)
         tdSql.checkData(2,3,'ready')
 
-    def check3mnode2drop(self):
+    def check3mnode2off(self):
         count=0
-        while count < 10:
+        while count < 40:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='offline':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -188,6 +197,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 2;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -197,7 +208,7 @@ class TDTestCase:
         tdSql.checkData(1,1,'%s:6130'%self.host)
         tdSql.checkData(1,2,'offline')
         tdSql.checkData(1,3,'ready')
-        tdSql.checkData(2,1,'%s:6230')
+        tdSql.checkData(2,1,'%s:6230'%self.host)
         tdSql.checkData(2,2,'follower')
         tdSql.checkData(2,3,'ready')
 
@@ -207,6 +218,7 @@ class TDTestCase:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[2][2]=='offline':
                     if  tdSql.queryResult[1][2]=='follower':
@@ -215,7 +227,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 3 failed in 10s")
-            
+            return -1
+        tdSql.error("drop mnode on dnode 3;")
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
         tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -262,8 +275,25 @@ class TDTestCase:
             for i in range(1,3):
                 tdLog.debug("drop mnode on dnode %d"%(i+1))
                 tdSql.execute("drop mnode on dnode %d"%(i+1))
+                tdSql.query("show mnodes;")
+                count=0
+                while count<10:
+                    time.sleep(1)
+                    tdSql.query("show mnodes;")
+                    if tdSql.checkRows(2):
+                        print("drop mnode %d successfully"%(i+1))
+                        break
+                    count+=1
                 tdLog.debug("create mnode on dnode %d"%(i+1))
                 tdSql.execute("create mnode on dnode %d"%(i+1))
+                count=0
+                while count<10:
+                    time.sleep(1)
+                    tdSql.query("show mnodes;")
+                    if tdSql.checkRows(3):
+                        print("drop mnode %d successfully"%(i+1))
+                        break
+                    count+=1
             dropcount+=1
         self.check3mnode()
 

tests/system-test/6-cluster/5dnode3mnodeStop.py

@@ -116,6 +116,7 @@ class TDTestCase:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :     
+                print("mnode is  three nodes")           
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -134,6 +135,7 @@ class TDTestCase:
             count+=1
         else:
             print("three mnodes is not ready in 10s ")
+            return -1
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -145,12 +147,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode1off(self):
-        tdSql.error("drop mnode on dnode 1;")
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='offline' :
                 if  tdSql.queryResult[1][2]=='leader':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -163,6 +165,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 1;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -175,12 +179,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode2off(self):
-        tdSql.error("drop mnode on dnode 2;")
         count=0
-        while count < 10:
+        while count < 40:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='offline':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -189,6 +193,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 2;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -203,12 +209,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode3off(self):
-        tdSql.error("drop mnode on dnode 3;")
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[2][2]=='offline':
                     if  tdSql.queryResult[1][2]=='follower':
@@ -217,7 +223,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 3 failed in 10s")
-            
+            return -1
+        tdSql.error("drop mnode on dnode 3;")
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
         tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -264,13 +271,13 @@ class TDTestCase:
 
         self.TDDnodes.stoptaosd(3)
         self.check3mnode3off()
-        self.TDDnodes.starttaosd(2)
+        self.TDDnodes.starttaosd(3)
 
         self.TDDnodes.stoptaosd(1)
         self.check3mnode1off()
         self.TDDnodes.starttaosd(1)
 
-        # self.check3mnode()
+        self.check3mnode()
         stopcount =0 
         while stopcount <= 2:
             for i in range(dnodenumber):

tests/system-test/6-cluster/5dnode3mnodeStopInsert.py

@@ -13,13 +13,50 @@ import time
 import socket
 import subprocess
 from multiprocessing import Process
-import threading as thd
+import threading 
+import time
+import inspect
+import ctypes
 class MyDnodes(TDDnodes):
     def __init__(self ,dnodes_lists):
         super(MyDnodes,self).__init__()
         self.dnodes = dnodes_lists  # dnode must be TDDnode instance
         self.simDeployed = False
 
+class MyThreadFunc(object):
+    '''
+    手动终止线程的方法
+    '''
+    def __init__(self, func):
+        self.myThread = threading.Thread(target=func)
+
+    def start(self):
+        print('线程启动')
+        self.myThread.start()
+
+    def stop(self):
+        print('线程终止')
+        try:
+            for i in range(5):
+                self._async_raise(self.myThread.ident, SystemExit)
+                time.sleep(1)
+        except Exception as e:
+            print(e)
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        tid = ctypes.c_long(tid)
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble,
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
 class TDTestCase:
 
     def init(self,conn ,logSql):
@@ -52,8 +89,11 @@ class TDTestCase:
     
     def insert_data(self,countstart,countstop):
         # fisrt add data : db\stable\childtable\general table
+        
         for couti in range(countstart,countstop):
+            tdLog.debug("drop database if exists db%d" %couti)
             tdSql.execute("drop database if exists db%d" %couti)
+            print("create database if not exists db%d replica 1 duration 300" %couti)
             tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
             tdSql.execute("use db%d" %couti)
             tdSql.execute(
@@ -78,6 +118,7 @@ class TDTestCase:
         hostname = socket.gethostname()
         dnodes = []
         start_port = 6030
+        start_port_sec = 6130
         for num in range(1, dnodes_nums+1):
             dnode = TDDnode(num)
             dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
@@ -85,6 +126,7 @@ class TDTestCase:
             dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
             dnode.addExtraCfg("monitorFqdn", hostname)
             dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
             dnodes.append(dnode)
         
         self.TDDnodes = MyDnodes(dnodes)
@@ -111,12 +153,39 @@ class TDTestCase:
         time.sleep(2)
         tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
 
+    def checkdnodes(self,dnodenumber):
+        count=0
+        while count < 10:
+            time.sleep(1)
+            statusReadyBumber=0
+            tdSql.query("show dnodes;")
+            if tdSql.checkRows(dnodenumber) :     
+                print("dnode is %d nodes"%dnodenumber)   
+            for i in range(dnodenumber):
+                if tdSql.queryResult[i][4] !='ready'  :
+                    status=tdSql.queryResult[i][4]
+                    print("dnode:%d status is %s "%(i,status))
+                    break
+                else:
+                    statusReadyBumber+=1
+            print(statusReadyBumber)
+            if statusReadyBumber == dnodenumber :
+                print("all of %d mnodes is ready in 10s "%dnodenumber)
+                return True
+                break
+            count+=1
+        else:
+            print("%d mnodes is not ready in 10s "%dnodenumber)
+            return False
+           
+
     def check3mnode(self):
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :     
+                print("mnode is  three nodes")           
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='follower':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -135,6 +204,7 @@ class TDTestCase:
             count+=1
         else:
             print("three mnodes is not ready in 10s ")
+            return -1
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -146,12 +216,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode1off(self):
-        tdSql.error("drop mnode on dnode 1;")
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='offline' :
                 if  tdSql.queryResult[1][2]=='leader':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -164,6 +234,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 1;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -176,12 +248,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode2off(self):
-        tdSql.error("drop mnode on dnode 2;")
         count=0
         while count < 40:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[1][2]=='offline':
                     if  tdSql.queryResult[2][2]=='follower':
@@ -190,6 +262,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 2;")
 
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
@@ -204,12 +278,12 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def check3mnode3off(self):
-        tdSql.error("drop mnode on dnode 3;")
         count=0
         while count < 10:
             time.sleep(1)
             tdSql.query("show mnodes;")
             if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
             if  tdSql.queryResult[0][2]=='leader' :
                 if  tdSql.queryResult[2][2]=='offline':
                     if  tdSql.queryResult[1][2]=='follower':
@@ -218,7 +292,8 @@ class TDTestCase:
             count+=1
         else:
             print("stop mnodes  on dnode 3 failed in 10s")
-            
+            return -1
+        tdSql.error("drop mnode on dnode 3;")
         tdSql.query("show mnodes;")       
         tdSql.checkRows(3) 
         tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -231,8 +306,6 @@ class TDTestCase:
         tdSql.checkData(2,2,'offline')
         tdSql.checkData(2,3,'ready')
 
-
-
     def five_dnode_three_mnode(self,dnodenumber):
         tdSql.query("show dnodes;")
         tdSql.checkData(0,1,'%s:6030'%self.host)
@@ -252,43 +325,29 @@ class TDTestCase:
         # fisrt check statut ready
         self.check3mnode()
 
-
         tdSql.error("create mnode on dnode 2")
-
         tdSql.query("show dnodes;")
         print(tdSql.queryResult)
+        tdLog.debug("stop all of mnode ") 
 
-        tdLog.debug("stop and follower of mnode") 
-        self.TDDnodes.stoptaosd(2)
-        self.check3mnode2off()
-        self.TDDnodes.starttaosd(2)
-
-        self.TDDnodes.stoptaosd(3)
-        self.check3mnode3off()
-        self.TDDnodes.starttaosd(3)
-
-        self.TDDnodes.stoptaosd(1)
-        self.check3mnode1off()
-        self.TDDnodes.starttaosd(1)
-
-        # self.check3mnode()
         stopcount =0 
         while stopcount <= 2:
             for i in range(dnodenumber):
-                threads = []
-                threads.append(thd.Thread(target=self.insert_data, args=(i*2,i*2+2))) 
-                threads[0].start()
+                # threads=[]
+                # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) 
+                threads=threading.Thread(target=self.insert_data, args=((stopcount+i)*2,(i+stopcount)*2+2))
+                threads.start()
                 self.TDDnodes.stoptaosd(i+1)
-                # if i == 1 :
-                #     self.check3mnode2off()
-                # elif i == 2 :
-                #     self.check3mnode3off()
-                # elif i == 0:
-                #     self.check3mnode1off()
-
                 self.TDDnodes.starttaosd(i+1)
-                threads[0].join()
                 
+                if self.checkdnodes(5):
+                    print("123")
+                    threads.join()
+                else:
+                    print("456")
+                    self._is_stopped = True
+                    assert 1 == 2 ,"some dnode started failed"
+                    return False
                 # self.check3mnode()
             stopcount+=1
         self.check3mnode()

tests/system-test/fulltest.sh

@@ -101,7 +101,7 @@ python3 ./test.py -f 2-query/tail.py
 
 python3 ./test.py -f 6-cluster/5dnode1mnode.py 
 python3 ./test.py -f 6-cluster/5dnode2mnode.py 
-# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py
+python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py
 python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py
 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py