[TD-12186]<test>import lib from any path in test frame

tests/pytest/test.py

@@ -37,68 +37,83 @@ if __name__ == "__main__":
     stop = 0
     restart = False
     windows = 0
-    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [
-        'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
+    opts, args = getopt.gnu_getopt(
+        sys.argv[1:],
+        "f:p:m:l:scghrw",
+        [
+            "file=",
+            "path=",
+            "master",
+            "logSql",
+            "stop",
+            "cluster",
+            "valgrind",
+            "help",
+            "windows",
+        ],
+    )
     for key, value in opts:
-        if key in ['-h', '--help']:
-            tdLog.printNoPrefix(
-                'A collection of test cases written using Python')
-            tdLog.printNoPrefix('-f Name of test case file written by Python')
-            tdLog.printNoPrefix('-p Deploy Path for Simulator')
-            tdLog.printNoPrefix('-m Master Ip for Simulator')
-            tdLog.printNoPrefix('-l <True:False> logSql Flag')
-            tdLog.printNoPrefix('-s stop All dnodes')
-            tdLog.printNoPrefix('-c Test Cluster Flag')
-            tdLog.printNoPrefix('-g valgrind Test Flag')
-            tdLog.printNoPrefix('-r taosd restart test')
-            tdLog.printNoPrefix('-w taos on windows')
+        if key in ["-h", "--help"]:
+            tdLog.printNoPrefix("A collection of test cases written using Python")
+            tdLog.printNoPrefix("-f Name of test case file written by Python")
+            tdLog.printNoPrefix("-p Deploy Path for Simulator")
+            tdLog.printNoPrefix("-m Master Ip for Simulator")
+            tdLog.printNoPrefix("-l <True:False> logSql Flag")
+            tdLog.printNoPrefix("-s stop All dnodes")
+            tdLog.printNoPrefix("-c Test Cluster Flag")
+            tdLog.printNoPrefix("-g valgrind Test Flag")
+            tdLog.printNoPrefix("-r taosd restart test")
+            tdLog.printNoPrefix("-w taos on windows")
             sys.exit(0)
 
-        if key in ['-r', '--restart']: 
+        if key in ["-r", "--restart"]:
             restart = True
 
-        if key in ['-f', '--file']:
+        if key in ["-f", "--file"]:
             fileName = value
 
-        if key in ['-p', '--path']:
+        if key in ["-p", "--path"]:
             deployPath = value
 
-        if key in ['-m', '--master']:
+        if key in ["-m", "--master"]:
             masterIp = value
 
-        if key in ['-l', '--logSql']:
-            if (value.upper() == "TRUE"):
+        if key in ["-l", "--logSql"]:
+            if value.upper() == "TRUE":
                 logSql = True
-            elif (value.upper() == "FALSE"):
+            elif value.upper() == "FALSE":
                 logSql = False
             else:
                 tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
                 sys.exit(0)
 
-        if key in ['-c', '--cluster']:
+        if key in ["-c", "--cluster"]:
             testCluster = True
 
-        if key in ['-g', '--valgrind']:
+        if key in ["-g", "--valgrind"]:
             valgrind = 1
 
-        if key in ['-s', '--stop']:
+        if key in ["-s", "--stop"]:
             stop = 1
 
-        if key in ['-w', '--windows']:
+        if key in ["-w", "--windows"]:
             windows = 1
 
-    if (stop != 0):
-        if (valgrind == 0):
+    if stop != 0:
+        if valgrind == 0:
             toBeKilled = "taosd"
         else:
             toBeKilled = "valgrind.bin"
 
-        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
+        killCmd = (
+            "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1"
+            % toBeKilled
+        )
 
         psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
         processID = subprocess.check_output(psCmd, shell=True)
 
-        while(processID):
+        while processID:
             os.system(killCmd)
             time.sleep(1)
             processID = subprocess.check_output(psCmd, shell=True)
@@ -115,25 +130,26 @@ if __name__ == "__main__":
         if valgrind:
             time.sleep(2)
 
-        tdLog.info('stop All dnodes')
-    
+        tdLog.info("stop All dnodes")
+
     if masterIp == "":
-        host = '127.0.0.1'
+        host = "127.0.0.1"
     else:
         host = masterIp
 
     tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    uModule = tdDnodes.import_module(fileName)
     if windows:
         tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")
         td_clinet = TDSimClient("C:\\TDengine")
         td_clinet.deploy()
-        remote_conn = Connection("root@%s"%host)
-        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+        remote_conn = Connection("root@%s" % host)
+        with remote_conn.cd(
+            "/var/lib/jenkins/workspace/TDinternal/community/tests/pytest"
+        ):
             remote_conn.run("python3 ./test.py")
-        conn = taos.connect(
-            host="%s"%(host),
-            config=td_clinet.cfgDir)
+        conn = taos.connect(host="%s" % (host), config=td_clinet.cfgDir)
         tdCases.runOneWindows(conn, fileName)
     else:
         tdDnodes.init(deployPath)
@@ -141,27 +157,24 @@ if __name__ == "__main__":
         tdDnodes.setValgrind(valgrind)
         tdDnodes.stopAll()
         is_test_framework = 0
-        key_word = 'tdCases.addLinux'
+        key_word = "tdCases.addLinux"
         try:
             if key_word in open(fileName).read():
                 is_test_framework = 1
         except:
             pass
         if is_test_framework:
-            moduleName = fileName.replace(".py", "").replace("/", ".")
-            uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()
-                tdDnodes.deploy(1,ucase.updatecfgDict)
-            except :
-                tdDnodes.deploy(1,{})
+                tdDnodes.deploy(1, ucase.updatecfgDict)
+                del uModule
+            except:
+                tdDnodes.deploy(1, {})
         else:
             pass
-            tdDnodes.deploy(1,{})
+            tdDnodes.deploy(1, {})
         tdDnodes.start(1)
 
-
-
         tdCases.logSql(logSql)
 
         if testCluster:
@@ -172,23 +185,21 @@ if __name__ == "__main__":
                 tdCases.runOneCluster(fileName)
         else:
             tdLog.info("Procedures for testing self-deployment")
-            conn = taos.connect(
-                host,
-                config=tdDnodes.getSimCfgPath())
+            conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
             if fileName == "all":
                 tdCases.runAllLinux(conn)
             else:
-                tdCases.runOneWindows(conn, fileName)
+                tdCases.runOneLinux(conn, fileName, uModule)
         if restart:
             if fileName == "all":
                 tdLog.info("not need to query ")
-            else:    
+            else:
                 sp = fileName.rsplit(".", 1)
                 if len(sp) == 2 and sp[1] == "py":
                     tdDnodes.stopAll()
                     tdDnodes.start(1)
-                    time.sleep(1)            
-                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
+                    time.sleep(1)
+                    conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
                     tdLog.info("Procedures for tdengine deployed in %s" % (host))
                     tdLog.info("query test after taosd restart")
                     tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")

tests/pytest/util/cases.py

@@ -17,6 +17,7 @@ import time
 import datetime
 import inspect
 import importlib
+from util.dnodes import *
 from util.log import *
 
 
@@ -34,8 +35,9 @@ class TDCases:
         self.clusterCases = []
 
     def __dynamicLoadModule(self, fileName):
-        moduleName = fileName.replace(".py", "").replace(os.sep, ".")
-        return importlib.import_module(moduleName, package='..')
+        # moduleName = fileName.replace(".py", "").replace(os.sep, ".")
+        # return importlib.import_module(moduleName, package='..')
+        return tdDnodes.import_module(fileName)
 
     def logSql(self, logSql):
         self._logSql = logSql
@@ -63,10 +65,11 @@ class TDCases:
 
         tdLog.info("total %d Linux test case(s) executed" % (runNum))
 
-    def runOneLinux(self, conn, fileName):
-        testModule = self.__dynamicLoadModule(fileName)
-
+    def runOneLinux(self, conn, fileName, testModule):
+        # testModule = self.__dynamicLoadModule(fileName)
         runNum = 0
+        if fileName.startswith(".."):
+            fileName = fileName[3:]
         for tmp in self.linuxCases:
             if tmp.name.find(fileName) != -1:
                 case = testModule.TDTestCase()
@@ -95,7 +98,7 @@ class TDCases:
         tdLog.notice("total %d Windows test case(s) executed" % (runNum))
 
     def runOneWindows(self, conn, fileName):
-        testModule = self.__dynamicLoadModule(fileName)
+        # testModule = self.__dynamicLoadModule(fileName)
 
         runNum = 0
         for tmp in self.windowsCases:
@@ -130,7 +133,7 @@ class TDCases:
         tdLog.notice("total %d Cluster test case(s) executed" % (runNum))
 
     def runOneCluster(self, fileName):
-        testModule = self.__dynamicLoadModule(fileName)
+        # testModule = self.__dynamicLoadModule(fileName)
 
         runNum = 0
         for tmp in self.clusterCases:

tests/pytest/util/dnodes.py

@@ -18,6 +18,7 @@ import platform
 import pathlib
 import shutil
 import subprocess
+import importlib
 from time import sleep
 from util.log import *
 
@@ -579,6 +580,13 @@ class TDDnodes:
     def check(self, index):
         if index < 1 or index > 10:
             tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
+    
+    def import_module(self,fileName):
+        module_dir, module_file = os.path.split(fileName)
+        module_name, module_ext = os.path.splitext(module_file)
+        spec = importlib.util.spec_from_file_location(module_name,fileName)
+        uModule = spec.loader.load_module()
+        return uModule
 
     def stopAll(self):
         tdLog.info("stop all dnodes")