[TD-14540]<impr>: taosdump code cleanup for2.4 (#11316)

* [TD-14540]<impr>: taosdump code cleanup

for 2.4

* update taos-tools

for 2.4

* refine 5-taos-tools/taosdump/taosdumpTestInspect.py

* [TD-14540]<impr>: refine tools/taosdumpTest2.py

for 2.4

* pick up tools/taosdumpTest2.py from develop branch

tests/pytest/test.py

@@ -55,15 +55,16 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-c Test Cluster Flag')
             tdLog.printNoPrefix('-g valgrind Test Flag')
             tdLog.printNoPrefix('-r taosd restart test')
-            tdLog.printNoPrefix('-w taos on windows')
+            tdLog.printNoPrefix('-w taos on windows')            
             sys.exit(0)
 
-        if key in ['-r', '--restart']: 
+        if key in ['-r', '--restart']:
             restart = True
 
         if key in ['-f', '--file']:
-            fileName = value
-
+            fileName = os.path.normpath(value)
+            print(fileName)
+        
         if key in ['-d', '--docker']:
             fileName = os.path.normpath(value)            
             docker = True
@@ -90,7 +91,7 @@ if __name__ == "__main__":
             valgrind = 1
 
         if key in ['-s', '--stop']:
-            stop = 1
+            stop = 1        
 
         if key in ['-w', '--windows']:
             windows = 1
@@ -124,13 +125,13 @@ if __name__ == "__main__":
             time.sleep(2)
 
         tdLog.info('stop All dnodes')
-    
+
     if masterIp == "":
         host = '127.0.0.1'
     else:
-        host = masterIp
+        host = masterIp    
 
-    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+    tdLog.info("Procedures for tdengine deployed in %s" % (host)) 
     if docker:        
         tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")             
@@ -176,13 +177,40 @@ if __name__ == "__main__":
         tdLog.info("Procedures for testing self-deployment")
         td_clinet = TDSimClient("C:\\TDengine")
         td_clinet.deploy()
-        remote_conn = Connection("root@%s"%host)
-        with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
-            remote_conn.run("python3 ./test.py")
+        if masterIp == "" or masterIp == "localhost":
+            tdDnodes.init(deployPath)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            is_test_framework = 0
+            key_word = 'tdCases.addWindows'
+            try:
+                if key_word in open(fileName).read():
+                    is_test_framework = 1
+            except:
+                pass
+            if is_test_framework:
+                moduleName = fileName.replace(".py", "").replace(os.sep, ".")
+                uModule = importlib.import_module(moduleName)
+                try:
+                    ucase = uModule.TDTestCase()
+                    tdDnodes.deploy(1,ucase.updatecfgDict)
+                except :
+                    tdDnodes.deploy(1,{})
+            else:
+                pass
+                tdDnodes.deploy(1,{})
+            tdDnodes.startWin(1)
+        else:
+            remote_conn = Connection("root@%s"%host)
+            with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
+                remote_conn.run("python3 ./test.py")
         conn = taos.connect(
-            host="%s"%(host),
+            host="%s" % (host),
             config=td_clinet.cfgDir)
         tdCases.runOneWindows(conn, fileName)
+        tdCases.logSql(logSql)
+
     else:
         tdDnodes.init(deployPath)
         tdDnodes.setTestCluster(testCluster)
@@ -193,23 +221,21 @@ if __name__ == "__main__":
         try:
             if key_word in open(fileName).read():
                 is_test_framework = 1
-        except:
+        except BaseException:
             pass
         if is_test_framework:
-            moduleName = fileName.replace(".py", "").replace("/", ".")
+            moduleName = fileName.replace(".py", "").replace(os.sep, ".")
             uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()
-                tdDnodes.deploy(1,ucase.updatecfgDict)
-            except :
-                tdDnodes.deploy(1,{})
+                tdDnodes.deploy(1, ucase.updatecfgDict)
+            except BaseException:
+                tdDnodes.deploy(1, {})
         else:
             pass
-            tdDnodes.deploy(1,{})
+            tdDnodes.deploy(1, {})
         tdDnodes.start(1)
 
-
-
         tdCases.logSql(logSql)
 
         if testCluster:
@@ -226,18 +252,20 @@ if __name__ == "__main__":
             if fileName == "all":
                 tdCases.runAllLinux(conn)
             else:
-                tdCases.runOneWindows(conn, fileName)
+                tdCases.runOneLinux(conn, fileName)
         if restart:
             if fileName == "all":
                 tdLog.info("not need to query ")
-            else:    
+            else:
                 sp = fileName.rsplit(".", 1)
                 if len(sp) == 2 and sp[1] == "py":
                     tdDnodes.stopAll()
                     tdDnodes.start(1)
-                    time.sleep(1)            
-                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
-                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    time.sleep(1)
+                    conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info(
+                        "Procedures for tdengine deployed in %s" %
+                        (host))
                     tdLog.info("query test after taosd restart")
                     tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
                 else:

tests/pytest/tools/taosdumpTest2.py

@@ -11,6 +11,7 @@
 
 # -*- coding: utf-8 -*-
 
+from logging.config import dictConfig
 import sys
 import os
 from util.log import *
@@ -22,6 +23,7 @@ import random
 
 
 class TDTestCase:
+
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
@@ -29,9 +31,8 @@ class TDTestCase:
         self.ts = 1601481600000
         self.numberOfTables = 1
         self.numberOfRecords = 15000
-        self.tmpdir = "tmp"
 
-    def getBuildPath(self):
+    def getPath(self, tool="taosdump"):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
         if ("community" in selfPath):
@@ -39,13 +40,16 @@ class TDTestCase:
         else:
             projPath = selfPath[:selfPath.find("tests")]
 
+        paths = []
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ((tool) in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
-                    buildPath = root[:len(root) - len("/build/bin")]
+                    paths.append(os.path.join(root, tool))
                     break
-        return buildPath
+        if (len(paths) == 0):
+            return ""
+        return paths[0]
 
     def generateString(self, length):
         chars = string.ascii_uppercase + string.ascii_lowercase
@@ -55,6 +59,11 @@ class TDTestCase:
         return v
 
     def run(self):
+        if not os.path.exists("./taosdumptest/tmp"):
+            os.makedirs("./taosdumptest/tmp")
+        else:
+            print("directory exists")
+
         tdSql.prepare()
 
         tdSql.execute("create table st(ts timestamp, c1 timestamp, c2 int, c3 bigint, c4 float, c5 double, c6 binary(8), c7 smallint, c8 tinyint, c9 bool, c10 nchar(8)) tags(t1 int)")
@@ -70,29 +79,23 @@ class TDTestCase:
                     break
             tdSql.execute(sql)
 
-        buildPath = self.getBuildPath()
-        if (buildPath == ""):
+        binPath = self.getPath()
+        if (binPath == ""):
             tdLog.exit("taosdump not found!")
         else:
-            tdLog.info("taosdump found in %s" % buildPath)
-        binPath = buildPath + "/build/bin/"
-
-        if not os.path.exists(self.tmpdir):
-            os.makedirs(self.tmpdir)
-        else:
-            print("directory exists")
-            os.system("rm -rf %s" % self.tmpdir)
-            os.makedirs(self.tmpdir)
+            tdLog.info("taosdump found in %s" % binPath)
 
+        os.system("rm ./taosdumptest/tmp/*.sql")
+        os.system("rm ./taosdumptest/tmp/*.avro*")
         os.system(
-            "%staosdump --databases db -o %s -y" %
-            (binPath, self.tmpdir))
+            "%s --databases db -o ./taosdumptest/tmp " %
+            binPath)
 
         tdSql.execute("drop database db")
         tdSql.query("show databases")
         tdSql.checkRows(0)
 
-        os.system("%staosdump -i %s -y" % (binPath, self.tmpdir))
+        os.system("%s -i ./taosdumptest/tmp -y" % binPath)
 
         tdSql.query("show databases")
         tdSql.checkRows(1)
@@ -105,7 +108,6 @@ class TDTestCase:
 
         tdSql.query("select count(*) from t1")
         tdSql.checkData(0, 0, self.numberOfRecords)
-        os.system("rm -rf %s" % self.tmpdir)
 
         # test case for TS-1225
         tdSql.execute("create database test")
@@ -118,15 +120,15 @@ class TDTestCase:
              self.generateString(16374),
              self.generateString(16374)))
 
-        os.system("rm /tmp/*.sql")
-        os.system("rm /tmp/*.avro*")
-        os.system("%staosdump -D test -o /tmp -y" % binPath)
+        os.system("rm ./taosdumptest/tmp/*.sql")
+        os.system("rm ./taosdumptest/tmp/*.avro*")
+        os.system("%s -D test -o ./taosdumptest/tmp -y" % binPath)
 
         tdSql.execute("drop database test")
         tdSql.query("show databases")
         tdSql.checkRows(1)
 
-        os.system("%staosdump -i /tmp -y" % binPath)
+        os.system("%s -i ./taosdumptest/tmp -y" % binPath)
 
         tdSql.execute("use test")
         tdSql.error("show vnodes '' ")