<enh>[tools]: taosdump code cleanup for2.4 (#11328)

* [TD-14540]<impr>: taosdump code cleanup

for 2.4

* update taos-tools

for 2.4

* refine 5-taos-tools/taosdump/taosdumpTestInspect.py

* [TD-14540]<impr>: refine tools/taosdumpTest2.py

for 2.4

* pick up tools/taosdumpTest2.py from develop branch

* update taos-tools

for 2.4

* reduce test batch of tools/taosdumpTest2.py

* merge develop branch

* replace getBuildPath with getPath

taos-tools @ 77660afb

-Subproject commit d1c2c6404e10d303117caa02b746665974e0e264
+Subproject commit 77660afbc8ebe44d32c8d5ca25b6afa7f5f9caee

tests/pytest/tools/taosdumpTest2.py

@@ -30,7 +30,7 @@ class TDTestCase:
 
         self.ts = 1601481600000
         self.numberOfTables = 1
-        self.numberOfRecords = 15000
+        self.numberOfRecords = 150
 
     def getPath(self, tool="taosdump"):
         selfPath = os.path.dirname(os.path.realpath(__file__))

tests/pytest/util/dnodes.py

@@ -65,9 +65,11 @@ class TDSimClient:
         cmd = "echo %s %s >> %s" % (option, value, self.cfgPath)
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
-    def os_string(self,path):
-        os_path = path.replace("/",os.sep)
+
+    def os_string(self, path):
+        os_path = path.replace("/", os.sep)
         return os_path
+
     def deploy(self):
         self.logDir = self.os_string("%s/sim/psim/log" % (self.path))
         self.cfgDir = self.os_string("%s/sim/psim/cfg" % (self.path))
@@ -76,11 +78,11 @@ class TDSimClient:
         # cmd = "rm -rf " + self.logDir
         # if os.system(cmd) != 0:
         #     tdLog.exit(cmd)
-        if os.path.exists(self.logDir): 
+        if os.path.exists(self.logDir):
             try:
                 shutil.rmtree(self.logDir)
-            except:
-                tdLog.exit("del %s failed"%self.logDir)
+            except BaseException:
+                tdLog.exit("del %s failed" % self.logDir)
         # cmd = "mkdir -p " + self.logDir
         # if os.system(cmd) != 0:
         #     tdLog.exit(cmd)
@@ -88,11 +90,11 @@ class TDSimClient:
         # cmd = "rm -rf " + self.cfgDir
         # if os.system(cmd) != 0:
         #     tdLog.exit(cmd)
-        if os.path.exists(self.cfgDir): 
+        if os.path.exists(self.cfgDir):
             try:
                 shutil.rmtree(self.cfgDir)
-            except:
-                tdLog.exit("del %s failed"%self.cfgDir)
+            except BaseException:
+                tdLog.exit("del %s failed" % self.cfgDir)
         # cmd = "mkdir -p " + self.cfgDir
         # if os.system(cmd) != 0:
         #     tdLog.exit(cmd)
@@ -102,8 +104,8 @@ class TDSimClient:
         #     tdLog.exit(cmd)
         try:
             pathlib.Path(self.cfgPath).touch()
-        except:
-             tdLog.exit("create %s failed"%self.cfgPath)
+        except BaseException:
+            tdLog.exit("create %s failed" % self.cfgPath)
         if self.testCluster:
             self.cfg("masterIp", "192.168.0.1")
             self.cfg("secondIp", "192.168.0.2")
@@ -123,36 +125,36 @@ class TDDnode:
         self.testCluster = False
         self.valgrind = 0
         self.cfgDict = {
-            "numOfLogLines":"100000000",
-            "mnodeEqualVnodeNum":"0",
-            "walLevel":"2",
-            "fsync":"1000",
-            "statusInterval":"1",
-            "numOfMnodes":"3",
-            "numOfThreadsPerCore":"2.0",
-            "monitor":"0",
-            "maxVnodeConnections":"30000",
-            "maxMgmtConnections":"30000",
-            "maxMeterConnections":"30000",
-            "maxShellConns":"30000",
-            "locale":"en_US.UTF-8",
-            "charset":"UTF-8",
-            "asyncLog":"0",
-            "anyIp":"0",
-            "telemetryReporting":"0",
-            "dDebugFlag":"135",
-            "tsdbDebugFlag":"135",
-            "mDebugFlag":"135",
-            "sdbDebugFlag":"135",
-            "rpcDebugFlag":"135",
-            "tmrDebugFlag":"131",
-            "cDebugFlag":"135",
-            "httpDebugFlag":"135",
-            "monitorDebugFlag":"135",
-            "udebugFlag":"135",
-            "jnidebugFlag":"135",
-            "qdebugFlag":"135",
-            "maxSQLLength":"1048576",
+            "numOfLogLines": "100000000",
+            "mnodeEqualVnodeNum": "0",
+            "walLevel": "2",
+            "fsync": "1000",
+            "statusInterval": "1",
+            "numOfMnodes": "3",
+            "numOfThreadsPerCore": "2.0",
+            "monitor": "0",
+            "maxVnodeConnections": "30000",
+            "maxMgmtConnections": "30000",
+            "maxMeterConnections": "30000",
+            "maxShellConns": "30000",
+            "locale": "en_US.UTF-8",
+            "charset": "UTF-8",
+            "asyncLog": "0",
+            "anyIp": "0",
+            "telemetryReporting": "0",
+            "dDebugFlag": "135",
+            "tsdbDebugFlag": "135",
+            "mDebugFlag": "135",
+            "sdbDebugFlag": "135",
+            "rpcDebugFlag": "135",
+            "tmrDebugFlag": "131",
+            "cDebugFlag": "135",
+            "httpDebugFlag": "135",
+            "monitorDebugFlag": "135",
+            "udebugFlag": "135",
+            "jnidebugFlag": "135",
+            "qdebugFlag": "135",
+            "maxSQLLength": "1048576",
             "enableCoreFile": "1",
         }
 
@@ -200,17 +202,11 @@ class TDDnode:
         if os.system(cmd) != 0:
             tdLog.exit(cmd)
 
-        cmd = "mkdir -p " + self.dataDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.dataDir, exist_ok=True)  # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.logDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.logDir, exist_ok=True)  # like "mkdir -p"
 
-        cmd = "mkdir -p " + self.cfgDir
-        if os.system(cmd) != 0:
-            tdLog.exit(cmd)
+        os.makedirs(self.cfgDir, exist_ok=True)  # like "mkdir -p"
 
         cmd = "touch " + self.cfgPath
         if os.system(cmd) != 0:
@@ -233,16 +229,16 @@ class TDDnode:
         isFirstDir = 1
         if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]:
             print(updatecfgDict[0][0])
-            for key,value in updatecfgDict[0][0].items():
-                if value == 'dataDir' :
+            for key, value in updatecfgDict[0][0].items():
+                if value == 'dataDir':
                     if isFirstDir:
                         self.cfgDict.pop('dataDir')
-                        self.cfg(value,key)
+                        self.cfg(value, key)
                         isFirstDir = 0
                     else:
-                        self.cfg(value,key)
+                        self.cfg(value, key)
                 else:
-                    self.addExtraCfg(key,value)
+                    self.addExtraCfg(key, value)
         for key, value in self.cfgDict.items():
             self.cfg(key, value)
 
@@ -251,8 +247,7 @@ class TDDnode:
             "dnode:%d is deployed and configured by %s" %
             (self.index, self.cfgPath))
 
-    def getBuildPath(self, tool="taosd"):
-        buildPath = ""
+    def getPath(self, tool="taosd"):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
         if ("community" in selfPath):
@@ -260,24 +255,30 @@ class TDDnode:
         else:
             projPath = selfPath[:selfPath.find("tests")]
 
+        paths = []
         for root, dirs, files in os.walk(projPath):
             if ((tool) in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
-                    buildPath = root[:len(root)-len("/build/bin")]
+                    paths.append(os.path.join(root, tool))
                     break
-        return buildPath
+        if (len(paths) == 0):
+                return ""
+        return paths[0]
 
     def start(self):
-        buildPath = self.getBuildPath()
+        binPath = self.getPath()
 
-        if (buildPath == ""):
+        if (binPath == ""):
             tdLog.exit("taosd not found!")
         else:
-            tdLog.info("taosd found in %s" % buildPath)
+            tdLog.info("taosd found: %s" % binPath)
 
-        binPath = buildPath + "/build/bin/taosd"
-        taosadapterBinPath = buildPath + "/build/bin/taosadapter"
+        taosadapterBinPath = self.getPath("taosadapter")
+        if (taosadapterBinPath == ""):
+            tdLog.info("taosAdapter not found!")
+        else:
+            tdLog.info("taosAdapter found: %s" % taosadapterBinPath)
 
         if self.deployed == 0:
             tdLog.exit("dnode:%d is not deployed" % (self.index))
@@ -294,7 +295,8 @@ class TDDnode:
             print(cmd)
 
         taosadapterCmd = "nohup %s --opentsdb_telnet.enable=true > /dev/null 2>&1 & " % (
-                taosadapterBinPath)
+            taosadapterBinPath)
+        tdLog.info(taosadapterCmd)
         if os.system(taosadapterCmd) != 0:
             tdLog.exit(taosadapterCmd)
 
@@ -306,18 +308,22 @@ class TDDnode:
         if self.valgrind == 0:
             time.sleep(0.1)
             key = 'from offline to online'
-            bkey = bytes(key,encoding="utf8")
+            bkey = bytes(key, encoding="utf8")
             logFile = self.logDir + "/taosdlog.0"
             i = 0
             while not os.path.exists(logFile):
                 sleep(0.1)
                 i += 1
-                if i>50:
+                if i > 50:
                     break
-            popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+            popen = subprocess.Popen(
+                'tail -f ' + logFile,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=True)
             pid = popen.pid
             # print('Popen.pid:' + str(pid))
-            timeout = time.time() + 60*2
+            timeout = time.time() + 60 * 2
             while True:
                 line = popen.stdout.readline().strip()
                 if bkey in line:
@@ -327,22 +333,89 @@ class TDDnode:
                     tdLog.exit('wait too long for taosd start')
             tdLog.debug("the dnode:%d has been started." % (self.index))
         else:
-            tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index))
+            tdLog.debug(
+                "wait 10 seconds for the dnode:%d to start." %
+                (self.index))
             time.sleep(10)
 
-        
         # time.sleep(5)
-    
+
+    def startWin(self):
+        binPath = self.getPath("taosd.exe")
+
+        if (binPath == ""):
+            tdLog.exit("taosd.exe not found!")
+        else:
+            tdLog.info("taosd.exe found: %s" % binPath)
+
+        taosadapterBinPath = self.getPath("taosadapter.exe")
+        if (taosadapterBinPath == ""):
+            tdLog.info("taosAdapter.exe not found!")
+        else:
+            tdLog.info("taosAdapter.exe found in %s" % taosadapterBuildPath)
+
+        if self.deployed == 0:
+            tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+        cmd = "mintty -h never -w hide %s -c %s" % (
+            binPath, self.cfgDir)
+
+        taosadapterCmd = "mintty -h never -w hide %s " % (
+            taosadapterBinPath)
+        if os.system(taosadapterCmd) != 0:
+            tdLog.exit(taosadapterCmd)
+
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        self.running = 1
+        tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
+        if self.valgrind == 0:
+            time.sleep(0.1)
+            key = 'from offline to online'
+            bkey = bytes(key, encoding="utf8")
+            logFile = self.logDir + "/taosdlog.0"
+            i = 0
+            while not os.path.exists(logFile):
+                sleep(0.1)
+                i += 1
+                if i > 50:
+                    break
+            popen = subprocess.Popen(
+                'tail -n +0 -f ' + logFile,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=True)
+            pid = popen.pid
+            # print('Popen.pid:' + str(pid))
+            timeout = time.time() + 60 * 2
+            while True:
+                line = popen.stdout.readline().strip()
+                if bkey in line:
+                    popen.kill()
+                    break
+                if time.time() > timeout:
+                    tdLog.exit('wait too long for taosd start')
+            tdLog.debug("the dnode:%d has been started." % (self.index))
+        else:
+            tdLog.debug(
+                "wait 10 seconds for the dnode:%d to start." %
+                (self.index))
+            time.sleep(10)
+
     def startWithoutSleep(self):
-        buildPath = self.getBuildPath()
+        binPath = self.getPath()
 
-        if (buildPath == ""):
+        if (binPath == ""):
             tdLog.exit("taosd not found!")
         else:
-            tdLog.info("taosd found in %s" % buildPath)
+            tdLog.info("taosd found: %s" % binPath)
 
-        binPath = buildPath + "/build/bin/taosd"
-        taosadapterBinPath = buildPath + "/build/bin/taosadapter"
+        taosadapterBinPath = self.getPath("taosadapter")
+        if (taosadapterBinPath == ""):
+            tdLog.exit("taosAdapter not found!")
+        else:
+            tdLog.info("taosAdapter found: %s" % taosadapterBinPath)
 
         if self.deployed == 0:
             tdLog.exit("dnode:%d is not deployed" % (self.index))
@@ -372,14 +445,14 @@ class TDDnode:
 
         taosadapterPsCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % taosadapterToBeKilled
         taosadapterProcessID = subprocess.check_output(
-                    taosadapterPsCmd, shell=True).decode("utf-8")
+            taosadapterPsCmd, shell=True).decode("utf-8")
 
         while(taosadapterProcessID):
             taosadapterKillCmd = "kill -INT %s > /dev/null 2>&1" % taosadapterProcessID
             os.system(taosadapterKillCmd)
             time.sleep(1)
             taosadapterProcessID = subprocess.check_output(
-                    taosadapterPsCmd, shell=True).decode("utf-8")
+                taosadapterPsCmd, shell=True).decode("utf-8")
 
         if self.valgrind == 0:
             toBeKilled = "taosd"
@@ -547,7 +620,11 @@ class TDDnodes:
     def start(self, index):
         self.check(index)
         self.dnodes[index - 1].start()
-    
+
+    def startWin(self, index):
+        self.check(index)
+        self.dnodes[index - 1].startWin()
+
     def startWithoutSleep(self, index):
         self.check(index)
         self.dnodes[index - 1].startWithoutSleep()

tests/system-test/5-taos-tools/basic.py

@@ -23,33 +23,31 @@ import subprocess
 class TDTestCase:
     def caseDescription(self):
         '''
-        case1<pxiao>: [TD-11977] start taosdump without taosd   
+        case1<pxiao>: [TD-11977] start taosdump without taosd
         case1<pxiao>: [TD-11977] start taosBenchmark without taosd
         case1<pxiao>: [TD-11977] start taosAdaptor without taosd
-        ''' 
+        '''
         return
-    
+
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
     def run(self):
-        tdSql.prepare()        
-        
-        tools = ["taosdump", "taosBenchmark", "taosAdaptor"]                
+        tdSql.prepare()
+
+        tools = ["taosdump", "taosBenchmark", "taosadaptor"]
         tdDnodes.stop(1)
 
-        for tool in tools:            
-            path = tdDnodes.dnodes[1].getBuildPath(tool)
+        for tool in tools:
+            path = tdDnodes.dnodes[1].getPath(tool)
 
             try:
-                path += "/build/bin/" 
-                print(f"{path}{tool}")
                 if tool == "taosBenchmark":
-                    os.system(f"{path}{tool} -y")            
+                    os.system(f"{path} -y")
                 else:
-                    os.system(f"{path}{tool}")
-            except:
+                    os.system(f"{path}")
+            except BaseException:
                 pass
 
     def stop(self):