Merge pull request #14244 from taosdata/fix/ZhiqiangWang/TD-16357-add-sim-full-test

test: add sim full test

tests/pytest/util/dnodes.py

@@ -508,7 +508,6 @@ class TDDnode:
 
     def stoptaosd(self):
         if (not self.remoteIP == ""):
-            print("123")
             self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
             tdLog.info("stop dnode%d"%self.index)
             return
@@ -518,18 +517,21 @@ class TDDnode:
             toBeKilled = "valgrind.bin"
 
         if self.running != 0:
-            psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
-            processID = subprocess.check_output(
-                psCmd, shell=True).decode("utf-8")
-
-            while(processID):
-                killCmd = "kill -INT %s > /dev/null 2>&1" % processID
-                os.system(killCmd)
-                time.sleep(1)
+            if platform.system().lower() == 'windows':
+                os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index)
+            else:
+                psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index)
                 processID = subprocess.check_output(
                     psCmd, shell=True).decode("utf-8")
-            if self.valgrind:
-                time.sleep(2)
+
+                while(processID):
+                    killCmd = "kill -INT %s > /dev/null 2>&1" % processID
+                    os.system(killCmd)
+                    time.sleep(1)
+                    processID = subprocess.check_output(
+                        psCmd, shell=True).decode("utf-8")
+                if self.valgrind:
+                    time.sleep(2)
 
             self.running = 0
             tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))

tests/script/test-all.bat

+@echo off
+SETLOCAL EnableDelayedExpansion
+for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do     rem"') do (  set "DEL=%%a")
+set /a a=0
+echo Windows Taosd Full Test
+set /a exitNum=0
+rm -rf failed.txt
+set caseFile="jenkins\\basic.txt"
+if not "%2" == "" (
+    set caseFile="%2"
+)
+for /F "usebackq tokens=*" %%i in (!caseFile!) do (
+    set line=%%i
+    if "!line:~,9!" == "./test.sh" (
+        set /a a+=1
+        echo !a! Processing %%i
+        call :GetTimeSeconds !time!
+        set time1=!_timeTemp!
+        echo Start at !time!
+        call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
+        if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+    )
+)
+exit !exitNum!
+
+:colorEcho
+set timeNow=%time%
+call :GetTimeSeconds %timeNow%
+set time2=%_timeTemp%
+set /a interTime=%time2% - %time1%
+echo End at %timeNow% , cast %interTime%s
+echo off
+<nul set /p ".=%DEL%" > "%~2"
+findstr /v /a:%1 /R "^$" "%~2" nul
+del "%~2" > nul 2>&1i
+goto :eof
+
+:GetTimeSeconds
+set tt=%1
+set tt=%tt:.= %
+set tt=%tt::= %
+set tt=%tt: 0= %
+set /a index=1
+for %%a in (%tt%) do (
+    if !index! EQU 1 (
+        set /a hh=%%a
+    )^
+    else if  !index! EQU 2 (
+        set /a mm=%%a
+ 
+    )^
+    else if  !index! EQU 3 (
+        set /a ss=%%a
+    )
+   set /a index=index+1
+)
+set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
+goto :eof

tests/system-test/6-cluster/5dnode1mnode.py

@@ -20,7 +20,7 @@ class MyDnodes(TDDnodes):
         self.simDeployed = False
         
 class TDTestCase:
-
+    noConn = True
     def init(self,conn ,logSql):
         tdLog.debug(f"start to excute {__file__}")
         self.TDDnodes = None
@@ -40,7 +40,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -81,7 +81,7 @@ class TDTestCase:
             dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
             dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
             dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
-            cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+            cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
             print(cmd)
             os.system(cmd)
         

tests/system-test/6-cluster/5dnode2mnode.py

@@ -20,6 +20,7 @@ class MyDnodes(TDDnodes):
         self.simDeployed = False
         
 class TDTestCase:
+    noConn = True
 
     def init(self,conn ,logSql):
         tdLog.debug(f"start to excute {__file__}")
@@ -40,7 +41,7 @@ class TDTestCase:
             projPath = selfPath[:selfPath.find("tests")]
 
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if ("taosd" in files or "taosd.exe" in files):
                 rootRealPath = os.path.dirname(os.path.realpath(root))
                 if ("packaging" not in rootRealPath):
                     buildPath = root[:len(root) - len("/build/bin")]
@@ -85,7 +86,7 @@ class TDTestCase:
             dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
             dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
             dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
-            cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+            cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\""
             print(cmd)
             os.system(cmd)
         

tests/system-test/7-tmq/tmqCommon.py

@@ -86,7 +86,13 @@ class TMQCom:
             shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes '
         
         if (platform.system().lower() == 'windows'):
-            shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath
+            processorName = buildPath + '\\build\\bin\\tmq_sim.exe'
+            if alias != 0:
+                processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe'
+                shellCmd = 'cp %s %s'%(processorName, processorNameNew)
+                os.system(shellCmd)
+                processorName = processorNameNew
+            shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath
             shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) 
             shellCmd += "> nul 2>&1 &"   
         else:

tests/system-test/7-tmq/tmqError.py

@@ -288,7 +288,10 @@ class TDTestCase:
             tdLog.exit("tmq consume rows error!")
 
         tdSql.query("drop topic %s"%topicFromStb1)
-        os.system('pkill tmq_sim')
+        if (platform.system().lower() == 'windows'):
+            os.system("TASKKILL /F /IM tmq_sim.exe")
+        else:
+            os.system('pkill tmq_sim')
 
         tdLog.printNoPrefix("======== test case 1 end ...... ")  
 

tests/system-test/test-all.bat

@@ -2,7 +2,7 @@
 SETLOCAL EnableDelayedExpansion
 for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do     rem"') do (  set "DEL=%%a")
 set /a a=0
-if %1 == full (
+if "%1" == "full" (
     echo Windows Taosd Full Test
     set /a exitNum=0
     del /Q /F failed.txt

tests/system-test/test.py

@@ -23,6 +23,7 @@ import platform
 import socket
 import threading
 from distutils.log import warn as printf
+from tkinter import N
 from fabric2 import Connection
 sys.path.append("../pytest")
 from util.log import *
@@ -187,9 +188,9 @@ if __name__ == "__main__":
 
     tdLog.info("Procedures for tdengine deployed in %s" % (host))
     if platform.system().lower() == 'windows':
+        fileName = fileName.replace("/", os.sep)
         if (masterIp == "" and not fileName[0:12] == "0-others\\udf"):
             threading.Thread(target=checkRunTimeError,daemon=True).start()
-        tdCases.logSql(logSql)
         tdLog.info("Procedures for testing self-deployment")
         tdDnodes.init(deployPath, masterIp)
         tdDnodes.setTestCluster(testCluster)
@@ -208,18 +209,46 @@ if __name__ == "__main__":
             uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()
-                if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)):
+                if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')):
                     updateCfgDict = ucase.updatecfgDict
                     updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode()
             except Exception as r:
                 print(r)
         else:
             pass
-        tdDnodes.deploy(1,updateCfgDict)
-        tdDnodes.start(1)
-        conn = taos.connect(
-            host="%s"%(host),
-            config=tdDnodes.sim.getCfgDir())
+        if dnodeNums == 1 :
+            tdDnodes.deploy(1,updateCfgDict)
+            tdDnodes.start(1)
+            tdCases.logSql(logSql)
+        else :
+            tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
+            dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums)
+            tdDnodes = ClusterDnodes(dnodeslist)
+            tdDnodes.init(deployPath, masterIp)
+            tdDnodes.setTestCluster(testCluster)
+            tdDnodes.setValgrind(valgrind)
+            tdDnodes.stopAll()
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.deploy(dnode.index,{})
+            for dnode in tdDnodes.dnodes:
+                tdDnodes.starttaosd(dnode.index)
+            tdCases.logSql(logSql)
+            conn = taos.connect(
+                host,
+                config=tdDnodes.getSimCfgPath())
+            print(tdDnodes.getSimCfgPath(),host)
+            cluster.create_dnode(conn)
+            try:
+                if cluster.check_dnode(conn) :
+                    print("check dnode ready")
+            except Exception as r:
+                print(r)
+        if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True:
+            conn = None
+        else:
+            conn = taos.connect(
+                host="%s"%(host),
+                config=tdDnodes.sim.getCfgDir())
         if is_test_framework:
             tdCases.runOneWindows(conn, fileName)
         else:
@@ -307,4 +336,5 @@ if __name__ == "__main__":
                     tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
                 else:
                     tdLog.info("not need to query")
-    conn.close()
+    if conn is not None:
+        conn.close()

tools/shell/src/shellEngine.c

@@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) {
     }
 
     fname = sptr + 2;
+    while (*fname == ' ') fname++;
     *sptr = '\0';
   }