Merge branch '3.0' of github.com:taosdata/TDengine into 3.0

tests/pytest/util/dnodes.py

@@ -388,14 +388,14 @@ class TDDnode:
             if os.system(cmd) != 0:
                 tdLog.exit(cmd)
         else:
-            self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.startWithoutSleep(%d)"%(self.index, self.index))
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.startWithoutSleep(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index))
 
         self.running = 1
         tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
 
     def stop(self):
         if (not self.remoteIP == ""):
-            self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index)
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
             tdLog.info("stop dnode%d"%self.index)
             return
         if self.valgrind == 0:
@@ -426,7 +426,7 @@ class TDDnode:
 
     def forcestop(self):
         if (not self.remoteIP == ""):
-            self.remoteExec(self.cfgDict, "tdDnodes.forcestop(%d)"%self.index)
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1))
             return
         if self.valgrind == 0:
             toBeKilled = "taosd"
@@ -497,46 +497,12 @@ class TDDnodes:
         self.killValgrind = 1
 
     def init(self, path, remoteIP = ""):
-        psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'"
-        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
-        while(processID):
-            killCmd = "kill -9 %s > /dev/null 2>&1" % processID
-            os.system(killCmd)
-            time.sleep(1)
-            processID = subprocess.check_output(
-                psCmd, shell=True).decode("utf-8")
-
-        if self.killValgrind == 1:
-            psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
-            processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
-            while(processID):
-                killCmd = "kill -9 %s > /dev/null 2>&1" % processID
-                os.system(killCmd)
-                time.sleep(1)
-                processID = subprocess.check_output(
-                    psCmd, shell=True).decode("utf-8")
-
         binPath = self.dnodes[0].getPath() + "/../../../"
         # tdLog.debug("binPath %s" % (binPath))
         binPath = os.path.realpath(binPath)
         # tdLog.debug("binPath real path %s" % (binPath))
 
-        # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
-        # tdLog.debug(cmd)
-        # os.system(cmd)
-
-        # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
-        # if os.system(cmd) != 0 :
-        #  tdLog.exit(cmd)
-        # tdLog.debug("execute %s" % (cmd))
-
-        # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
-        # if os.system(cmd) != 0 :
-        # tdLog.exit(cmd)
-        # tdLog.debug("execute %s" % (cmd))
-
         if path == "":
-            # self.path = os.path.expanduser('~')
             self.path = os.path.abspath(binPath + "../../")
         else:
             self.path = os.path.realpath(path)

tests/system-test/0-others/taosShellNetChk.py

@@ -187,50 +187,51 @@ class TDTestCase:
         # stop taosd
         tdDnodes.stop(1)        
 
-        role   = 'server'
-        if platform.system().lower() == 'windows':
-            taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c']
-            taosCmd = taosCmd.replace('\\','\\\\')
-            taosCmd = taosCmd + ' -n ' + role
-        else:
-            taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c']
-            taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &'
-        print (taosCmd)
-        os.system(taosCmd)
-
-        pktLen = '2000'
-        pktNum = '10'
-        role   = 'client'
-        if platform.system().lower() == 'windows':
-            taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c']
-            taosCmd = taosCmd.replace('\\','\\\\')
-        else:
-            taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c']
-        taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' +  pktNum
-        print (taosCmd)
-        child = taosExpect.spawn(taosCmd, timeout=3)
-        i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
-
-        if platform.system().lower() == 'windows':
-            retResult = child.before
-        else:
-            retResult = child.before.decode()
-        print("expect() return code: %d, content:\n %s\n"%(i, retResult))
-        #print(child.after.decode())
-        if i == 0:
-            tdLog.exit('taos -n server fail!')
-        
-        expectString1 = 'response is received, size:' + pktLen
-        expectSTring2 = pktNum + '/' + pktNum
-        if expectString1 in retResult and expectSTring2 in retResult:
-            tdLog.info("taos -n client success")
-        else:
-            tdLog.exit('taos -n client fail!')
+        try:
+            role   = 'server'
+            if platform.system().lower() == 'windows':
+                taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c']
+                taosCmd = taosCmd.replace('\\','\\\\')
+                taosCmd = taosCmd + ' -n ' + role
+            else:
+                taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c']
+                taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &'
+            print (taosCmd)
+            os.system(taosCmd)
+
+            pktLen = '2000'
+            pktNum = '10'
+            role   = 'client'
+            if platform.system().lower() == 'windows':
+                taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c']
+                taosCmd = taosCmd.replace('\\','\\\\')
+            else:
+                taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c']
+            taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' +  pktNum
+            print (taosCmd)
+            child = taosExpect.spawn(taosCmd, timeout=3)
+            i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
 
-        if platform.system().lower() == 'windows':
-            os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
-        else:
-            os.system('pkill taos')
+            if platform.system().lower() == 'windows':
+                retResult = child.before
+            else:
+                retResult = child.before.decode()
+            print("expect() return code: %d, content:\n %s\n"%(i, retResult))
+            #print(child.after.decode())
+            if i == 0:
+                tdLog.exit('taos -n server fail!')
+            
+            expectString1 = 'response is received, size:' + pktLen
+            expectSTring2 = pktNum + '/' + pktNum
+            if expectString1 in retResult and expectSTring2 in retResult:
+                tdLog.info("taos -n client success")
+            else:
+                tdLog.exit('taos -n client fail!')
+        finally:
+            if platform.system().lower() == 'windows':
+                os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
+            else:
+                os.system('pkill taos')
 
     def stop(self):
         tdSql.close()

tests/system-test/0-others/telemetry.py

@@ -8,6 +8,7 @@ import http.server
 import gzip
 import threading
 import json
+import pickle
 
 from util.log import *
 from util.sql import *
@@ -136,13 +137,19 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler):
         telemetryInfoCheck(infoDict)
 
         # 4. shutdown the server and exit case 
-        assassin = threading.Thread(target=httpServer.shutdown)
+        assassin = threading.Thread(target=self.server.shutdown)
         assassin.daemon = True
         assassin.start()
         print ("==== shutdown http server ====")
 
 class TDTestCase:
     hostname = socket.gethostname()
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP)
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     serverPort = '7080'
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
@@ -177,17 +184,20 @@ class TDTestCase:
         sql = "create database db3 vgroups " + vgroups
         tdSql.query(sql)
 
-        # loop to wait request
-        httpServer.serve_forever()
+        # create http server: bing ip/port , and  request processor
+        if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+            RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode()
+            telemetryInfoCheckStr = base64.b64encode(pickle.dumps(telemetryInfoCheck)).decode()
+            cmdStr = "import pickle\nimport http\ntelemetryInfoCheck=pickle.loads(base64.b64decode(\"%s\".encode()))\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nhttp.server.HTTPServer((\"\", %d), RequestHandlerImpl).serve_forever()"%(telemetryInfoCheckStr,RequestHandlerImplStr,int(telemetryPort))
+            tdDnodes.dnodes[0].remoteExec({}, cmdStr)
+        else:
+            serverAddress = ("", int(telemetryPort))
+            http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever()
 
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")
 
-# create http server: bing ip/port , and  request processor
-serverAddress = ("", int(telemetryPort))
-httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl)
-
 tdCases.addLinux(__file__, TDTestCase())
 tdCases.addWindows(__file__, TDTestCase())
 

tests/system-test/0-others/udf_create.py

@@ -9,6 +9,8 @@ from util.sql import *
 from util.cases import *
 from util.dnodes import *
 import subprocess
+# import win32gui
+# import threading
 
 class TDTestCase:
 
@@ -533,8 +535,17 @@ class TDTestCase:
         return udf1_sqls ,udf2_sqls
 
 
+    # def checkRunTimeError(self):
+    #     while 1:
+    #         time.sleep(1)
+    #         hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library")
+    #         if hwnd:
+    #             os.system("TASKKILL /F /IM udfd.exe")
 
     def unexpected_create(self):
+        # if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""):
+        #     checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True)
+        #     checkErrorThread.start()
 
         tdLog.info(" create function with out bufsize ")
         tdSql.query("drop function udf1 ")

tests/system-test/fulltest.bat

 
-python3 .\test.py -f 0-others\taosShell.py
+@REM python3 .\test.py -f 0-others\taosShell.py
 python3 .\test.py -f 0-others\taosShellError.py
 python3 .\test.py -f 0-others\taosShellNetChk.py
+python3 .\test.py -f 0-others\telemetry.py
+@REM python3 .\test.py -f 0-others\taosdMonitor.py
 python3 .\test.py -f 0-others\udfTest.py
 python3 .\test.py -f 0-others\udf_create.py
-python3 .\test.py -f 0-others\udf_restart_taosd.py
\ No newline at end of file
+python3 .\test.py -f 0-others\udf_restart_taosd.py
+@REM python3 .\test.py -f 0-others\cachelast.py
+
+@REM python3 .\test.py -f 0-others\user_control.py
+@REM python3 .\test.py -f 0-others\fsync.py
+
+@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
+@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
+@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
+@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py
+@REM python3 .\test.py -f 1-insert\alter_stable.py
+@REM python3 .\test.py -f 1-insert\alter_table.py
+@REM python3 .\test.py -f 2-query\between.py
+@REM python3 .\test.py -f 2-query\distinct.py
+@REM python3 .\test.py -f 2-query\varchar.py
+@REM python3 .\test.py -f 2-query\ltrim.py
+@REM python3 .\test.py -f 2-query\rtrim.py
+@REM python3 .\test.py -f 2-query\length.py
+@REM python3 .\test.py -f 2-query\char_length.py
+@REM python3 .\test.py -f 2-query\upper.py
+@REM python3 .\test.py -f 2-query\lower.py
+@REM python3 .\test.py -f 2-query\join.py
+@REM python3 .\test.py -f 2-query\join2.py
+@REM python3 .\test.py -f 2-query\cast.py
+@REM python3 .\test.py -f 2-query\union.py
+@REM python3 .\test.py -f 2-query\union1.py
+@REM python3 .\test.py -f 2-query\concat.py
+@REM python3 .\test.py -f 2-query\concat2.py
+@REM python3 .\test.py -f 2-query\concat_ws.py
+@REM python3 .\test.py -f 2-query\concat_ws2.py
+@REM python3 .\test.py -f 2-query\check_tsdb.py
+@REM python3 .\test.py -f 2-query\spread.py
+@REM python3 .\test.py -f 2-query\hyperloglog.py
+
+
+@REM python3 .\test.py -f 2-query\timezone.py
+@REM python3 .\test.py -f 2-query\Now.py
+@REM python3 .\test.py -f 2-query\Today.py
+@REM python3 .\test.py -f 2-query\max.py
+@REM python3 .\test.py -f 2-query\min.py
+@REM python3 .\test.py -f 2-query\count.py
+@REM python3 .\test.py -f 2-query\last.py
+@REM python3 .\test.py -f 2-query\first.py
+@REM python3 .\test.py -f 2-query\To_iso8601.py
+@REM python3 .\test.py -f 2-query\To_unixtimestamp.py
+@REM python3 .\test.py -f 2-query\timetruncate.py
+@REM python3 .\test.py -f 2-query\diff.py
+@REM python3 .\test.py -f 2-query\Timediff.py
+
+@REM python3 .\test.py -f 2-query\top.py
+@REM python3 .\test.py -f 2-query\bottom.py
+@REM python3 .\test.py -f 2-query\percentile.py
+@REM python3 .\test.py -f 2-query\apercentile.py
+@REM python3 .\test.py -f 2-query\abs.py
+@REM python3 .\test.py -f 2-query\ceil.py
+@REM python3 .\test.py -f 2-query\floor.py
+@REM python3 .\test.py -f 2-query\round.py
+@REM python3 .\test.py -f 2-query\log.py
+@REM python3 .\test.py -f 2-query\pow.py
+@REM python3 .\test.py -f 2-query\sqrt.py
+@REM python3 .\test.py -f 2-query\sin.py
+@REM python3 .\test.py -f 2-query\cos.py
+@REM python3 .\test.py -f 2-query\tan.py
+@REM python3 .\test.py -f 2-query\arcsin.py
+@REM python3 .\test.py -f 2-query\arccos.py
+@REM python3 .\test.py -f 2-query\arctan.py
+@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py
+@REM # python3 .\test.py -f 2-query\nestedQuery.py
+@REM # TD-15983 subquery output duplicate name column. 
+@REM # Please Xiangyang Guo modify the following script
+@REM # python3 .\test.py -f 2-query\nestedQuery_str.py
+
+@REM python3 .\test.py -f 2-query\avg.py
+@REM python3 .\test.py -f 2-query\elapsed.py
+@REM python3 .\test.py -f 2-query\csum.py
+@REM python3 .\test.py -f 2-query\mavg.py
+@REM python3 .\test.py -f 2-query\diff.py
+@REM python3 .\test.py -f 2-query\sample.py
+@REM python3 .\test.py -f 2-query\function_diff.py
+@REM python3 .\test.py -f 2-query\unique.py
+@REM python3 .\test.py -f 2-query\stateduration.py
+@REM python3 .\test.py -f 2-query\function_stateduration.py
+@REM python3 .\test.py -f 2-query\statecount.py
+
+@REM python3 .\test.py -f 7-tmq\basic5.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb0.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb1.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb0.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb1.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb2.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb3.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb4.py
+@REM python3 .\test.py -f 7-tmq\db.py
\ No newline at end of file

tests/system-test/test-all.bat

@@ -11,10 +11,12 @@ set /a a=0
 @REM )
 echo Linux Taosd Test
 for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
-    echo Processing %%i
-    set /a a+=1
-    call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt
-    if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
+    for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
+        echo Processing %%i
+        set /a a+=1
+        call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt
+        if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
+    )
 )
 exit