diff --git a/Jenkinsfile b/Jenkinsfile index 3cbcad14233a9de6bd78651a1c0a96d53643c3cd..9d986e742f654ca3b71a1af0cc7a629cb3eacc72 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,5 @@ import hudson.model.Result +import hudson.model.*; import jenkins.model.CauseOfInterruption properties([pipelineTriggers([githubPush()])]) node { @@ -6,6 +7,7 @@ node { } def skipbuild=0 +def win_stop=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -110,7 +112,83 @@ def pre_test(){ ''' return 1 } +def pre_test_win(){ + bat ''' + cd C:\\ + rd /s /Q C:\\TDengine + cd C:\\workspace\\TDinternal + rd /s /Q C:\\workspace\\TDinternal\\debug + cd C:\\workspace\\TDinternal\\community + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout 2.0 + ''' + } + else{ + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout develop + ''' + } + } + bat''' + cd C:\\workspace\\TDinternal\\community + git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + git clean -dfx + cd C:\\workspace\\TDinternal + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\TDinternal + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + bat ''' + cd C:\\workspace\\TDinternal + git checkout 2.0 + ''' + } + else{ + bat ''' + cd C:\\workspace\\TDinternal + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\TDinternal + git pull + date + git clean -dfx + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 + cmake ../ -G "NMake Makefiles" + nmake + nmake install + xcopy /e/y/i/f C:\\workspace\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + cd C:\\workspace\\TDinternal\\community\\src\\connector\\python + python -m pip install . + + ''' + return 1 +} pipeline { agent none environment{ @@ -236,9 +314,9 @@ pipeline { node nodejsChecker.js host=localhost ''' sh ''' - cd ${WKC}/tests/examples/C#/taosdemo - mcs -out:taosdemo *.cs > /dev/null 2>&1 - echo '' |./taosdemo -c /etc/taos + cd ${WKC}/tests/examples/C#/taosdemo + mcs -out:taosdemo *.cs > /dev/null 2>&1 + ./taosdemo -c /etc/taos -y ''' sh ''' cd ${WKC}/tests/gotest @@ -369,7 +447,37 @@ pipeline { date''' } } - } + } + + stage('build'){ + agent{label " wintest "} + steps { + pre_test() + script{ + while(win_stop == 0){ + sleep(1) + } + } + } + } + stage('test'){ + agent{label "win"} + steps{ + + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + pre_test_win() + bat''' + cd C:\\workspace\\TDinternal\\community\\tests\\pytest + .\\test-all.bat Wintest + ''' + } + script{ + win_stop=1 + } + } + } + + } } } @@ -451,4 +559,4 @@ pipeline { ) } } -} \ No newline at end of file +} diff --git a/tests/pytest/fulltest.bat b/tests/pytest/fulltest.bat new file mode 100644 index 0000000000000000000000000000000000000000..ad3803b01fc3733d2c17a7706e3dc4dee4d97f04 --- /dev/null +++ b/tests/pytest/fulltest.bat @@ -0,0 +1,21 @@ +python .\test.py -f insert\basic.py +python .\test.py -f insert\int.py +python .\test.py -f insert\float.py +python .\test.py -f insert\bigint.py +python .\test.py -f insert\bool.py +python .\test.py -f insert\double.py +python .\test.py -f insert\smallint.py +python .\test.py -f insert\tinyint.py +python .\test.py -f insert\date.py +python .\test.py -f insert\binary.py +python .\test.py -f insert\nchar.py + +python .\test.py -f query\filter.py +python .\test.py -f query\filterCombo.py +python .\test.py -f query\queryNormal.py +python .\test.py -f query\queryError.py +python .\test.py -f query\filterAllIntTypes.py +python .\test.py -f query\filterFloatAndDouble.py +python .\test.py -f query\filterOtherTypes.py +python .\test.py -f query\querySort.py +python .\test.py -f query\queryJoin.py \ No newline at end of file diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0cbb7876c6194041a160f8fee7271f0c76d3b90c..e91a20e65cd04dd64a88af88259e8e25eebf595c 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import platform import sys from util.log import * from util.cases import * @@ -53,9 +54,10 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') - result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) - if "Query OK" not in result: tdLog.exit("err:insert '';") + if platform.system() == "Linux": + config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') + result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) + if "Query OK" not in result: tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') diff --git a/tests/pytest/insert/nchar.py b/tests/pytest/insert/nchar.py index 5ad52b96a1555b3ccd622fd4bf88c7a0b26051b5..023da5b014864a2d010e6ec6acc16a33ccb20424 100644 --- a/tests/pytest/insert/nchar.py +++ b/tests/pytest/insert/nchar.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +import platform class TDTestCase: @@ -37,7 +38,7 @@ class TDTestCase: tdSql.error("insert into tb values (now, 'taosdata001')") - tdSql.error("insert into tb(now, 😀)") + if platform.system() == "Linux" : tdSql.error("insert into tb(now, 😀)") tdSql.query("select * from tb") tdSql.checkRows(2) diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 52e49a57c6883f6fe57df887756bbf2d27199806..a1789c8909f542ba3dcae83042ab50cde9e58e32 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -17,6 +17,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import * +import platform class TDTestCase: def init(self, conn, logSql): @@ -137,8 +138,9 @@ class TDTestCase: tdSql.checkData(1, 1, 421) tdSql.checkData(1, 2, "tm1") - tdDnodes.stop(1) - tdDnodes.start(1) + if platform.system() == "Linux": + tdDnodes.stop(1) + tdDnodes.start(1) tdSql.query("select last(*) from m1 group by tbname") tdSql.checkData(0, 0, "2020-03-01 01:01:01") diff --git a/tests/pytest/test-all.bat b/tests/pytest/test-all.bat new file mode 100644 index 0000000000000000000000000000000000000000..1f1e2c1727527e91f7632213992607d6221eac85 --- /dev/null +++ b/tests/pytest/test-all.bat @@ -0,0 +1,15 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( + echo Processing %%i + call %%i ARG1 -w 1 -m %1 > result.txt 2>error.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +) +exit + +:colorEcho +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i \ No newline at end of file diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 97dca6be1811ee87a31661e018616f469d5fd4ca..a96ac21496431b811f26fa82091c92f6ae8ecb9a 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -18,6 +18,7 @@ import getopt import subprocess import time from distutils.log import warn as printf +from fabric2 import Connection from util.log import * from util.dnodes import * @@ -35,8 +36,9 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help']) + windows = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -49,6 +51,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-c Test Cluster Flag') tdLog.printNoPrefix('-g valgrind Test Flag') tdLog.printNoPrefix('-r taosd restart test') + tdLog.printNoPrefix('-w taos on windows') sys.exit(0) if key in ['-r', '--restart']: @@ -81,6 +84,9 @@ if __name__ == "__main__": if key in ['-s', '--stop']: stop = 1 + if key in ['-w', '--windows']: + windows = 1 + if (stop != 0): if (valgrind == 0): toBeKilled = "taosd" @@ -111,66 +117,81 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') - tdDnodes.init(deployPath) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - is_test_framework = 0 - key_word = 'tdCases.addLinux' - try: - if key_word in open(fileName).read(): - is_test_framework = 1 - except: - pass - if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") - uModule = importlib.import_module(moduleName) - try: - ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) - else: - tdDnodes.deploy(1,{}) - tdDnodes.start(1) - if masterIp == "": host = '127.0.0.1' else: host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) - - tdCases.logSql(logSql) - - if testCluster: - tdLog.info("Procedures for testing cluster") - if fileName == "all": - tdCases.runAllCluster() - else: - tdCases.runOneCluster(fileName) - else: + if windows: + tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") + td_clinet = TDSimClient("C:\\TDengine") + td_clinet.deploy() + remote_conn = Connection("root@%s"%host) + with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn.run("python3 ./test.py") conn = taos.connect( - host, - config=tdDnodes.getSimCfgPath()) - if fileName == "all": - tdCases.runAllLinux(conn) + host="%s"%(host), + config=td_clinet.cfgDir) + tdCases.runOneWindows(conn, fileName) + else: + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addLinux' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + pass + tdDnodes.deploy(1,{}) + tdDnodes.start(1) + + + + tdCases.logSql(logSql) + + if testCluster: + tdLog.info("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) else: - tdCases.runOneLinux(conn, fileName) - if restart: - if fileName == "all": - tdLog.info("not need to query ") - else: - sp = fileName.rsplit(".", 1) - if len(sp) == 2 and sp[1] == "py": - tdDnodes.stopAll() - tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - tdLog.info("Procedures for tdengine deployed in %s" % (host)) - tdLog.info("query test after taosd restart") - tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + tdLog.info("Procedures for testing self-deployment") + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) else: - tdLog.info("not need to query") + tdCases.runOneWindows(conn, fileName) + if restart: + if fileName == "all": + tdLog.info("not need to query ") + else: + sp = fileName.rsplit(".", 1) + if len(sp) == 2 and sp[1] == "py": + tdDnodes.stopAll() + tdDnodes.start(1) + time.sleep(1) + conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + tdLog.info("query test after taosd restart") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + else: + tdLog.info("not need to query") conn.close() diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py index def50e04661b1752668202359eec7dd89df9b6f0..7e3c65a0a2f2e3c0b01977b0b28cb0ec8a2530ea 100644 --- a/tests/pytest/update/merge_commit_data2_update0.py +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -27,7 +27,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) - tdDnodes.startWithoutSleep(1) + tdDnodes.start(1) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 2fc1ac8515e47f9354483ebb590897eea96dcc57..fd3926a6f1bc79fee81c7d438dceb8eedcb7803d 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -34,7 +34,7 @@ class TDCases: self.clusterCases = [] def __dynamicLoadModule(self, fileName): - moduleName = fileName.replace(".py", "").replace("/", ".") + moduleName = fileName.replace(".py", "").replace(os.sep, ".") return importlib.import_module(moduleName, package='..') def logSql(self, logSql): @@ -80,7 +80,7 @@ class TDCases: runNum += 1 continue - def runAllWindows(self, conn): + def runAllWindows(self, conn, fileName): # TODO: load all Windows cases here runNum = 0 for tmp in self.windowsCases: @@ -101,12 +101,17 @@ class TDCases: for tmp in self.windowsCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() - case.init(conn) - case.run() + case.init(conn, self._logSql) + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 continue tdLog.notice("total %d Windows case(s) executed" % (runNum)) + def runAllCluster(self): # TODO: load all cluster case module here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index cd720ab62d4d71661a214ef24df8df81164fd71b..ff3c271cd8ab1ea2480f3d122513badab09016fc 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -14,15 +14,18 @@ import sys import os import os.path +import platform +import pathlib +import shutil import subprocess from time import sleep from util.log import * class TDSimClient: - def __init__(self): + def __init__(self, path): self.testCluster = False - + self.path = path self.cfgDict = { "numOfLogLines": "100000000", "numOfThreadsPerCore": "2.0", @@ -41,10 +44,7 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", "telemetryReporting": "0", - } - def init(self, path): - self.__init__() - self.path = path + } def getLogDir(self): self.logDir = "%s/sim/psim/log" % (self.path) @@ -61,35 +61,48 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) - + def os_string(self,path): + os_path = path.replace("/",os.sep) + return os_path def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - + self.logDir = self.os_string("%s/sim/psim/log" % (self.path)) + self.cfgDir = self.os_string("%s/sim/psim/cfg" % (self.path)) + self.cfgPath = self.os_string("%s/sim/psim/cfg/taos.cfg" % (self.path)) + + # cmd = "rm -rf " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + if os.path.exists(self.logDir): + try: + shutil.rmtree(self.logDir) + except: + tdLog.exit("del %s failed"%self.logDir) + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.logDir) + # cmd = "rm -rf " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + if os.path.exists(self.cfgDir): + try: + shutil.rmtree(self.cfgDir) + except: + tdLog.exit("del %s failed"%self.cfgDir) + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.cfgDir) + # cmd = "touch " + self.cfgPath + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + try: + pathlib.Path(self.cfgPath).touch() + except: + tdLog.exit("create %s failed"%self.cfgPath) if self.testCluster: self.cfg("masterIp", "192.168.0.1") self.cfg("secondIp", "192.168.0.2") @@ -246,7 +259,7 @@ class TDDnode: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if (("taosd") in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] @@ -404,7 +417,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -480,8 +493,7 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) + self.sim = TDSimClient(self.path) def setTestCluster(self, value): self.testCluster = value