From aaa1da1bf3fcfba7370c4a917f0a7fbd118fb3ef Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 17 Dec 2021 15:39:14 +0800 Subject: [PATCH] [TD-12111]update CI runstep --- Jenkinsfile | 331 +++++++++---------------------------- tests/develop-test/test.py | 119 +++++++------ tests/system-test/test.py | 119 +++++++------ tests/test-CI.sh | 218 ++++++++++++++++++++++++ 4 files changed, 425 insertions(+), 362 deletions(-) create mode 100755 tests/test-CI.sh diff --git a/Jenkinsfile b/Jenkinsfile index e82888c5bf..ef055bec64 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -359,60 +359,31 @@ pipeline { script{ abort_previous() abortPreviousBuilds() - println env.BRANCH_NAME,env.CHANGE_BRANCH + println env.CHANGE_BRANCH if(env.CHANGE_FORK){ - scope = ['full'] + scope = ['connector','query','insert','other','tools'] } else{ sh''' - cd ${WKC} - git fetch - git checkout ${BRANCH_NAME} - git pull + cd ${WKC} + git fetch + git checkout ${CHANGE_BRANCH} + git pull ''' - gitlog = sh(script: "git log -1 --pretty=%B ", returnStdout:true) - fun_git = (gitlog =~ \(.?\)) - scope = fun_git.split(",") - println scope - } - } - // sh''' - // rm -rf ${WORKSPACE}.tes - // cp -r ${WORKSPACE} ${WORKSPACE}.tes - // cd ${WORKSPACE}.tes - // git fetch - // ''' - // script { - // if (env.CHANGE_TARGET == 'master') { - // sh ''' - // git checkout master - // ''' - // } - // else if(env.CHANGE_TARGET == '2.0'){ - // sh ''' - // git checkout 2.0 - // ''' - // } - // else{ - // sh ''' - // git checkout develop - // ''' - // } - // } - // sh''' - // git fetch origin +refs/pull/${CHANGE_ID}/merge - // git checkout -qf FETCH_HEAD - // ''' + dir('/var/lib/jenkins/workspace/TDinternal/community'){ + gitlog = sh(script: "git log -1 --pretty=%B ", returnStdout:true) + println gitlog + if (!(gitlog =~ /\((.*)\)/)){ + autoCancelled = true + error('Aborting the build.') + } + temp = (gitlog =~ /\((.*)\)/) + temp = temp[0].remove(1) + scope = temp.split(",") + } - // script{ - // skipbuild='2' - // skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true) - // println skipbuild - // } - // sh''' - // rm -rf ${WORKSPACE}.tes - // ''' - // } + } + } } } stage('Parallel test stage') { @@ -425,239 +396,90 @@ pipeline { } } parallel { - stage('python_1_s1') { - agent{label " slave1 || slave11 "} + stage('python_1') { + agent{label " slave1 || slave 6 || slave11 || slave16 "} steps { pre_test() timeout(time: 55, unit: 'MINUTES'){ - sh ''' - date - cd ${WKC}/tests - ./test-all.sh p1 - date''' - } + script{ + scope.each { + sh """ + date + cd ${WKC}/tests + ./test-CI.sh ${it} p1 + date""" + } + } + } } } - stage('python_2_s5') { - agent{label " slave5 || slave15 "} + stage('python_2') { + agent{label " slave2 || slave7 || slave12 || slave17 "} steps { pre_test() timeout(time: 55, unit: 'MINUTES'){ - sh ''' - date - cd ${WKC}/tests - ./test-all.sh p2 - date''' - } - } - } - stage('python_3_s6') { - agent{label " slave6 || slave16 "} - steps { - timeout(time: 55, unit: 'MINUTES'){ - pre_test() - sh ''' - date - cd ${WKC}/tests - ./test-all.sh p3 - date''' + script{ + scope.each { + sh """ + date + cd ${WKC}/tests + ./test-CI.sh ${it} p2 + date""" + } + } } } } - stage('test_b1_s2') { - agent{label " slave2 || slave12 "} + stage('python_3') { + agent{label " slave3 || slave8 || slave13 ||slave18 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() - sh ''' - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - nohup taosd >/dev/null & - sleep 10 - ''' - - sh ''' - cd ${WKC}/src/connector/python - export PYTHONPATH=$PWD/ - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - pip3 install pytest - pytest tests/ - - python3 examples/bind-multi.py - python3 examples/bind-row.py - python3 examples/demo.py - python3 examples/insert-lines.py - python3 examples/pep-249.py - python3 examples/query-async.py - python3 examples/query-objectively.py - python3 examples/subscribe-sync.py - python3 examples/subscribe-async.py - ''' - - sh ''' - cd ${WKC}/src/connector/nodejs - npm install - npm run test - cd ${WKC}/tests/examples/nodejs - npm install td2.0-connector > /dev/null 2>&1 - node nodejsChecker.js host=localhost - node test1970.js - cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport - npm install td2.0-connector > /dev/null 2>&1 - node nanosecondTest.js - ''' - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/src/connector/C# - dotnet test - dotnet run --project src/test/Cases/Cases.csproj - - cd ${WKC}/tests/examples/C# - dotnet run --project C#checker/C#checker.csproj - dotnet run --project TDengineTest/TDengineTest.csproj - dotnet run --project schemaless/schemaless.csproj - - cd ${WKC}/tests/examples/C#/taosdemo - dotnet build -c Release - tree | true - ./bin/Release/net5.0/taosdemo -c /etc/taos -y - ''' + script{ + scope.each { + sh """ + date + cd ${WKC}/tests + ./test-CI.sh ${it} p3 + date""" + } } - sh ''' - cd ${WKC}/tests/gotest - bash batchtest.sh - ''' - sh ''' - cd ${WKC}/tests - ./test-all.sh b1fq - date''' } } } - stage('test_crash_gen_s3') { - agent{label " slave3 || slave13 "} - - steps { - pre_test() - timeout(time: 60, unit: 'MINUTES'){ - sh ''' - cd ${WKC}/tests/pytest - ./crash_gen.sh -a -p -t 4 -s 2000 - ''' - } - timeout(time: 60, unit: 'MINUTES'){ - sh ''' - cd ${WKC}/tests/pytest - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - ./handle_crash_gen_val_log.sh - ''' - sh ''' - cd ${WKC}/tests/pytest - rm -rf /var/lib/taos/* - rm -rf /var/log/taos/* - ./handle_taosd_val_log.sh - ''' - } - timeout(time: 55, unit: 'MINUTES'){ - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b2fq - date - ''' - } - } - } - stage('test_valgrind_s4') { - agent{label " slave4 || slave14 "} - - steps { - pre_test() - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - cd ${WKC}/tests/pytest - ./valgrind-test.sh 2>&1 > mem-error-out.log - ./handle_val_log.sh - ''' - } - timeout(time: 55, unit: 'MINUTES'){ - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b3fq - date''' - sh ''' - date - cd ${WKC}/tests - ./test-all.sh full example - date''' - } - } - } - stage('test_b4_s7') { - agent{label " slave7 || slave17 "} - steps { - timeout(time: 105, unit: 'MINUTES'){ - pre_test() - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b4fq - cd ${WKC}/tests - ./test-all.sh p4 - ''' - // cd ${WKC}/tests - // ./test-all.sh full jdbc - // cd ${WKC}/tests - // ./test-all.sh full unit - } - } - } - stage('test_b5_s8') { - agent{label " slave8 || slave18 "} - steps { - timeout(time: 55, unit: 'MINUTES'){ - pre_test() - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b5fq - date''' - } - } - } - stage('test_b6_s9') { - agent{label " slave9 || slave19 "} + stage('python_4') { + agent{label " slave4 || slave9 || slave14 || slave19 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() - sh ''' - cd ${WKC}/tests - ./test-all.sh develop-test - ''' - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b6fq - date''' + script{ + scope.each { + sh """ + date + cd ${WKC}/tests + ./test-CI.sh ${it} p4 + date""" + } + } + } } } - stage('test_b7_s10') { - agent{label " slave10 || slave20 "} + stage('python_5') { + agent{label " slave5 || slave10 || slave15 || slave20 "} steps { timeout(time: 55, unit: 'MINUTES'){ pre_test() - sh ''' - cd ${WKC}/tests - ./test-all.sh system-test - ''' - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b7fq - date''' + script{ + scope.each { + sh """ + date + cd ${WKC}/tests + ./test-CI.sh ${it} p5 + date""" + } + } + } } } @@ -830,3 +652,4 @@ pipeline { } } } + diff --git a/tests/develop-test/test.py b/tests/develop-test/test.py index b39b95c903..ad12198129 100644 --- a/tests/develop-test/test.py +++ b/tests/develop-test/test.py @@ -28,7 +28,6 @@ import taos if __name__ == "__main__": - fileName = "all" deployPath = "" masterIp = "" @@ -38,68 +37,83 @@ if __name__ == "__main__": stop = 0 restart = False windows = 0 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) + opts, args = getopt.gnu_getopt( + sys.argv[1:], + "f:p:m:l:scghrw", + [ + "file=", + "path=", + "master", + "logSql", + "stop", + "cluster", + "valgrind", + "help", + "windows", + ], + ) for key, value in opts: - if key in ['-h', '--help']: - tdLog.printNoPrefix( - 'A collection of test cases written using Python') - tdLog.printNoPrefix('-f Name of test case file written by Python') - tdLog.printNoPrefix('-p Deploy Path for Simulator') - tdLog.printNoPrefix('-m Master Ip for Simulator') - tdLog.printNoPrefix('-l logSql Flag') - tdLog.printNoPrefix('-s stop All dnodes') - tdLog.printNoPrefix('-c Test Cluster Flag') - tdLog.printNoPrefix('-g valgrind Test Flag') - tdLog.printNoPrefix('-r taosd restart test') - tdLog.printNoPrefix('-w taos on windows') + if key in ["-h", "--help"]: + tdLog.printNoPrefix("A collection of test cases written using Python") + tdLog.printNoPrefix("-f Name of test case file written by Python") + tdLog.printNoPrefix("-p Deploy Path for Simulator") + tdLog.printNoPrefix("-m Master Ip for Simulator") + tdLog.printNoPrefix("-l logSql Flag") + tdLog.printNoPrefix("-s stop All dnodes") + tdLog.printNoPrefix("-c Test Cluster Flag") + tdLog.printNoPrefix("-g valgrind Test Flag") + tdLog.printNoPrefix("-r taosd restart test") + tdLog.printNoPrefix("-w taos on windows") sys.exit(0) - if key in ['-r', '--restart']: + if key in ["-r", "--restart"]: restart = True - if key in ['-f', '--file']: + if key in ["-f", "--file"]: fileName = value - if key in ['-p', '--path']: + if key in ["-p", "--path"]: deployPath = value - if key in ['-m', '--master']: + if key in ["-m", "--master"]: masterIp = value - if key in ['-l', '--logSql']: - if (value.upper() == "TRUE"): + if key in ["-l", "--logSql"]: + if value.upper() == "TRUE": logSql = True - elif (value.upper() == "FALSE"): + elif value.upper() == "FALSE": logSql = False else: tdLog.printNoPrefix("logSql value %s is invalid" % logSql) sys.exit(0) - if key in ['-c', '--cluster']: + if key in ["-c", "--cluster"]: testCluster = True - if key in ['-g', '--valgrind']: + if key in ["-g", "--valgrind"]: valgrind = 1 - if key in ['-s', '--stop']: + if key in ["-s", "--stop"]: stop = 1 - if key in ['-w', '--windows']: + if key in ["-w", "--windows"]: windows = 1 - if (stop != 0): - if (valgrind == 0): + if stop != 0: + if valgrind == 0: toBeKilled = "taosd" else: toBeKilled = "valgrind.bin" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + killCmd = ( + "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" + % toBeKilled + ) psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) - while(processID): + while processID: os.system(killCmd) time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) @@ -116,25 +130,26 @@ if __name__ == "__main__": if valgrind: time.sleep(2) - tdLog.info('stop All dnodes') - + tdLog.info("stop All dnodes") + if masterIp == "": - host = '127.0.0.1' + host = "127.0.0.1" else: host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) + uModule = tdDnodes.import_module(fileName) if windows: tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") td_clinet = TDSimClient("C:\\TDengine") td_clinet.deploy() - remote_conn = Connection("root@%s"%host) - with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn = Connection("root@%s" % host) + with remote_conn.cd( + "/var/lib/jenkins/workspace/TDinternal/community/tests/pytest" + ): remote_conn.run("python3 ./test.py") - conn = taos.connect( - host="%s"%(host), - config=td_clinet.cfgDir) + conn = taos.connect(host="%s" % (host), config=td_clinet.cfgDir) tdCases.runOneWindows(conn, fileName) else: tdDnodes.init(deployPath) @@ -142,27 +157,24 @@ if __name__ == "__main__": tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() is_test_framework = 0 - key_word = 'tdCases.addLinux' + key_word = "tdCases.addLinux" try: if key_word in open(fileName).read(): is_test_framework = 1 except: pass if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") - uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, ucase.updatecfgDict) + del uModule + except: + tdDnodes.deploy(1, {}) else: pass - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, {}) tdDnodes.start(1) - - tdCases.logSql(logSql) if testCluster: @@ -173,26 +185,25 @@ if __name__ == "__main__": tdCases.runOneCluster(fileName) else: tdLog.info("Procedures for testing self-deployment") - conn = taos.connect( - host, - config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneWindows(conn, fileName) + tdCases.runOneLinux(conn, fileName, uModule) if restart: if fileName == "all": tdLog.info("not need to query ") - else: + else: sp = fileName.rsplit(".", 1) if len(sp) == 2 and sp[1] == "py": tdDnodes.stopAll() tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + time.sleep(1) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: tdLog.info("not need to query") conn.close() + diff --git a/tests/system-test/test.py b/tests/system-test/test.py index b39b95c903..ad12198129 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -28,7 +28,6 @@ import taos if __name__ == "__main__": - fileName = "all" deployPath = "" masterIp = "" @@ -38,68 +37,83 @@ if __name__ == "__main__": stop = 0 restart = False windows = 0 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) + opts, args = getopt.gnu_getopt( + sys.argv[1:], + "f:p:m:l:scghrw", + [ + "file=", + "path=", + "master", + "logSql", + "stop", + "cluster", + "valgrind", + "help", + "windows", + ], + ) for key, value in opts: - if key in ['-h', '--help']: - tdLog.printNoPrefix( - 'A collection of test cases written using Python') - tdLog.printNoPrefix('-f Name of test case file written by Python') - tdLog.printNoPrefix('-p Deploy Path for Simulator') - tdLog.printNoPrefix('-m Master Ip for Simulator') - tdLog.printNoPrefix('-l logSql Flag') - tdLog.printNoPrefix('-s stop All dnodes') - tdLog.printNoPrefix('-c Test Cluster Flag') - tdLog.printNoPrefix('-g valgrind Test Flag') - tdLog.printNoPrefix('-r taosd restart test') - tdLog.printNoPrefix('-w taos on windows') + if key in ["-h", "--help"]: + tdLog.printNoPrefix("A collection of test cases written using Python") + tdLog.printNoPrefix("-f Name of test case file written by Python") + tdLog.printNoPrefix("-p Deploy Path for Simulator") + tdLog.printNoPrefix("-m Master Ip for Simulator") + tdLog.printNoPrefix("-l logSql Flag") + tdLog.printNoPrefix("-s stop All dnodes") + tdLog.printNoPrefix("-c Test Cluster Flag") + tdLog.printNoPrefix("-g valgrind Test Flag") + tdLog.printNoPrefix("-r taosd restart test") + tdLog.printNoPrefix("-w taos on windows") sys.exit(0) - if key in ['-r', '--restart']: + if key in ["-r", "--restart"]: restart = True - if key in ['-f', '--file']: + if key in ["-f", "--file"]: fileName = value - if key in ['-p', '--path']: + if key in ["-p", "--path"]: deployPath = value - if key in ['-m', '--master']: + if key in ["-m", "--master"]: masterIp = value - if key in ['-l', '--logSql']: - if (value.upper() == "TRUE"): + if key in ["-l", "--logSql"]: + if value.upper() == "TRUE": logSql = True - elif (value.upper() == "FALSE"): + elif value.upper() == "FALSE": logSql = False else: tdLog.printNoPrefix("logSql value %s is invalid" % logSql) sys.exit(0) - if key in ['-c', '--cluster']: + if key in ["-c", "--cluster"]: testCluster = True - if key in ['-g', '--valgrind']: + if key in ["-g", "--valgrind"]: valgrind = 1 - if key in ['-s', '--stop']: + if key in ["-s", "--stop"]: stop = 1 - if key in ['-w', '--windows']: + if key in ["-w", "--windows"]: windows = 1 - if (stop != 0): - if (valgrind == 0): + if stop != 0: + if valgrind == 0: toBeKilled = "taosd" else: toBeKilled = "valgrind.bin" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + killCmd = ( + "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" + % toBeKilled + ) psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled processID = subprocess.check_output(psCmd, shell=True) - while(processID): + while processID: os.system(killCmd) time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) @@ -116,25 +130,26 @@ if __name__ == "__main__": if valgrind: time.sleep(2) - tdLog.info('stop All dnodes') - + tdLog.info("stop All dnodes") + if masterIp == "": - host = '127.0.0.1' + host = "127.0.0.1" else: host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) + uModule = tdDnodes.import_module(fileName) if windows: tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") td_clinet = TDSimClient("C:\\TDengine") td_clinet.deploy() - remote_conn = Connection("root@%s"%host) - with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn = Connection("root@%s" % host) + with remote_conn.cd( + "/var/lib/jenkins/workspace/TDinternal/community/tests/pytest" + ): remote_conn.run("python3 ./test.py") - conn = taos.connect( - host="%s"%(host), - config=td_clinet.cfgDir) + conn = taos.connect(host="%s" % (host), config=td_clinet.cfgDir) tdCases.runOneWindows(conn, fileName) else: tdDnodes.init(deployPath) @@ -142,27 +157,24 @@ if __name__ == "__main__": tdDnodes.setValgrind(valgrind) tdDnodes.stopAll() is_test_framework = 0 - key_word = 'tdCases.addLinux' + key_word = "tdCases.addLinux" try: if key_word in open(fileName).read(): is_test_framework = 1 except: pass if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") - uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, ucase.updatecfgDict) + del uModule + except: + tdDnodes.deploy(1, {}) else: pass - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, {}) tdDnodes.start(1) - - tdCases.logSql(logSql) if testCluster: @@ -173,26 +185,25 @@ if __name__ == "__main__": tdCases.runOneCluster(fileName) else: tdLog.info("Procedures for testing self-deployment") - conn = taos.connect( - host, - config=tdDnodes.getSimCfgPath()) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneWindows(conn, fileName) + tdCases.runOneLinux(conn, fileName, uModule) if restart: if fileName == "all": tdLog.info("not need to query ") - else: + else: sp = fileName.rsplit(".", 1) if len(sp) == 2 and sp[1] == "py": tdDnodes.stopAll() tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + time.sleep(1) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: tdLog.info("not need to query") conn.close() + diff --git a/tests/test-CI.sh b/tests/test-CI.sh new file mode 100755 index 0000000000..731cc3c367 --- /dev/null +++ b/tests/test-CI.sh @@ -0,0 +1,218 @@ +#!/bin/bash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +tests_dir=`pwd` +IN_TDINTERNAL="community" + +function stopTaosd { + echo "Stop taosd" + sudo systemctl stop taosd || echo 'no sudo or systemctl or stop fail' + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function dohavecore(){ + corefile=`find $corepath -mmin 1` + if [ -n "$corefile" ];then + core_file=`echo $corefile|cut -d " " -f2` + proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,` + echo 'taosd or taos has generated core' + rm case.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then + cd ../../../ + tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + if [[ $2 == 1 ]];then + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` + else + cd community + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + else + cd ../../ + if [[ $1 == 1 ]];then + tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + fi + if [[ $1 == 1 ]];then + echo '\n'|gdb $proc $core_file -ex "bt 10" -ex quit + exit 8 + fi + fi +} + + +function runPyCaseOneByOne { + while read -r line; do + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi + start_time=`date +%s` + date +%F\ %T | tee -a pytest-out.log + echo -n $case + $line > /dev/null 2>&1 && \ + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log + end_time=`date +%s` + out_log=`tail -1 pytest-out.log ` + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + fi + done < $1 +} + +function runPyCaseOneByOnefq() { + start=`sed -n "/$2-start/=" $1` + end=`sed -n "/$2-end/=" $1` + + for ((i=$start;i<=$end;i++)) ; do + line=`sed -n "$i"p $1` + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi + start_time=`date +%s` + date +%F\ %T | tee -a pytest-out.log + echo -n $case + if [[ $1 =~ full ]] ; then + line=$line" -s" + fi + $line > case.log 2>&1 && \ + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log + end_time=`date +%s` + out_log=`tail -1 pytest-out.log ` + if [[ $out_log =~ 'failed' ]];then + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + echo '=====================log===================== ' + cat case.log + rm -rf case.log + dohavecore $3 2 + if [[ $3 == 1 ]];then + exit 8 + fi + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + dohavecore $3 2 + fi + done + rm -rf ../../sim/case.log +} + +###################### +# main entry +###################### + +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) OS=Linux;; + Darwin*) OS=Darwin;; + CYGWIN*) OS=Windows;; + *) OS=Unknown;; +esac + +case "${OS}" in + Linux*) TAOSLIB=libtaos.so;; + Darwin*) TAOSLIB=libtaos.dylib;; + Windows*) TAOSLIB=taos.dll;; + Unknown) TAOSLIB="UNKNOWN:${unameOut}";; +esac + +echo TAOSLIB is ${TAOSLIB} + +totalFailed=0 +totalPyFailed=0 +totalJDBCFailed=0 +totalUnitFailed=0 +totalExampleFailed=0 +totalApiFailed=0 + +if [ "${OS}" == "Linux" ]; then + corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` + if [ -z "$corepath" ];then + echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied" + corepath="/coredump/" + fi +fi + + +echo "### run Python test case ###" + +cd $tests_dir + +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../ +fi + +TOP_DIR=`pwd` +TAOSLIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1` +if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then + LIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1|cut -d '/' -f 2,3,4,5` +else + LIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1|cut -d '/' -f 2,3,4` +fi + +export LD_LIBRARY_PATH=$TOP_DIR/$LIB_DIR:$LD_LIBRARY_PATH + +cd $tests_dir/pytest + +[ -f pytest-out.log ] && rm -f pytest-out.log + +if [ "$1" == "full" ]; then + echo "### run Python full test ###" + runPyCaseOneByOne fulltest-tools.sh + runPyCaseOneByOne fulltest-query.sh + runPyCaseOneByOne fulltest-other.sh + runPyCaseOneByOne fulltest-insert.sh + runPyCaseOneByOne fulltest-connector.sh +else + echo "### run $1 $2 test ###" + if [ "$1" != "query" ] && [ "$1" != "other" ] && [ "$1" != "tools" ] && [ "$1" != "insert" ] && [ "$1" != "connector" ] ;then + echo " wrong option:$1 must one of [queyr,other,tools,insert,connector]" + exit 8 + fi + runPyCaseOneByOnefq fulltest-$1.sh $2 1 +fi +totalPySuccess=`grep 'success' pytest-out.log | wc -l` + +if [ "$totalPySuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" +fi + +totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` +if [ "$totalPyFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" +# exit $totalPyFailed +fi + +exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed + $totalExampleFailed)) -- GitLab