diff --git a/tests/perftest-scripts/coverage_test.sh b/tests/perftest-scripts/coverage_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..152e20756b848a342f0994323274ff4130d0fe92 --- /dev/null +++ b/tests/perftest-scripts/coverage_test.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.cover +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + cd $TDENGINE_DIR/debug + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git reset --hard + git pull + + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + cmake -DCOVER=true -DRANDOM_FILE_FAIL=true .. > /dev/null + make > /dev/null + fi + + make install > /dev/null +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + general_case=`echo $line | grep -w general` + + if [ -n "$general_case" ]; then + case=`echo $line |grep general| awk '{print $NF}'` + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + + if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess coverage test case(s) succeed! ### ${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo -e "${RED} ### Total $totalFailed coverage test case(s) failed! ### ${NC}\n" | tee -a $TDENGINE_COVERAGE_REPORT +# exit $totalPyFailed + fi + + cd $TDENGINE_DIR/tests + rm -rf ../sim + ./test-all.sh full python | tee -a $TDENGINE_COVERAGE_REPORT + + # Test Connector + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package + mvn test | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + $TDENGINE_DIR/debug/build/bin/utilTest > /dev/null && echo "utilTest pass!" || echo "utilTest failed!" + $TDENGINE_DIR/debug/build/bin/queryTest > /dev/null && echo "queryTest pass!" || echo "queryTest failed!" + $TDENGINE_DIR/debug/build/bin/cliTest > /dev/null && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + coveralls-lcov coverage.info | tee -a $TDENGINE_COVERAGE_REPORT +} + +function sendReport { + echo "send report" + receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/home/shuduo/work/taosdata + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +rm /tmp/core-* + +stopTaosd +buildTDengine + +runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +lcovFunc +sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/full_test.sh b/tests/perftest-scripts/full_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..1738f69f35207161121c5cbd00816ae37a46745e --- /dev/null +++ b/tests/perftest-scripts/full_test.sh @@ -0,0 +1,191 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.orig +TDENGINE_FULLTEST_REPORT=$TDENGINE_DIR/tests/full-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + + need_rebuild=false + + if [ ! -d $TDENGINE_DIR ]; then + echo "No TDengine source code found!" + git clone https://github.com/taosdata/TDengine $TDENGINE_DIR + need_rebuild=true + fi + + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git pull + need_rebuild=true + fi + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + [ -f $TDENGINE_DIR/debug/build/bin/taosd ] || need_rebuild=true + + if $need_rebuild ; then + echo "rebuild.." + + LOCAL_COMMIT=`git rev-parse --short @` + rm -rf * + cmake .. > /dev/null + make > /dev/null + fi + + make install > /dev/null +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + general_case=`echo $line | grep -w general` + + if [ -n "$general_case" ]; then + case=`echo $line | awk '{print $NF}'` + + start_time=`date +%s` + ./test.sh -f $case > /dev/null 2>&1 && ret=0 || ret = 1 + end_time=`date +%s` + + if [[ ret -eq 0 ]]; then + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_FULLTEST_REPORT + else + casename=`echo $case|sed 's/\//\-/g'` + find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} + + echo -e "${RED}$case failed and log saved${NC}" | tee -a $TDENGINE_FULLTEST_REPORT + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_FULLTEST_REPORT + fi + fi + done < $1 +} + +function runPyCaseOneByOne { + while read -r line; do + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + case=`echo $line|awk '{print $NF}'` + start_time=`date +%s` + $line > /dev/null 2>&1 && ret=0 || ret=1 + end_time=`date +%s` + + if [[ ret -eq 0 ]]; then + echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log + else + casename=`echo $case|sed 's/\//\-/g'` + find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} + + echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + fi + done < $1 +} + +function runTest { + echo "Run Test" + cd $TDENGINE_DIR/tests/script + + [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim + [ -f $TDENGINE_FULLTEST_REPORT ] && rm $TDENGINE_FULLTEST_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + totalSuccess=`grep 'success' $TDENGINE_FULLTEST_REPORT | wc -l` + + if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess SIM case(s) succeed! ### ${NC}" \ + | tee -a $TDENGINE_FULLTEST_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_FULLTEST_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo -e "${RED} ### Total $totalFailed SIM case(s) failed! ### ${NC}\n" \ + | tee -a $TDENGINE_FULLTEST_REPORT + fi + + cd $TDENGINE_DIR/tests/pytest + [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim + [ -f pytest-out.log ] && rm -f pytest-out.log + runPyCaseOneByOne fulltest.sh + + totalPySuccess=`grep 'success' pytest-out.log | wc -l` + totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` + + cat pytest-out.log >> $TDENGINE_FULLTEST_REPORT + if [ "$totalPySuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \ + | tee -a $TDENGINE_FULLTEST_REPORT + fi + + if [ "$totalPyFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \ + | tee -a $TDENGINE_FULLTEST_REPORT + fi +} + +function sendReport { + echo "Send Report" + receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR/tests + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_FULLTEST_REPORT + BODY_CONTENT=`cat $TDENGINE_FULLTEST_REPORT` + + cd $TDENGINE_DIR + tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz + + echo -e "to: ${receiver}\nsubject: Full test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_FULLTEST_REPORT fulltest-report-$today.log) | \ + (cat - && uuencode $TDENGINE_DIR/fulltest-$today.tar.gz fulltest-$today.tar.gz) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +WORK_DIR=/home/shuduo/work/taosdata + +date >> $WORK_DIR/cron.log +echo "Run Full Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine +runTest +sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Full Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/tdinternal_coverage_test.sh b/tests/perftest-scripts/tdinternal_coverage_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..91d204baf9822804f9106712c272d728e5747c7b --- /dev/null +++ b/tests/perftest-scripts/tdinternal_coverage_test.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal.cover +TDINTERNAL_COVERAGE_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDinternal { + echo "check if TDinternal need build" + cd $TDINTERNAL_DIR + NEED_COMPILE=0 +# git remote update + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "TDinternal repo is up-to-date" + else + echo "repo need to pull" +# git pull + +# NEED_COMPILE=1 + fi + + lcov -d . --zerocounters +# git submodule update --init --recursive + cd $TDINTERNAL_DIR/community + TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + TDENGINE_LOCAL_COMMIT=`git rev-parse --short @` + if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then + echo "community repo is up-to-date" + else + echo "repo need to pull" +# git checkout develop +# git pull +# NEED_COMPILE=1 + fi + + cd $TDINTERNAL_DIR/debug + + if [[ $NEED_COMPILE -eq 1 ]]; then + LOCAL_COMMIT=`git rev-parse --short @` + rm -rf * + cmake .. > /dev/null + make > /dev/null + fi + + make install > /dev/null +} + +function runUniqueCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | awk '{print $NF}'` + start_time=`date +%s` + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_COVERAGE_REPORT + fi + done < $1 +} + +function runTest { + echo "Run Test" + cd $TDINTERNAL_DIR/community/tests/script + [ -d ../../sim ] && rm -rf ../../sim + + [ -f $TDINTERNAL_COVERAGE_REPORT ] && rm $TDINTERNAL_COVERAGE_REPORT + + runUniqueCaseOneByOne jenkins/basic.txt + + totalSuccess=`grep 'success' $TDINTERNAL_COVERAGE_REPORT | wc -l` + + if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDINTERNAL_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_COVERAGE_REPORT +# exit $totalPyFailed + fi + + # Test Python test case + cd $TDINTERNAL_DIR/community/tests + /usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_COVERAGE_REPORT + + # Test Connector + stopTaosd + $TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null & + sleep 10 + + cd $TDINTERNAL_DIR/community/src/connector/jdbc + mvn clean package + mvn test | tee -a $TDINTERNAL_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDINTERNAL_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDINTERNAL_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir\nwalDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDINTERNAL_COVERAGE_REPORT + fi + + stopTaosd +} + +function sendReport { + echo "Send Report" + receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDINTERNAL_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT + + BODY_CONTENT=`cat $TDINTERNAL_COVERAGE_REPORT` + echo -e "to: ${receiver}\nsubject: TDinternal coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode tdinternal-coverage-report-$today.tar.gz tdinternal-coverage-report-$today.tar.gz) | \ + (cat - && uuencode $TDINTERNAL_COVERAGE_REPORT tdinternal-coverage-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDINTERNAL_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtmml_branch_coverage=1 --no-external -b $TDINTERNAL_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDINTERNAL_COVERAGE_REPORT + + genhtml -o html coverage.info + + tar czf tdinternal-coverage-report-$today.tar.gz html coverage.info $TDINTERNAL_COVERAGE_REPORT + # push result to coveralls.io +# coveralls-lcov coverage.info | tee -a tdinternal-coverage-report-$today.log +} + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +WORK_DIR=/home/shuduo/work/taosdata + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test for TDinternal" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDinternal +runTest +lcovFunc +sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of TDinternal Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/perftest-scripts/tdinternal_test.sh b/tests/perftest-scripts/tdinternal_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..4de0549be06f2da7b17924a6dd87c18e007ebcc0 --- /dev/null +++ b/tests/perftest-scripts/tdinternal_test.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal +TDINTERNAL_TEST_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDinternal { + echo "check if TDinternal need build" + cd $TDINTERNAL_DIR + NEED_COMPILE=0 +# git remote update + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "TDinternal repo is up-to-date" + else + echo "repo need to pull" +# git pull + +# NEED_COMPILE=1 + fi + +# git submodule update --init --recursive + cd $TDINTERNAL_DIR/community + TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + TDENGINE_LOCAL_COMMIT=`git rev-parse --short @` + if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then + echo "community repo is up-to-date" + else + echo "repo need to pull" +# git checkout develop +# git pull +# NEED_COMPILE=1 + fi + + cd $TDINTERNAL_DIR/debug + + if [[ $NEED_COMPILE -eq 1 ]]; then + LOCAL_COMMIT=`git rev-parse --short @` + rm -rf * + cmake .. > /dev/null + make > /dev/null + fi + + make install > /dev/null +} + +function runUniqueCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | awk '{print $NF}'` + start_time=`date +%s` + ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_TEST_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_TEST_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_TEST_REPORT + fi + done < $1 +} + +function runPyCaseOneByOne { + while read -r line; do + if [[ $line =~ ^python.* ]]; then + if [[ $line != *sleep* ]]; then + case=`echo $line|awk '{print $NF}'` + start_time=`date +%s` + $line > /dev/null 2>&1 && ret=0 || ret=1 + end_time=`date +%s` + + if [[ ret -eq 0 ]]; then + echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log + else + casename=`echo $case|sed 's/\//\-/g'` + find $TDINTERNAL_DIR/community/sim -name "*log" -exec tar czf $TDINTERNAL_DIR/fulltest-$today-$casename.log.tar.gz {} + + echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log + fi + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log + else + $line > /dev/null 2>&1 + fi + fi + done < $1 +} + +function runTest { + echo "Run Test" + cd $TDINTERNAL_DIR/community/tests/script + [ -d $TDINTERNAL_DIR/sim ] && rm -rf $TDINTERNAL_DIR/sim + + [ -f $TDINTERNAL_TEST_REPORT ] && rm $TDINTERNAL_TEST_REPORT + + runUniqueCaseOneByOne jenkins/basic.txt + + totalSuccess=`grep 'success' $TDINTERNAL_TEST_REPORT | wc -l` + + if [ "$totalSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_TEST_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDINTERNAL_TEST_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_TEST_REPORT +# exit $totalPyFailed + fi + + cd $TDINTERNAL_DIR/community/tests/pytest + [ -d $TDINTERNAL_DIR/community/sim ] && rm -rf $TDINTERNAL_DIR/community/sim + [ -f pytest-out.log ] && rm -f pytest-out.log + + /usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_TEST_REPORT + runPyCaseOneByOne fulltest.sh + + totalPySuccess=`grep 'success' pytest-out.log | wc -l` + totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l` + + cat pytest-out.log >> $TDINTERNAL_TEST_REPORT + if [ "$totalPySuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \ + | tee -a $TDINTERNAL_TEST_REPORT + fi + + if [ "$totalPyFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \ + | tee -a $TDINTERNAL_TEST_REPORT + fi +} + +function sendReport { + echo "Send Report" + receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDINTERNAL_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_TEST_REPORT + BODY_CONTENT=`cat $TDINTERNAL_TEST_REPORT` + + cd $TDINTERNAL_DIR + tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz + + echo -e "to: ${receiver}\nsubject: TDinternal test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDINTERNAL_TEST_REPORT tdinternal-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -KILL -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +WORK_DIR=/home/shuduo/work/taosdata + +date >> $WORK_DIR/cron.log +echo "Run Test for TDinternal" | tee -a $WORK_DIR/cron.log + +buildTDinternal +runTest +sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of TDinternal Test" | tee -a $WORK_DIR/cron.log