Merge pull request #2771 from taosdata/feature/sangshuduo/perftest-script-improve

add coverage test scripts.

tests/perftest-scripts/coverage_test.sh

+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.cover
+TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+	echo "check if TDengine need build"
+	cd $TDENGINE_DIR
+  git remote prune origin > /dev/null
+	git remote update > /dev/null
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+
+	# reset counter
+	lcov -d . --zerocounters
+
+	cd $TDENGINE_DIR/debug
+
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+    git reset --hard
+		git pull
+
+		LOCAL_COMMIT=`git rev-parse --short @`
+
+		rm -rf *
+		cmake -DCOVER=true -DRANDOM_FILE_FAIL=true .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runGeneralCaseOneByOne {
+	while read -r line; do
+		if [[ $line =~ ^./test.sh* ]]; then
+			general_case=`echo $line | grep -w general`
+
+			if [ -n "$general_case" ]; then
+				case=`echo $line |grep general| awk '{print $NF}'`
+				./test.sh -f $case > /dev/null 2>&1 && \
+				echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \
+				echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+			fi
+		fi
+	done < $1
+}
+
+function runTest {
+	echo "run Test"
+
+	cd $TDENGINE_DIR/tests/script
+
+	[ -d ../../sim ] && rm -rf ../../sim
+	[ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
+
+	runGeneralCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess coverage test case(s) succeed! ### ${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed coverage test case(s) failed! ### ${NC}\n" | tee -a $TDENGINE_COVERAGE_REPORT
+#  exit $totalPyFailed
+	fi
+
+	cd $TDENGINE_DIR/tests
+	rm -rf ../sim
+	./test-all.sh full python | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test Connector
+	stopTaosd
+	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+
+	cd $TDENGINE_DIR/src/connector/jdbc
+	mvn clean package
+	mvn test | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test C Demo
+	stopTaosd
+	$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+	yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# Test waltest
+	dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+	walDir=`find $dataDir -name "wal"|head -n1`
+	echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
+	if [ -n "$walDir" ]; then
+		yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDENGINE_COVERAGE_REPORT
+	fi
+
+  # run Unit Test
+  echo "Run Unit Test: utilTest, queryTest and cliTest"
+  $TDENGINE_DIR/debug/build/bin/utilTest > /dev/null && echo "utilTest pass!" || echo "utilTest failed!"
+  $TDENGINE_DIR/debug/build/bin/queryTest > /dev/null && echo "queryTest pass!" || echo "queryTest failed!"
+  $TDENGINE_DIR/debug/build/bin/cliTest > /dev/null && echo "cliTest pass!" || echo "cliTest failed!"
+
+	stopTaosd
+}
+
+function lcovFunc {
+	echo "collect data by lcov"
+	cd $TDENGINE_DIR
+
+	# collect data
+	lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+	# remove exclude paths
+	lcov --remove coverage.info \
+    '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
+    --rc lcov_branch_coverage=1 -o coverage.info
+
+	# generate result
+	lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+
+	# push result to coveralls.io
+	coveralls-lcov coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
+}
+
+function sendReport {
+	echo "send report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDENGINE_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
+	BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
+	echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+    pkill -TERM -x taosd
+    sleep 1
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+function runTestRandomFail {
+  exec_random_fail_sh=$1
+  default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
+	[ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
+
+  dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
+  default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
+	[ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
+
+  runTest NoRandomFail
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
+
+rm /tmp/core-*
+
+stopTaosd
+buildTDengine
+
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh 
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh 
+runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh 
+
+lcovFunc
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log

tests/perftest-scripts/full_test.sh

+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.orig
+TDENGINE_FULLTEST_REPORT=$TDENGINE_DIR/tests/full-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDengine {
+  echo "check if TDengine need build"
+
+  need_rebuild=false
+
+  if [ ! -d $TDENGINE_DIR ]; then
+    echo "No TDengine source code found!"
+    git clone https://github.com/taosdata/TDengine $TDENGINE_DIR
+    need_rebuild=true
+  fi
+
+  cd $TDENGINE_DIR
+  git remote prune origin > /dev/null
+  git remote update > /dev/null
+  REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+  LOCAL_COMMIT=`git rev-parse --short @`
+  echo " LOCAL: $LOCAL_COMMIT"
+  echo "REMOTE: $REMOTE_COMMIT"
+
+  if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+    echo "repo up-to-date"
+  else
+    echo "repo need to pull"
+    git pull
+    need_rebuild=true
+  fi
+
+  [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
+  cd $TDENGINE_DIR/debug
+  [ -f $TDENGINE_DIR/debug/build/bin/taosd ] || need_rebuild=true
+
+  if $need_rebuild ; then
+    echo "rebuild.."
+
+    LOCAL_COMMIT=`git rev-parse --short @`
+    rm -rf *
+    cmake .. > /dev/null
+    make > /dev/null
+  fi
+
+  make install > /dev/null
+}
+
+function runGeneralCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      general_case=`echo $line | grep -w general`
+
+      if [ -n "$general_case" ]; then
+        case=`echo $line | awk '{print $NF}'`
+
+        start_time=`date +%s`
+        ./test.sh -f $case > /dev/null 2>&1 && ret=0 || ret = 1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_FULLTEST_REPORT
+      fi
+    fi
+  done < $1
+}
+
+function runPyCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^python.* ]]; then
+      if [[ $line != *sleep* ]]; then
+        case=`echo $line|awk '{print $NF}'`
+        start_time=`date +%s`
+        $line > /dev/null 2>&1 && ret=0 || ret=1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+      else
+        $line > /dev/null 2>&1
+      fi
+    fi
+  done < $1
+}
+
+function runTest {
+  echo "Run Test"
+  cd $TDENGINE_DIR/tests/script
+
+  [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
+  [ -f $TDENGINE_FULLTEST_REPORT ] && rm $TDENGINE_FULLTEST_REPORT
+
+  runGeneralCaseOneByOne jenkins/basic.txt
+
+  totalSuccess=`grep 'success' $TDENGINE_FULLTEST_REPORT | wc -l`
+
+  if [ "$totalSuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalSuccess SIM case(s) succeed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  totalFailed=`grep 'failed\|fault' $TDENGINE_FULLTEST_REPORT | wc -l`
+  if [ "$totalFailed" -ne "0" ]; then
+    echo -e "${RED} ### Total $totalFailed SIM case(s) failed! ### ${NC}\n" \
+    | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  cd $TDENGINE_DIR/tests/pytest
+  [ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
+  [ -f pytest-out.log ] && rm -f pytest-out.log
+  runPyCaseOneByOne fulltest.sh
+
+  totalPySuccess=`grep 'success' pytest-out.log | wc -l`
+  totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
+
+  cat pytest-out.log >> $TDENGINE_FULLTEST_REPORT
+  if [ "$totalPySuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+
+  if [ "$totalPyFailed" -ne "0" ]; then
+    echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
+      | tee -a $TDENGINE_FULLTEST_REPORT
+  fi
+}
+
+function sendReport {
+  echo "Send Report"
+  receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+  mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+  cd $TDENGINE_DIR/tests
+
+  sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_FULLTEST_REPORT
+  BODY_CONTENT=`cat $TDENGINE_FULLTEST_REPORT`
+
+  cd $TDENGINE_DIR
+  tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
+
+  echo -e "to: ${receiver}\nsubject: Full test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+  (cat - && uuencode $TDENGINE_FULLTEST_REPORT fulltest-report-$today.log) | \
+  (cat - && uuencode $TDENGINE_DIR/fulltest-$today.tar.gz fulltest-$today.tar.gz) | \
+  ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+  echo "Stop taosd"
+  systemctl stop taosd
+  PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+  while [ -n "$PID" ]
+  do
+    pkill -TERM -x taosd
+    sleep 1
+    PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+  done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Full Test" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDengine
+runTest
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of Full Test" | tee -a $WORK_DIR/cron.log

tests/perftest-scripts/tdinternal_coverage_test.sh

+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal.cover
+TDINTERNAL_COVERAGE_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-coverage-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDinternal {
+	echo "check if TDinternal need build"
+	cd $TDINTERNAL_DIR
+	NEED_COMPILE=0
+#	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "TDinternal repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git pull
+
+#		NEED_COMPILE=1
+	fi
+
+  lcov -d . --zerocounters
+#	git submodule update --init --recursive
+	cd $TDINTERNAL_DIR/community
+	TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
+	if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
+		echo "community repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git checkout develop
+#		git pull
+#		NEED_COMPILE=1
+	fi
+
+	cd $TDINTERNAL_DIR/debug
+
+	if [[ $NEED_COMPILE -eq 1 ]]; then
+		LOCAL_COMMIT=`git rev-parse --short @`
+		rm -rf *
+		cmake .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runUniqueCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      case=`echo $line | awk '{print $NF}'`
+      start_time=`date +%s`
+      ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT || \
+        echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
+      end_time=`date +%s`
+      echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_COVERAGE_REPORT
+    fi
+  done < $1
+}
+
+function runTest {
+	echo "Run Test"
+	cd $TDINTERNAL_DIR/community/tests/script
+	[ -d ../../sim ] && rm -rf ../../sim
+
+	[ -f $TDINTERNAL_COVERAGE_REPORT ] && rm $TDINTERNAL_COVERAGE_REPORT
+
+	runUniqueCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDINTERNAL_COVERAGE_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDINTERNAL_COVERAGE_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_COVERAGE_REPORT
+#  exit $totalPyFailed
+	fi
+
+	# Test Python test case
+	cd $TDINTERNAL_DIR/community/tests
+	/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test Connector
+	stopTaosd
+	$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+
+	cd $TDINTERNAL_DIR/community/src/connector/jdbc
+	mvn clean package
+	mvn test | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test C Demo
+	stopTaosd
+	$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
+	sleep 10
+	yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	# Test waltest
+	dataDir=`grep dataDir $TDINTERNAL_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
+	walDir=`find $dataDir -name "wal"|head -n1`
+	echo "dataDir: $dataDir\nwalDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
+	if [ -n "$walDir" ]; then
+		yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDINTERNAL_COVERAGE_REPORT
+	fi
+
+	stopTaosd
+}
+
+function sendReport {
+	echo "Send Report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
+
+	BODY_CONTENT=`cat $TDINTERNAL_COVERAGE_REPORT`
+	echo -e "to: ${receiver}\nsubject: TDinternal coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode tdinternal-coverage-report-$today.tar.gz tdinternal-coverage-report-$today.tar.gz) | \
+	(cat - && uuencode $TDINTERNAL_COVERAGE_REPORT tdinternal-coverage-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function lcovFunc {
+	echo "collect data by lcov"
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
+	# collect data
+	lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtmml_branch_coverage=1 --no-external -b $TDINTERNAL_DIR -o coverage.info
+
+	# remove exclude paths
+	lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
+    --rc lcov_branch_coverage=1 -o coverage.info
+
+	# generate result
+	lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDINTERNAL_COVERAGE_REPORT
+
+	genhtml -o html coverage.info
+
+	tar czf tdinternal-coverage-report-$today.tar.gz html coverage.info $TDINTERNAL_COVERAGE_REPORT
+	# push result to coveralls.io
+#	coveralls-lcov coverage.info | tee -a tdinternal-coverage-report-$today.log
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+        	pkill -TERM -x taosd
+        	sleep 1
+  		PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Coverage Test for TDinternal" | tee -a $WORK_DIR/cron.log
+
+stopTaosd
+buildTDinternal
+runTest
+lcovFunc
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of TDinternal Coverage Test" | tee -a $WORK_DIR/cron.log

tests/perftest-scripts/tdinternal_test.sh

+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal
+TDINTERNAL_TEST_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-report-$today.log
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function buildTDinternal {
+	echo "check if TDinternal need build"
+	cd $TDINTERNAL_DIR
+	NEED_COMPILE=0
+#	git remote update
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "TDinternal repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git pull
+
+#		NEED_COMPILE=1
+	fi
+
+#	git submodule update --init --recursive
+	cd $TDINTERNAL_DIR/community
+	TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
+	if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
+		echo "community repo is up-to-date"
+	else
+		echo "repo need to pull"
+#		git checkout develop
+#		git pull
+#		NEED_COMPILE=1
+	fi
+
+	cd $TDINTERNAL_DIR/debug
+
+	if [[ $NEED_COMPILE -eq 1 ]]; then
+		LOCAL_COMMIT=`git rev-parse --short @`
+		rm -rf *
+		cmake .. > /dev/null
+		make > /dev/null
+	fi
+
+	make install > /dev/null
+}
+
+function runUniqueCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^./test.sh* ]]; then
+      case=`echo $line | awk '{print $NF}'`
+      start_time=`date +%s`
+      ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_TEST_REPORT || \
+        echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_TEST_REPORT
+      end_time=`date +%s`
+      echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_TEST_REPORT
+    fi
+  done < $1
+}
+
+function runPyCaseOneByOne {
+  while read -r line; do
+    if [[ $line =~ ^python.* ]]; then
+      if [[ $line != *sleep* ]]; then
+        case=`echo $line|awk '{print $NF}'`
+        start_time=`date +%s`
+        $line > /dev/null 2>&1 && ret=0 || ret=1
+        end_time=`date +%s`
+
+        if [[ ret -eq 0 ]]; then
+          echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
+        else
+          casename=`echo $case|sed 's/\//\-/g'`
+          find $TDINTERNAL_DIR/community/sim -name "*log" -exec tar czf $TDINTERNAL_DIR/fulltest-$today-$casename.log.tar.gz {} +
+          echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
+        fi
+        echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
+      else
+        $line > /dev/null 2>&1
+      fi
+    fi
+  done < $1
+}
+
+function runTest {
+	echo "Run Test"
+	cd $TDINTERNAL_DIR/community/tests/script
+	[ -d $TDINTERNAL_DIR/sim ] && rm -rf $TDINTERNAL_DIR/sim
+
+	[ -f $TDINTERNAL_TEST_REPORT ] && rm $TDINTERNAL_TEST_REPORT
+
+	runUniqueCaseOneByOne jenkins/basic.txt
+
+	totalSuccess=`grep 'success' $TDINTERNAL_TEST_REPORT | wc -l`
+
+	if [ "$totalSuccess" -gt "0" ]; then
+		echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_TEST_REPORT
+	fi
+
+	totalFailed=`grep 'failed\|fault' $TDINTERNAL_TEST_REPORT | wc -l`
+	if [ "$totalFailed" -ne "0" ]; then
+		echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_TEST_REPORT
+#  exit $totalPyFailed
+	fi
+
+	cd $TDINTERNAL_DIR/community/tests/pytest
+  [ -d $TDINTERNAL_DIR/community/sim ] && rm -rf $TDINTERNAL_DIR/community/sim
+  [ -f pytest-out.log ] && rm -f pytest-out.log
+
+	/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_TEST_REPORT
+  runPyCaseOneByOne fulltest.sh
+
+  totalPySuccess=`grep 'success' pytest-out.log | wc -l`
+  totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
+
+  cat pytest-out.log >> $TDINTERNAL_TEST_REPORT
+  if [ "$totalPySuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
+      | tee -a $TDINTERNAL_TEST_REPORT
+  fi
+
+  if [ "$totalPyFailed" -ne "0" ]; then
+    echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
+      | tee -a $TDINTERNAL_TEST_REPORT
+  fi
+}
+
+function sendReport {
+	echo "Send Report"
+	receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDINTERNAL_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_TEST_REPORT
+	BODY_CONTENT=`cat $TDINTERNAL_TEST_REPORT`
+
+  cd $TDINTERNAL_DIR
+  tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
+
+	echo -e "to: ${receiver}\nsubject: TDinternal test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $TDINTERNAL_TEST_REPORT tdinternal-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+function stopTaosd {
+	echo "Stop taosd"
+        systemctl stop taosd
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+        	pkill -KILL -x taosd
+        	sleep 1
+  		PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+WORK_DIR=/home/shuduo/work/taosdata
+
+date >> $WORK_DIR/cron.log
+echo "Run Test for TDinternal" | tee -a $WORK_DIR/cron.log
+
+buildTDinternal
+runTest
+sendReport
+stopTaosd
+
+date >> $WORK_DIR/cron.log
+echo "End of TDinternal Test" | tee -a $WORK_DIR/cron.log