diff --git a/tests/perftest-scripts/perftest-csv2png.gnuplot b/tests/perftest-scripts/perftest-csv2png.gnuplot index 9c34ebe403fe7317c963ab8693069025acc79a64..eabab268cab099e0ad06b8999961ddc5cf7b85d8 100644 --- a/tests/perftest-scripts/perftest-csv2png.gnuplot +++ b/tests/perftest-scripts/perftest-csv2png.gnuplot @@ -2,7 +2,7 @@ reset set terminal png -set title "Performance Test Report" font ",20" +set title filename font ",20" set ylabel "Time in Seconds" diff --git a/tests/perftest-scripts/perftest-daily.sh b/tests/perftest-scripts/perftest-daily.sh index db9de11381062a8593ccf585a853f91dedbdd97d..b44387047ca12d78bd669e9cb098a380b557733c 100755 --- a/tests/perftest-scripts/perftest-daily.sh +++ b/tests/perftest-scripts/perftest-daily.sh @@ -16,9 +16,26 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # -function set-Wal { +function setMaxConnections { + echo "/etc/taos/taos.cfg maxConnection will be set to $1" + + hasText=`grep "maxConnections" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "maxConnections $1" >> /etc/taos/taos.cfg + else + sed -i 's/^maxConnections.*$/maxConnections '"$1"'/g' /etc/taos/taos.cfg + fi +} + +function setWal { echo "/etc/taos/taos.cfg walLevel will be set to $1" - sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg + + hasText=`grep "walLevel" /etc/taos/taos.cfg` + if [[ -z "$hasText" ]]; then + echo "walLevel $1" >> /etc/taos/taos.cfg + else + sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg + fi } function collectSysInfo { @@ -70,15 +87,25 @@ function sendReport { mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" echo -e "to: ${receiver}\nsubject: Perf test report ${today}, commit ID: ${LOCAL_COMMIT}\n" | \ - (cat - && uuencode perftest-1d-$today.log perftest-1d-$today.log)| \ - (cat - && uuencode perftest-1d-report.csv perftest-1d-report-$today.csv) | \ - (cat - && uuencode perftest-1d-report.png perftest-1d-report-$today.png) | \ - (cat - && uuencode perftest-13d-$today.log perftest-13d-$today.log)| \ - (cat - && uuencode perftest-13d-report.csv perftest-13d-report-$today.csv) | \ - (cat - && uuencode perftest-13d-report.png perftest-13d-report-$today.png) | \ - (cat - && uuencode taosdemo-$today.log taosdemo-$today.log) | \ - (cat - && uuencode taosdemo-report.csv taosdemo-report-$today.csv) | \ - (cat - && uuencode taosdemo-report.png taosdemo-report-$today.png) | \ + (cat - && uuencode perftest-1d-wal1-$today.log perftest-1d-wal1-$today.log)| \ + (cat - && uuencode perftest-1d-wal1-report.csv perftest-1d-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-1d-wal1-report.png perftest-1d-wal1-report-$today.png) | \ + (cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \ + (cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \ + (cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \ + (cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \ + (cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \ + (cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \ + (cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \ + (cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \ + (cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \ + (cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \ + (cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \ + (cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \ + (cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \ + (cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \ (cat - && uuencode sysinfo.log sysinfo.txt) | \ (cat - && uuencode taos.cfg taos-cfg-$today.txt) | \ ssmtp "${receiver}" @@ -91,17 +118,34 @@ echo -e "cron-ran-at-${today}" >> cron.log echoInfo "Build TDengine" buildTDengine -set-Wal "2" +############################ +setMaxConnections 100 + +############################ +setWal "2" + +cd /root +./perftest-tsdb-compare-1d.sh "wal2" + +cd /root +./perftest-tsdb-compare-13d.sh "wal2" + +cd /root +./perftest-taosdemo.sh "wal2" + +############################# +setWal "1" cd /root -./perftest-tsdb-compare-1d.sh +./perftest-tsdb-compare-1d.sh "wal1" cd /root -./perftest-tsdb-compare-13d.sh +./perftest-tsdb-compare-13d.sh "wal1" cd /root -./perftest-taosdemo.sh +./perftest-taosdemo.sh "wal1" +############################# collectSysInfo echoInfo "Send Report" diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index 511ec22fec47a496b5e79b35c8d0b42d61d0a336..0dd9a0572a5bc296eed10f7179a5b5c64981b8e3 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -1,20 +1,20 @@ #!/bin/bash -# Coloured Echoes # -function red_echo { echo -e "\033[31m$@\033[0m"; } # -function green_echo { echo -e "\033[32m$@\033[0m"; } # -function yellow_echo { echo -e "\033[33m$@\033[0m"; } # -function white_echo { echo -e "\033[1;37m$@\033[0m"; } # -# Coloured Printfs # -function red_printf { printf "\033[31m$@\033[0m"; } # -function green_printf { printf "\033[32m$@\033[0m"; } # -function yellow_printf { printf "\033[33m$@\033[0m"; } # -function white_printf { printf "\033[1;37m$@\033[0m"; } # -# Debugging Outputs # -function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # -function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # -function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # -function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # +# Coloured Echoes +function red_echo { echo -e "\033[31m$@\033[0m"; } +function green_echo { echo -e "\033[32m$@\033[0m"; } +function yellow_echo { echo -e "\033[33m$@\033[0m"; } +function white_echo { echo -e "\033[1;37m$@\033[0m"; } +# Coloured Printfs +function red_printf { printf "\033[31m$@\033[0m"; } +function green_printf { printf "\033[32m$@\033[0m"; } +function yellow_printf { printf "\033[33m$@\033[0m"; } +function white_printf { printf "\033[1;37m$@\033[0m"; } +# Debugging Outputs +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } function restartTaosd { systemctl stop taosd @@ -32,39 +32,57 @@ function runCreateTableOnly { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$today.log" - demoTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log" + demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` +} + +function runDeleteTableOnly { + echoInfo "Restart Taosd" + restartTaosd + + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log" + demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` } function runCreateTableThenInsert { echoInfo "Restart Taosd" restartTaosd - /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$today.log" + /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log" demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'` - demoRPS=`grep "records\/second" taosdemo-$today.log | tail -n1 | awk '{print $13}'` + demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'` } function generateTaosdemoPlot { - echo "${today}, demoTableOnly: ${demoTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log - echo "${today}, ${demoTableOnly}, ${demoTableAndInsert}, ${demoRPS}" >> taosdemo-report.csv + echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log + echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv + echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv - csvLines=`cat taosdemo-report.csv | wc -l` + csvLines=`cat taosdemo-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' taosdemo-report.csv + sed -i '1d' taosdemo-$1-report.csv fi - gnuplot -p taosdemo-csv2png.gnuplot + csvLines=`cat taosdemo-rps-$1-report.csv | wc -l` + + if [ "$csvLines" -gt "10" ]; then + sed -i '1d' taosdemo-rps-$1-report.csv + fi + + gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot + gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "Test Create Table Only " -runCreateTableOnly +runCreateTableOnly $1 +echoInfo "Test Create Table then Insert data" +runDeleteTableOnly $1 echoInfo "Test Create Table then Insert data" -runCreateTableThenInsert +runCreateTableThenInsert $1 echoInfo "Generate plot for taosdemo" -generateTaosdemoPlot +generateTaosdemoPlot $1 echoInfo "End of TaosDemo Test" diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 4b3ed6818c433c96fdde2513a3e38bb3110dfdbf..8b326d2d4109d53b455fa825cc3204180fbaeb9e 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -33,26 +33,26 @@ function runPerfTest13d { restartTaosd cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-13d-csv.sh 2>&1 | tee /root/perftest-13d-$today.log + ./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log } function generatePerfPlot13d { cd /root - csvLines=`cat perftest-13d-report.csv | wc -l` + csvLines=`cat perftest-13d-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '1d' perftest-13d-report.csv + sed -i '1d' perftest-13d-$1-report.csv fi - gnuplot -e "filename='perftest-13d-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "run Performance Test with 13 days data" -runPerfTest13d +runPerfTest13d $1 echoInfo "Generate plot of 13 days data" -generatePerfPlot13d +generatePerfPlot13d $1 echoInfo "End of TSDB-Compare 13-days-data Test" diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index ebe34cde72ce3d6af4ff03ac242c73efecb53475..0931480e2150370ad55d673ebfbca06cefb6dd73 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -33,26 +33,26 @@ function runPerfTest1d { restartTaosd cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare - ./runreal-1d-csv.sh 2>&1 | tee /root/perftest-1d-$today.log + ./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log } function generatePerfPlot1d { cd /root - csvLines=`cat perftest-1d-report.csv | wc -l` + csvLines=`cat perftest-1d-$1-report.csv | wc -l` if [ "$csvLines" -gt "10" ]; then - sed -i '2d' perftest-1d-report.csv + sed -i '2d' perftest-1d-$1-report.csv fi - gnuplot -e "filename='perftest-1d-report'" -p perftest-csv2png.gnuplot + gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot } today=`date +"%Y%m%d"` cd /root echoInfo "run Performance Test with 1 day data" -runPerfTest1d +runPerfTest1d $1 echoInfo "Generate plot of 1 day data" -generatePerfPlot1d +generatePerfPlot1d $1 echoInfo "End of TSDB-Compare 1-day-data Test" diff --git a/tests/perftest-scripts/runreal-13d-csv.sh b/tests/perftest-scripts/runreal-13d-csv.sh index ff7ce41d4bd2f85d6911cf5f9d194a487fc4df07..48ef01e6a08b37e9bceb2bb599f7ded6bee2dcd1 100755 --- a/tests/perftest-scripts/runreal-13d-csv.sh +++ b/tests/perftest-scripts/runreal-13d-csv.sh @@ -143,7 +143,7 @@ echo "------------------------------------------------------" echo today=`date +"%Y%m%d"` -echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-report.csv +echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-$1-report.csv #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" diff --git a/tests/perftest-scripts/runreal-1d-csv.sh b/tests/perftest-scripts/runreal-1d-csv.sh index 5cd113aadf1c05bb67a061b969b842872a163bca..20c8b4138c5625e816cbe08a368d9f5fc7fc67dc 100755 --- a/tests/perftest-scripts/runreal-1d-csv.sh +++ b/tests/perftest-scripts/runreal-1d-csv.sh @@ -143,7 +143,7 @@ echo "------------------------------------------------------" echo today=`date +"%Y%m%d"` -echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-report.csv +echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-$1-report.csv #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" diff --git a/tests/perftest-scripts/taosdemo-csv2png.gnuplot b/tests/perftest-scripts/taosdemo-csv2png.gnuplot index 9fcd4bb3d99cd64d97b3e2aa2257e384e19fc3ab..7cdcb84fce7cdf52dd223e1af6f6d68a46b13893 100644 --- a/tests/perftest-scripts/taosdemo-csv2png.gnuplot +++ b/tests/perftest-scripts/taosdemo-csv2png.gnuplot @@ -2,7 +2,7 @@ reset set terminal png -set title "TaosDemo Performance Report" font ",20" +set title filename font ",20" set ylabel "Time in Seconds" @@ -14,13 +14,13 @@ set xlabel "Date" set style data linespoints set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10' -set output 'taosdemo-report.png' +set output filename . '.png' set datafile separator ',' set key reverse Left outside set grid -plot 'taosdemo-report.csv' using 1:2 title "Create 10,000 Table", \ - "" using 1:3 title "Create 10,000 Table and Insert 100,000 data", \ - "" using 1:4 title "Request Per Second of Insert 100,000 data" +plot filename . '.csv' using 1:2 title "Create 10,000 Tables", \ + "" using 1:3 title "Delete 10,000 Tables", \ + "" using 1:4 title "Create 10,000 Tables and Insert 100,000 records"