diff --git a/tests/comparisonTest/tdengine/q5.txt b/tests/comparisonTest/tdengine/q5.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f36955dd6a426cd3fae0a15d264facf4633502f --- /dev/null +++ b/tests/comparisonTest/tdengine/q5.txt @@ -0,0 +1 @@ +select * from db.devices; diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index de925263376933f55b5fc67d25d7a253da65d2b6..3d78a3d0a54ac571aceb13c1bb9a6f133589bd86 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -13,8 +13,9 @@ typedef struct { char sql[256]; char dataDir[256]; int filesNum; - int writeClients; + int clients; int rowsPerRequest; + int write; } ProArgs; typedef struct { @@ -41,7 +42,7 @@ int main(int argc, char *argv[]) { statis.totalRows = 0; parseArg(argc, argv); - if (arguments.writeClients > 0) { + if (arguments.write) { writeData(); } else { readData(); @@ -52,7 +53,7 @@ void parseArg(int argc, char *argv[]) { strcpy(arguments.sql, "./sqlCmd.txt"); strcpy(arguments.dataDir, "./testdata"); arguments.filesNum = 2; - arguments.writeClients = 0; + arguments.clients = 1; arguments.rowsPerRequest = 100; for (int i = 1; i < argc; ++i) { @@ -83,12 +84,12 @@ void parseArg(int argc, char *argv[]) { exit(EXIT_FAILURE); } } - else if (strcmp(argv[i], "-writeClients") == 0) { + else if (strcmp(argv[i], "-clients") == 0) { if (i < argc - 1) { - arguments.writeClients = atoi(argv[++i]); + arguments.clients = atoi(argv[++i]); } else { - fprintf(stderr, "'-writeClients' requires a parameter, default:%d\n", arguments.writeClients); + fprintf(stderr, "'-clients' requires a parameter, default:%d\n", arguments.clients); exit(EXIT_FAILURE); } } @@ -101,6 +102,9 @@ void parseArg(int argc, char *argv[]) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-w") == 0) { + arguments.write = 1; + } } } @@ -215,7 +219,7 @@ void writeDataImp(void *param) { void writeData() { printf("write data\n"); - printf("---- writeClients: %d\n", arguments.writeClients); + printf("---- clients: %d\n", arguments.clients); printf("---- dataDir: %s\n", arguments.dataDir); printf("---- numOfFiles: %d\n", arguments.filesNum); printf("---- rowsPerRequest: %d\n", arguments.rowsPerRequest); @@ -243,12 +247,12 @@ void writeData() { int64_t st = getTimeStampMs(); - int a = arguments.filesNum / arguments.writeClients; - int b = arguments.filesNum % arguments.writeClients; + int a = arguments.filesNum / arguments.clients; + int b = arguments.filesNum % arguments.clients; int last = 0; - ThreadObj *threads = calloc((size_t)arguments.writeClients, sizeof(ThreadObj)); - for (int i = 0; i < arguments.writeClients; ++i) { + ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj)); + for (int i = 0; i < arguments.clients; ++i) { ThreadObj *pthread = threads + i; pthread_attr_t thattr; pthread->threadId = i + 1; @@ -264,7 +268,7 @@ void writeData() { pthread_create(&pthread->pid, &thattr, (void *(*)(void *))writeDataImp, pthread); } - for (int i = 0; i < arguments.writeClients; i++) { + for (int i = 0; i < arguments.clients; i++) { pthread_join(threads[i].pid, NULL); } @@ -272,17 +276,15 @@ void writeData() { float seconds = (float)elapsed / 1000; float rs = (float)statis.totalRows / seconds; + free(threads); + printf("---- Spent %f seconds to insert %ld records, speed: %f Rows/Second\n", seconds, statis.totalRows, rs); } -void readData() { - printf("read data\n"); - printf("---- sql: %s\n", arguments.sql); - - void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) - taos_error(taos); - +void readDataImp(void *param) +{ + ThreadObj *pThread = (ThreadObj *)param; + printf("Thread %d\n", pThread->threadId); FILE *fp = fopen(arguments.sql, "r"); if (fp == NULL) { printf("failed to open file %s\n", arguments.sql); @@ -290,6 +292,10 @@ void readData() { } printf("open file %s success\n", arguments.sql); + void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + if (taos == NULL) + taos_error(taos); + char *line = NULL; size_t len = 0; while (!feof(fp)) { @@ -325,9 +331,36 @@ void readData() { int64_t elapsed = getTimeStampMs() - st; float seconds = (float)elapsed / 1000; - printf("---- Spent %f seconds to query: %s", seconds, line); + printf("---- Spent %f seconds to retrieve %d records, Thread:%d query: %s\n", seconds, rows, pThread->threadId, line); } fclose(fp); } +void readData() { + printf("read data\n"); + printf("---- sql: %s\n", arguments.sql); + printf("---- clients: %d\n", arguments.clients); + + void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + if (taos == NULL) + taos_error(taos); + + ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj)); + + for (int i = 0; i < arguments.clients; ++i) { + ThreadObj *pthread = threads + i; + pthread_attr_t thattr; + pthread->threadId = i + 1; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + pthread_create(&pthread->pid, &thattr, (void *(*)(void *))readDataImp, pthread); + } + + for (int i = 0; i < arguments.clients; i++) { + pthread_join(threads[i].pid, NULL); + } + + free(threads); +} + diff --git a/tests/perftest-scripts/tdengineTestQ5Loop.sh b/tests/perftest-scripts/tdengineTestQ5Loop.sh new file mode 100755 index 0000000000000000000000000000000000000000..9eff74f563fdd21fddd885324096daef26170f94 --- /dev/null +++ b/tests/perftest-scripts/tdengineTestQ5Loop.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TDTESTQ5OUT=tdengineTestQ5.out + +function runTest { + totalThroughput=0 + for i in `seq 1 $NUM_LOOP`; do + for c in `seq 1 $clients`; do + records[$c]=0 + spentTime[$c]=0 + throughput[$c]=0 + done + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -clients $clients -sql q5.txt" + restartTaosd + beginMS=`date +%s%3N` + $TDTEST_DIR/tdengineTest \ + -clients $clients -sql $TDTEST_DIR/q5.txt > $TDTESTQ5OUT + endMS=`date +%s%3N` + totalRecords=0 + for c in `seq 1 $clients`; do + records[$c]=`grep Thread:$c $TDTESTQ5OUT | awk '{print $7}'` + totalRecords=`echo "$totalRecords + ${records[$c]}"|bc` + done + spending=`echo "scale=4; x = ($endMS - $beginMS)/1000; if (x<1) print 0; x"|bc` + throughput=`echo "scale=4; x= $totalRecords / $spending; if (x<1) print 0; x" | bc` + printTo "spending: $spending sec, throughput: $throughput" + totalThroughput=`echo "scale=4; x = $totalThroughput + $throughput; if(x<1) print 0; x"|bc` + done + avgThrougput=`echo "scale=4; x = $totalThroughput / $NUM_LOOP; if (x<1) print 0; x"|bc` + echo "avg Throughput: $avgThrougput" +} + +function restartTaosd { + printTo "Stop taosd" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + + printTo "Start taosd" + $TAOSD_DIR/taosd > /dev/null 2>&1 & + sleep 10 +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + master) + master=true + develop=false + shift ;; + + develop) + master=false + develop=true + shift ;; + + -c) + clients=$2 + shift 2;; + + *) + break ;; + esac +done + +if $master ; then + printTo "Test master branch.." + cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg + WORK_DIR=/mnt/root/TDengine.master +else + printTo "Test develop branch.." + cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg + WORK_DIR=/mnt/root/TDengine +fi + +TAOSD_DIR=$WORK_DIR/debug/build/bin +TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index 9f54bb6fc3e04144e9e21ce90ffcdc3fd222863d..4cbb7916780316f186417f49ef86499a52c25834 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -5,7 +5,6 @@ NUM_LOOP=5 NUM_OF_FILES=100 rowsPerRequest=(1 100 500 1000 2000) -numOfClients=(1 2 3 4 5 6 7) function printTo { if $verbose ; then @@ -15,7 +14,7 @@ function printTo { function runTest { printf "R/R, " - for c in ${numOfClients[@]}; do + for c in `seq 1 $clients`; do if [ "$c" == "1" ]; then printf "$c client, " else @@ -26,7 +25,7 @@ function runTest { for r in ${rowsPerRequest[@]}; do printf "$r, " - for c in ${numOfClients[@]}; do + for c in `seq 1 $clients`; do totalRPR=0 for i in `seq 1 $NUM_LOOP`; do restartTaosd @@ -34,12 +33,12 @@ function runTest { printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -writeClients $c \ + -w -clients $c \ -rowsPerRequest $r" RPR=`$TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles 1 \ - -writeClients $c \ + -w -clients $c \ -rowsPerRequest $r \ | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` @@ -73,25 +72,29 @@ function restartTaosd { master=false develop=true verbose=false +clients=1 -for arg in "$@" -do - case $arg in +while : ; do + case $1 in -v) verbose=true - ;; + shift ;; master) master=true develop=false - ;; + shift ;; develop) master=false develop=true - ;; + shift ;; + + -c) + clients=$2 + shift 2;; *) - ;; + break ;; esac done