Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
未验证 提交 8f86e3bd 编写于 作者: H huili 提交者： GitHub
浏览文件
操作

Merge pull request #5631 from taosdata/xiaoping/add_test_case 

[TD-3602]<test>: update perf test script
上级 3fca391b 3631d276
隐藏空白更改
内联 并排
Showing with 114 addition and 84 deletion
tests/perftest-scripts/perftest-query.sh
浏览文件 @ 8f86e3bd
......@@ -50,7 +50,7 @@ function buildTDengine {
echo "repo up-to-date"
else
echo "repo need to pull"
git pull > /dev/null
git pull > /dev/null 2>&1
LOCAL_COMMIT=`git rev-parse --short @`
cd debug
......@@ -71,27 +71,14 @@ function runQueryPerfTest {
python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -x > taosdemoperf.txt
CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'`
INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'`
REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'`
delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'`
AVGDELAY=`echo ${delay:0:${#delay}-3}`
delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'`
MAXDELAY=`echo ${delay:0:${#delay}-3}`
delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'`
MINDELAY=`echo ${delay:0:${#delay}-2}`
python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT
[ -f taosdemoperf.txt ] && rm taosdemoperf.txt
python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
}
function sendReport {
echo "send report"
receiver="pxiao@taosdata.com"
receiver="develop@taosdata.com"
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
cd $TDENGINE_DIR
......
tests/pytest/tools/taosdemoPerformance.py
浏览文件 @ 8f86e3bd
......@@ -11,26 +11,16 @@
# -*- coding: utf-8 -*-
import sys
import taos
import time
import datetime
import csv
import random
import pandas as pd
import argparse
import os.path
import json
class taosdemoPerformace:
def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay):
def __init__(self, commitID, dbName):
self.commitID = commitID
self.dbName = dbName
self.createTableTime = createTableTime
self.insertRecordsTime = insertRecordsTime
self.recordsPerSecond = recordsPerSecond
self.avgDelay = avgDelay
self.maxDelay = maxDelay
self.minDelay = minDelay
self.dbName = dbName
self.host = "127.0.0.1"
self.user = "root"
self.password = "taosdata"
......@@ -39,8 +29,97 @@ class taosdemoPerformace:
self.host,
self.user,
self.password,
self.config)
self.config)
self.insertDB = "insertDB";
def generateJson(self):
db = {
"name": "%s" % self.insertDB,
"drop": "yes",
"replica": 1
}
stb = {
"name": "meters",
"child_table_exists":"no",
"childtable_count": 10000,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"data_source": "rand",
"batch_create_tbl_num": 10,
"insert_mode": "taosc",
"insert_rows": 100000,
"multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 0,
"rows_per_tbl": 100,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [
{"type": "INT", "count": 4}
],
"tags": [
{"type": "INT", "count":1},
{"type": "BINARY", "len": 16}
]
}
stables = []
stables.append(stb)
db = {
"dbinfo": db,
"super_tables": stables
}
insert_data = {
"filetype": "insert",
"cfgdir": "/etc/taosperf",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 10,
"thread_count_create_tbl": 10,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"num_of_records_per_req": 30000,
"databases": [db]
}
insert_json_file = f"/tmp/insert.json"
with open(insert_json_file, 'w') as f:
json.dump(insert_data, f)
return insert_json_file
def getCMDOutput(self, cmd):
cmd = os.popen(cmd)
output = cmd.read()
cmd.close()
return output
def insertData(self):
os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson())
self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'")
self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'")
self.commitID = self.getCMDOutput("git rev-parse --short HEAD")
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'")
self.avgDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'")
self.maxDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'")
self.minDelay = delay[:-3]
os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt")
def createTablesAndStoreData(self):
cursor = self.conn.cursor()
......@@ -48,14 +127,15 @@ class taosdemoPerformace:
cursor.execute("use %s" % self.dbName)
cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)")
print("==================== taosdemo performance ====================")
print("create tables time: %f" % self.createTableTime)
print("insert records time: %f" % self.insertRecordsTime)
print("records per second: %f" % self.recordsPerSecond)
print("avg delay: %f" % self.avgDelay)
print("max delay: %f" % self.maxDelay)
print("min delay: %f" % self.minDelay)
cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay))
cursor.execute("drop database if exists taosdemo_insert_test")
print("create tables time: %f" % float(self.createTableTime))
print("insert records time: %f" % float(self.insertRecordsTime))
print("records per second: %f" % float(self.recordsPerSecond))
print("avg delay: %f" % float(self.avgDelay))
print("max delay: %f" % float(self.maxDelay))
print("min delay: %f" % float(self.minDelay))
cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" %
(float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay)))
cursor.execute("drop database if exists %s" % self.insertDB)
cursor.close()
......@@ -64,7 +144,7 @@ if __name__ == '__main__':
parser.add_argument(
'-c',
'--commit-id',
action='store',
action='store',
type=str,
help='git commit id (default: null)')
parser.add_argument(
......@@ -74,46 +154,9 @@ if __name__ == '__main__':
default='perf',
type=str,
help='Database name to be created (default: perf)')
parser.add_argument(
'-t',
'--create-table',
action='store',
type=float,
help='create table time')
parser.add_argument(
'-i',
'--insert-records',
action='store',
type=float,
help='insert records time')
parser.add_argument(
'-r',
'---records-per-second',
action='store',
type=float,
help='records per request')
parser.add_argument(
'-avg',
'---avg-delay',
action='store',
type=float,
help='avg delay')
parser.add_argument(
'-max',
'---max-delay',
action='store',
type=float,
help='max delay')
parser.add_argument(
'-min',
'---min-delay',
action='store',
type=float,
help='min delay')
args = parser.parse_args()
perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second,
args.avg_delay, args.max_delay, args.min_delay)
perftest = taosdemoPerformace(args.commit_id, args.database_name)
perftest.insertData()
perftest.createTablesAndStoreData()
\ No newline at end of file
tests/test-all.sh
浏览文件 @ 8f86e3bd
......@@ -463,10 +463,10 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
echo "Running tests"
./apitest > /dev/null 2>&1
if [ $? != "0" ]; then
echo "prepare failed"
echo "apitest failed"
totalExampleFailed=`expr $totalExampleFailed + 1`
else
echo "prepare pass"
echo "apitest pass"
totalExamplePass=`expr $totalExamplePass + 1`
fi
......@@ -481,19 +481,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" !=
./subscribe -test > /dev/null 2>&1
if [ $? != "0" ]; then
echo "prepare failed"
echo "subscribe failed"
totalExampleFailed=`expr $totalExampleFailed + 1`
else
echo "prepare pass"
echo "subscribe pass"
totalExamplePass=`expr $totalExamplePass + 1`
fi
yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1
if [ $? != "0" ]; then
echo "prepare failed"
echo "asyncdemo failed"
totalExampleFailed=`expr $totalExampleFailed + 1`
else
echo "prepare pass"
echo "asyncdemo pass"
totalExamplePass=`expr $totalExamplePass + 1`
fi
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或