diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index 81103252d8794032ee17b143f814276e171863f2..29e5cb19b75b0943c24382d268e81daebed01cdf 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -17,6 +17,7 @@ import os import taos import time import argparse +import json class taosdemoQueryPerformace: @@ -48,7 +49,7 @@ class taosdemoQueryPerformace: cursor2 = self.conn2.cursor() cursor2.execute("create database if not exists %s" % self.dbName) cursor2.execute("use %s" % self.dbName) - cursor2.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50), branch binary(50), type binary(20)) tags(query_id int, query_sql binary(300))" % self.stbName) + cursor2.execute("create table if not exists %s(ts timestamp, query_time_avg float, query_time_max float, query_time_min float, commit_id binary(50), branch binary(50), type binary(20)) tags(query_id int, query_sql binary(300))" % self.stbName) sql = "select count(*) from test.meters" tableid = 1 @@ -74,7 +75,7 @@ class taosdemoQueryPerformace: tableid = 6 cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) - sql = "select * from meters" + sql = "select * from meters limit 10000" tableid = 7 cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) @@ -87,37 +88,96 @@ class taosdemoQueryPerformace: cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) cursor2.close() + + def generateQueryJson(self): + + sqls = [] + cursor2 = self.conn2.cursor() + cursor2.execute("select query_id, query_sql from %s.%s" % (self.dbName, self.stbName)) + i = 0 + for data in cursor2: + sql = { + "sql": data[1], + "result_mode": "onlyformat", + "result_file": "./query_sql_res%d.txt" % i + } + sqls.append(sql) + i += 1 + + query_data = { + "filetype": "query", + "cfgdir": "/etc/perf", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "test", + "specified_table_query": { + "query_times": 100, + "concurrent": 1, + "sqls": sqls + } + } + + query_json_file = f"/tmp/query.json" + + with open(query_json_file, 'w') as f: + json.dump(query_data, f) + return query_json_file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosdemo" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def getCMDOutput(self, cmd): + cmd = os.popen(cmd) + output = cmd.read() + cmd.close() + return output def query(self): - cursor = self.conn.cursor() - print("==================== query performance ====================") + buildPath = self.getBuildPath() + if (buildPath == ""): + print("taosdemo not found!") + sys.exit(1) + + binPath = buildPath + "/build/bin/" + os.system( + "%sperfMonitor -f %s > query_res.txt" % + (binPath, self.generateQueryJson())) + cursor = self.conn2.cursor() + print("==================== query performance ====================") cursor.execute("use %s" % self.dbName) - cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName) + cursor.execute("select tbname, query_sql from %s" % self.stbName) + i = 0 for data in cursor: table_name = data[0] - query_id = data[1] - sql = data[2] + sql = data[1] + + self.avgDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $2}'" % (i + 1)) + self.maxDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $5}'" % (i + 1)) + self.minDelay = self.getCMDOutput("grep 'avgDelay' query_res.txt | awk 'NR==%d{print $8}'" % (i + 1)) + i += 1 + + print("query time for: %s %f seconds" % (sql, float(self.avgDelay))) + c = self.conn2.cursor() + c.execute("insert into %s.%s values(now, %f, %f, %f, '%s', '%s', '%s')" % (self.dbName, table_name, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.commitID, self.branch, self.type)) - totalTime = 0 - cursor2 = self.conn.cursor() - cursor2.execute("use test") - for i in range(100): - if(self.clearCache == True): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - - startTime = time.time() - cursor2.execute(sql) - totalTime += time.time() - startTime - cursor2.close() - print("query time for: %s %f seconds" % (sql, totalTime / 100)) - - cursor3 = self.conn2.cursor() - cursor3.execute("insert into %s.%s values(now, %f, '%s', '%s', '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID, self.branch, self.type)) - - cursor3.close() + c.close() cursor.close() if __name__ == '__main__': @@ -174,4 +234,4 @@ if __name__ == '__main__': args = parser.parse_args() perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix, args.git_branch, args.build_type) perftest.createPerfTables() - perftest.query() + perftest.query() \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 1d28a2708fa1911e22aca97afa246a124dd2d6fc..51b064a08e5cd55401f9cf803a8683653f722679 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -49,24 +49,18 @@ class taosdemoPerformace: def generateJson(self): db = { "name": "%s" % self.insertDB, - "drop": "yes", - "replica": 1 + "drop": "yes" } stb = { "name": "meters", - "child_table_exists": "no", "childtable_count": self.numOfTables, "childtable_prefix": "stb_", - "auto_create_table": "no", - "data_source": "rand", "batch_create_tbl_num": 10, - "insert_mode": "taosc", + "insert_mode": "rand", "insert_rows": self.numOfRows, - "interlace_rows": 0, - "max_sql_len": 1024000, - "disorder_ratio": 0, - "disorder_range": 1000, + "batch_rows": 1000000, + "max_sql_len": 1048576, "timestamp_step": 1, "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", @@ -100,11 +94,8 @@ class taosdemoPerformace: "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "thread_count_create_tbl": 4, "result_file": "./insert_res.txt", - "confirm_parameter_prompt": "no", - "insert_interval": 0, - "num_of_records_per_req": 30000, "databases": [db] } @@ -145,7 +136,7 @@ class taosdemoPerformace: binPath = buildPath + "/build/bin/" os.system( - "%staosdemo -f %s > /dev/null 2>&1" % + "%sperfMonitor -f %s > /dev/null 2>&1" % (binPath, self.generateJson())) self.createTableTime = self.getCMDOutput( "grep 'Spent' insert_res.txt | awk 'NR==1{print $2}'")