From 08f11d51eebc2e3b7ae7c6bc20c7837ca3c0ece4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 2 Jul 2021 10:14:41 +0800 Subject: [PATCH] [TD-5007]: use build directory's taosdump to test. (#6711) btw, solve two taosdemo test cases too. --- tests/pytest/tools/taosdemoPerformance.py | 83 +++++++++++++++++------ tests/pytest/tools/taosdemoTest.py | 6 +- tests/pytest/tools/taosdumpTest.py | 47 +++++++++---- tests/pytest/tools/taosdumpTest2.py | 43 +++++++++--- 4 files changed, 134 insertions(+), 45 deletions(-) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index a45393e222..c4b125969a 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -17,10 +17,11 @@ import argparse import os.path import json + class taosdemoPerformace: def __init__(self, commitID, dbName): self.commitID = commitID - self.dbName = dbName + self.dbName = dbName self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -30,8 +31,8 @@ class taosdemoPerformace: self.user, self.password, self.config) - self.insertDB = "insertDB"; - + self.insertDB = "insertDB" + def generateJson(self): db = { "name": "%s" % self.insertDB, @@ -41,7 +42,7 @@ class taosdemoPerformace: stb = { "name": "meters", - "child_table_exists":"no", + "child_table_exists": "no", "childtable_count": 10000, "childtable_prefix": "stb_", "auto_create_table": "no", @@ -57,12 +58,12 @@ class taosdemoPerformace: "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", - "tags_file": "", + "tags_file": "", "columns": [ {"type": "INT", "count": 4} - ], + ], "tags": [ - {"type": "INT", "count":1}, + {"type": "INT", "count": 1}, {"type": "BINARY", "len": 16} ] } @@ -88,7 +89,7 @@ class taosdemoPerformace: "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 30000, - "databases": [db] + "databases": [db] } insert_json_file = f"/tmp/insert.json" @@ -103,24 +104,56 @@ class taosdemoPerformace: cmd.close() return output - def insertData(self): - os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) - self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") - self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") - self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosdemo" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def insertData(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdemo not found!") + else: + tdLog.info("taosdemo found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + + os.system( + "%staosdemo -f %s > taosdemoperf.txt 2>&1" % + (binPath, self.generateJson())) + self.createTableTime = self.getCMDOutput( + "grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") + self.insertRecordsTime = self.getCMDOutput( + "grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") + self.recordsPerSecond = self.getCMDOutput( + "grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") self.commitID = self.getCMDOutput("git rev-parse --short HEAD") - delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") + delay = self.getCMDOutput( + "grep 'delay' taosdemoperf.txt | awk '{print $4}'") self.avgDelay = delay[:-4] - delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") + delay = self.getCMDOutput( + "grep 'delay' taosdemoperf.txt | awk '{print $6}'") self.maxDelay = delay[:-4] - delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") + delay = self.getCMDOutput( + "grep 'delay' taosdemoperf.txt | awk '{print $8}'") self.minDelay = delay[:-3] os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") def createTablesAndStoreData(self): cursor = self.conn.cursor() - + cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") @@ -130,13 +163,21 @@ class taosdemoPerformace: print("records per second: %f" % float(self.recordsPerSecond)) print("avg delay: %f" % float(self.avgDelay)) print("max delay: %f" % float(self.maxDelay)) - print("min delay: %f" % float(self.minDelay)) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % - (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) + print("min delay: %f" % float(self.minDelay)) + cursor.execute( + "insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % + (float( + self.createTableTime), float( + self.insertRecordsTime), float( + self.recordsPerSecond), self.commitID, float( + self.avgDelay), float( + self.maxDelay), float( + self.minDelay))) cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( @@ -155,6 +196,6 @@ if __name__ == '__main__': args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name) + perftest = taosdemoPerformace(args.commit_id, args.database_name) perftest.insertData() perftest.createTablesAndStoreData() diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index ff5921be60..fec69bed64 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -36,7 +36,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosdemo" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -47,9 +47,9 @@ class TDTestCase: tdSql.prepare() buildPath = self.getBuildPath() if (buildPath == ""): - tdLog.exit("taosd not found!") + tdLog.exit("taosdemo not found!") else: - tdLog.info("taosd found in %s" % buildPath) + tdLog.info("taosdemo found in %s" % buildPath) binPath = buildPath + "/build/bin/" os.system("%staosdemo -y -t %d -n %d" % (binPath, self.numberOfTables, self.numberOfRecords)) diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 8746f4ecdf..2eb512a575 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -23,43 +23,66 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + self.ts = 1538548685000 self.numberOfTables = 10000 self.numberOfRecords = 100 - + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosdump" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + def run(self): tdSql.prepare() - tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") - tdSql.execute("create table t1 using st tags(1, 'beijing')") + tdSql.execute( + "create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") + tdSql.execute("create table t1 using st tags(1, 'beijing')") sql = "insert into t1 values" currts = self.ts for i in range(100): sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) tdSql.execute(sql) - tdSql.execute("create table t2 using st tags(2, 'shanghai')") + tdSql.execute("create table t2 using st tags(2, 'shanghai')") sql = "insert into t2 values" currts = self.ts for i in range(100): sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) tdSql.execute(sql) - + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % buildPath) + binPath = buildPath + "/build/bin/" os.system("rm /tmp/*.sql") - os.system("taosdump --databases db -o /tmp") - + os.system("%staosdump --databases db -o /tmp" % binPath) + tdSql.execute("drop database db") tdSql.query("show databases") tdSql.checkRows(0) - - os.system("taosdump -i /tmp") + + os.system("%staosdump -i /tmp" % binPath) tdSql.query("show databases") tdSql.checkRows(1) tdSql.checkData(0, 0, 'db') - + tdSql.execute("use db") tdSql.query("show stables") tdSql.checkRows(1) @@ -88,4 +111,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index 51a73555a8..bed0564139 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -23,11 +23,27 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + self.ts = 1601481600000 self.numberOfTables = 1 self.numberOfRecords = 15000 - + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosdump" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + def run(self): tdSql.prepare() @@ -41,22 +57,31 @@ class TDTestCase: sql += "(%d, 1019774612, 29931, 1442173978, 165092.468750, 1128.643179, 'MOCq1pTu', 18405, 82, 0, 'g0A6S0Fu')" % (currts + i) finish = i + 1 if (1048576 - len(sql)) < 16384: - break + break tdSql.execute(sql) + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + os.system("rm /tmp/*.sql") - os.system("taosdump --databases db -o /tmp -B 32766 -L 1048576") - + os.system( + "%staosdump --databases db -o /tmp -B 32766 -L 1048576" % + binPath) + tdSql.execute("drop database db") tdSql.query("show databases") tdSql.checkRows(0) - - os.system("taosdump -i /tmp") + + os.system("%staosdump -i /tmp" % binPath) tdSql.query("show databases") tdSql.checkRows(1) tdSql.checkData(0, 0, 'db') - + tdSql.execute("use db") tdSql.query("show stables") tdSql.checkRows(1) @@ -71,4 +96,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) -- GitLab