From 4f709c0cab59632426a361237699548c727980b6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 19 Oct 2022 14:40:25 +0800 Subject: [PATCH] feat: taosbenchmark support partial col 9284147 for 2.6 (#17457) * feat: taosbenchmark support partial col 9284147 for 2.6 * fix: free cfgs * fix: better handle command line and json memory * test: fix two taosbenchmark test cases * feat: update taos-tools cc973e0 * test: fix 5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py --- src/kit/taos-tools | 2 +- tests/pytest/query/query1970YearsAf.py | 87 +++++++++++-------- .../taosdemoTestInsertWithJson.py | 69 ++++++++------- 3 files changed, 90 insertions(+), 68 deletions(-) diff --git a/src/kit/taos-tools b/src/kit/taos-tools index 217a267b53..cc973e0c29 160000 --- a/src/kit/taos-tools +++ b/src/kit/taos-tools @@ -1 +1 @@ -Subproject commit 217a267b5365b7a4089eb6ee2d4b2ceedfc4df66 +Subproject commit cc973e0c2999a739fa2d06c25ab5adb2a7537c8f diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 62b435fbef..82b2668242 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -25,8 +25,8 @@ from util.cases import * from util.dnodes import * from util.dnodes import TDDnode -class TDTestCase: +class TDTestCase: def __init__(self): self.path = "" @@ -68,7 +68,7 @@ class TDTestCase: "cachelast": 0, "quorum": 1, "fsync": 3000, - "update": 0 + "update": 0, } # set stable schema @@ -99,8 +99,7 @@ class TDTestCase: {"type": "TINYINT", "count": 2}, {"type": "BOOL", "count": 2}, {"type": "NCHAR", "len": 3, "count": 1}, - {"type": "BINARY", "len": 8, "count": 1} - + {"type": "BINARY", "len": 8, "count": 1}, ], "tags": [ {"type": "INT", "count": 2}, @@ -111,17 +110,14 @@ class TDTestCase: {"type": "TINYINT", "count": 2}, {"type": "BOOL", "count": 2}, {"type": "NCHAR", "len": 3, "count": 1}, - {"type": "BINARY", "len": 8, "count": 1} - ] + {"type": "BINARY", "len": 8, "count": 1}, + ], } # create different stables like stable1 and add to list super_tables super_tables = [] super_tables.append(stable1) - database = { - "dbinfo": dbinfo, - "super_tables": super_tables - } + database = {"dbinfo": dbinfo, "super_tables": super_tables} cfgdir = self.getCfgDir() create_table = { @@ -137,33 +133,60 @@ class TDTestCase: "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 100, - "databases": [database] + "databases": [database], } return create_table + def getPath(self, tool="taosBenchmark"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] + elif "/tests/" in selfPath: + projPath = selfPath[: selfPath.find("/tests/")] + else: + tdLog.exit("path: %s is not supported" % selfPath) + + paths = [] + for root, dirs, files in os.walk(projPath): + if (tool) in files: + rootRealPath = os.path.dirname(os.path.realpath(root)) + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) + break + if len(paths) == 0: + return "" + return paths[0] + def createinsertfile(self): create_table = self.creatcfg() date = datetime.datetime.now().strftime("%Y%m%d%H%M") file_create_table = f"/tmp/insert_{date}.json" - with open(file_create_table, 'w') as f: + with open(file_create_table, "w") as f: json.dump(create_table, f) return file_create_table def inserttable(self, filepath): - create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1" + binPath = self.getPath("taosBenchmark") + if binPath == "": + tdLog.exit("taosBenchmark not found!") + else: + tdLog.info("taosBenchmark found in %s" % binPath) + + create_table_cmd = "%s -f %s > /dev/null 2>&1" % (binPath, filepath) _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") def sqlsquery(self): # stable query - tdSql.query( - "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " - ) + tdSql.query("select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' ") tdSql.checkRows(43200) - tdSql.query( - "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " - ) + tdSql.query("select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' ") tdSql.checkRows(6800) tdSql.query( @@ -172,14 +195,10 @@ class TDTestCase: tdSql.checkRows(3590) # child-tables query - tdSql.query( - "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " - ) + tdSql.query("select * from t0 where t0.ts < '1970-01-01 00:00:00.000' ") tdSql.checkRows(4320) - tdSql.query( - "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " - ) + tdSql.query("select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' ") tdSql.checkRows(680) tdSql.query( @@ -192,9 +211,7 @@ class TDTestCase: ) tdSql.checkRows(680) - tdSql.query( - "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " - ) + tdSql.query("select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' ") tdSql.checkRows(679) tdSql.query( @@ -203,9 +220,7 @@ class TDTestCase: tdSql.checkRows(43200) # query with timestamp in 'where ...' - tdSql.query( - "select * from stb2 where stb2.ts > -28800000 " - ) + tdSql.query("select * from stb2 where stb2.ts > -28800000 ") tdSql.checkRows(6790) tdSql.query( @@ -219,14 +234,16 @@ class TDTestCase: tdSql.checkRows(3590) def run(self): - s = 'reset query cache' + s = "reset query cache" tdSql.execute(s) - s = 'create database if not exists db' + s = "create database if not exists db" tdSql.execute(s) - s = 'use db' + s = "use db" tdSql.execute(s) - tdLog.info("==========step1:create table stable and child table,then insert data automatically") + tdLog.info( + "==========step1:create table stable and child table,then insert data automatically" + ) insertfile = self.createinsertfile() self.inserttable(insertfile) diff --git a/tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py b/tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py index 6fbb6464a4..0c903c7d2a 100644 --- a/tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py +++ b/tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py @@ -24,36 +24,44 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - def getBuildPath(self): + def getPath(self, tool="taosBenchmark"): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] + elif "/tests/" in selfPath: + projPath = selfPath[: selfPath.find("/tests/")] else: - projPath = selfPath[:selfPath.find("tests")] + tdLog.exit("path: %s is not supported" % selfPath) + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if (tool) in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + if "packaging" not in rootRealPath: + paths.append(os.path.join(root, tool)) break - return buildPath + if len(paths) == 0: + return "" + return paths[0] def run(self): - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosd not found!") + binPath = self.getPath() + if binPath == "": + tdLog.exit("taosBenchmark not found!") else: - tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" + tdLog.info("taosBenchmark found in %s" % binPath) testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert*_res.txt*") - os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename ) + os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename) # insert: create one or mutiple tables per sql and insert multiple rows per sql - os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath) + os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 11) @@ -69,7 +77,7 @@ class TDTestCase: tdSql.checkData(0, 0, 2000) # restful connector insert data - os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath) + os.system("%s -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 10) @@ -84,19 +92,19 @@ class TDTestCase: tdSql.query("select count(*) from stb1") tdSql.checkData(0, 0, 200) - # default values json files - tdSql.execute("drop database if exists db") - os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath) + # default values json files + tdSql.execute("drop database if exists db") + os.system("%s -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath) tdSql.query("show databases;") for i in range(tdSql.queryRows): - if tdSql.queryResult[i][0] == 'db': - tdSql.checkData(i, 2, 100) - tdSql.checkData(i, 4, 1) - tdSql.checkData(i, 6, 10) - tdSql.checkData(i, 16, 'ms') - + if tdSql.queryResult[i][0] == "db": + tdSql.checkData(i, 2, 100) + tdSql.checkData(i, 4, 1) + tdSql.checkData(i, 6, 10) + tdSql.checkData(i, 16, "ms") + # insert: create mutiple tables per sql and insert one rows per sql . - os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath) + os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath) tdSql.execute("use db") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 10) @@ -113,7 +121,9 @@ class TDTestCase: # insert: using parament "insert_interval to controls spped of insert. # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。 - os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath) + os.system( + "%s -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath + ) tdSql.execute("use db") tdSql.query("show stables") tdSql.checkData(0, 4, 10) @@ -131,11 +141,6 @@ class TDTestCase: # rm useless files os.system("rm -rf ./insert*_res.txt*") - - - - - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) -- GitLab