From 9640e6f161deebb10a08eca6e3aac38e6d557287 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Mar 2022 00:21:06 +0800 Subject: [PATCH] [TD-14331]: make util/dnodes.py flexible (#11020) * [TD-14331]: make util/dnodes.py flexible * fix 5-taos-tools/basic.py * fix 5-taos-tools/basic.py typo * make util/dnodes.py getPath logic clear * fix a few files * fix a few taosdump/taosdemo test cases --- tests/pytest/tools/taosdemoTest.py | 20 +- tests/pytest/tools/taosdumpTest.py | 29 +-- tests/pytest/tools/taosdumpTestBenchmark.py | 44 ++-- tests/pytest/tools/taosdumpTestNanoSupport.py | 112 +++++----- tests/pytest/util/dnodes-default.py | 32 +-- tests/pytest/util/dnodes-no-random-fail.py | 31 ++- tests/pytest/util/dnodes-random-fail.py | 31 ++- tests/pytest/util/dnodes.py | 193 ++++++++++-------- tests/system-test/5-taos-tools/basic.py | 23 ++- 9 files changed, 268 insertions(+), 247 deletions(-) diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 54e0906672..3de3961faf 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -27,7 +27,7 @@ class TDTestCase: self.numberOfTables = 1000 self.numberOfRecords = 100 - def getBuildPath(self): + def getPath(self, tool="taosBenchmark"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -35,23 +35,23 @@ class TDTestCase: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def run(self): tdSql.prepare() - buildPath = self.getBuildPath() - if (buildPath == ""): - tdLog.exit("taosdemo not found!") + binPath = self.getPath("taosBenchmark") + if (binPath == ""): + tdLog.exit("taosBenchmark not found!") else: - tdLog.info("taosdemo found in %s" % buildPath) - binPath = buildPath + "/build/bin/" - os.system("%staosBenchmark -y -t %d -n %d -b INT,INT,INT,INT" % + tdLog.info("taosBenchmark found: %s" % binPath) + os.system("%s -y -t %d -n %d -b INT,INT,INT,INT" % (binPath, self.numberOfTables, self.numberOfRecords)) tdSql.execute("use test") diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index 628617e27b..10e37dd26f 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -35,7 +35,7 @@ class TDTestCase: else: return True - def getBuildPath(self): + def getPath(self, tool="taosdump"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -43,13 +43,14 @@ class TDTestCase: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosdump" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def run(self): if not os.path.exists("./taosdumptest/tmp1"): @@ -78,16 +79,15 @@ class TDTestCase: sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) tdSql.execute(sql) - buildPath = self.getBuildPath() - if (buildPath == ""): + binPath = self.getPath() + if (binPath == ""): tdLog.exit("taosdump not found!") else: - tdLog.info("taosdump found in %s" % buildPath) - binPath = buildPath + "/build/bin/" + tdLog.info("taosdump found: %s" % binPath) - os.system("%staosdump --databases db -o ./taosdumptest/tmp1" % binPath) + os.system("%s --databases db -o ./taosdumptest/tmp1" % binPath) os.system( - "%staosdump --databases db1 -o ./taosdumptest/tmp2" % + "%s --databases db1 -o ./taosdumptest/tmp2" % binPath) tdSql.execute("drop database db") @@ -95,8 +95,8 @@ class TDTestCase: tdSql.query("show databases") tdSql.checkRows(0) - os.system("%staosdump -i ./taosdumptest/tmp1" % binPath) - os.system("%staosdump -i ./taosdumptest/tmp2" % binPath) + os.system("%s -i ./taosdumptest/tmp1" % binPath) + os.system("%s -i ./taosdumptest/tmp2" % binPath) tdSql.execute("use db") tdSql.query("show databases") @@ -168,9 +168,10 @@ class TDTestCase: tdSql.query("show stables") tdSql.checkRows(2) os.system( - "%staosdump --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % binPath) + "%s --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % + binPath) tdSql.execute("drop database db12312313231231321312312312_323") - os.system("%staosdump -i ./taosdumptest/tmp1" % binPath) + os.system("%s -i ./taosdumptest/tmp1" % binPath) tdSql.execute("use db12312313231231321312312312_323") tdSql.query("show stables") tdSql.checkRows(2) diff --git a/tests/pytest/tools/taosdumpTestBenchmark.py b/tests/pytest/tools/taosdumpTestBenchmark.py index 97dcf3e54b..75238b7f73 100644 --- a/tests/pytest/tools/taosdumpTestBenchmark.py +++ b/tests/pytest/tools/taosdumpTestBenchmark.py @@ -35,7 +35,7 @@ class TDTestCase: else: return True - def getBuildPath(self): + def getPath(self, tool="taosdump"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -43,13 +43,14 @@ class TDTestCase: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosdump" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def insert_data(self, tbname, ts_start, count): pre_insert = "insert into %s values" % tbname @@ -81,12 +82,11 @@ class TDTestCase: os.system("rm -rf ./taosdumptest/tmp%d" % i) os.makedirs("./taosdumptest/tmp%d" % i) - buildPath = self.getBuildPath() - if (buildPath == ""): + binPath = self.getPath("taosdump") + if (binPath == ""): tdLog.exit("taosdump not found!") else: - tdLog.info("taosdump found in %s" % buildPath) - binPath = buildPath + "/build/bin/" + tdLog.info("taosdump found: %s" % binPath) # create db1 , one stables and one table ; create general tables tdSql.execute("drop database if exists dp1") @@ -158,7 +158,7 @@ class TDTestCase: 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, self.ts)) intData.append(i + 1) floatData.append(i + 0.1) - # os.system("%staosBenchmark -f tools/taosdump-insert-dp1.json -y " % binPath) + # os.system("%staosBenchmark -f tools/taosdump-insert-dp1.json -y " % benchBinPath) # create db1 , three stables:stb0,include ctables stb0_0 \ stb0_1,stb1 include ctables stb1_0 and stb1_1 # \stb3,include ctables stb3_0 and stb3_1 @@ -200,22 +200,22 @@ class TDTestCase: "insert into gt1 values(1614218413800,8638,78.862020199)") tdSql.execute("insert into gt2 values(1614218413900,8639,78.863)") # self.insert_data("t", self.ts, 300*10000); - # os.system("%staosBenchmark -f tools/taosdump-insert-dp2.json -y " % binPath) + # os.system("%staosBenchmark -f tools/taosdump-insert-dp2.json -y " % benchBinPath) # # taosdump data - # os.system("%staosdump -o ./taosdumptest/tmp1 taosdump -h -ptaosdata -P 6030 -u root -o taosdumptest \ + # os.system("%s -o ./taosdumptest/tmp1 taosdump -h -ptaosdata -P 6030 -u root -o taosdumptest \ # -D dp1,dp3 -N -c /home/chr/TDinternal/community/sim/dnode1/cfg/taos.cfg -s -d deflate" % binPath) os.system( - "%staosdump -o ./taosdumptest/tmp0 -D dp2,dp1 -T 8" % + "%s -o ./taosdumptest/tmp0 -D dp2,dp1 -T 8" % binPath) os.system( - "%staosdump -o ./taosdumptest/tmp1 dp2 st0 st1_0 gt0 -T 8" % + "%s -o ./taosdumptest/tmp1 dp2 st0 st1_0 gt0 -T 8" % binPath) # check taosdumptest/tmp0 tdSql.execute("drop database dp1") tdSql.execute("drop database dp2") - os.system("%staosdump -i ./taosdumptest/tmp0 -T 8 " % binPath) + os.system("%s -i ./taosdumptest/tmp0 -T 8 " % binPath) tdSql.execute("reset query cache") tdSql.execute("use dp1") @@ -265,7 +265,7 @@ class TDTestCase: # check taosdumptest/tmp1 tdSql.execute("drop database dp1") tdSql.execute("drop database dp2") - os.system("%staosdump -i ./taosdumptest/tmp1 -T 8 " % binPath) + os.system("%s -i ./taosdumptest/tmp1 -T 8 " % binPath) tdSql.execute("reset query cache") tdSql.execute("use dp2") tdSql.query("show stables") @@ -283,7 +283,7 @@ class TDTestCase: # #check taosdumptest/tmp2 # tdSql.execute("drop database dp1") # tdSql.execute("drop database dp2") - # os.system("%staosdump -i ./taosdumptest/tmp2 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp2 -T 8 " % binPath) # tdSql.execute("use dp1") # tdSql.query("show stables") # tdSql.checkRows(1) @@ -301,7 +301,7 @@ class TDTestCase: # #check taosdumptest/tmp3 # tdSql.execute("drop database dp1") - # os.system("%staosdump -i ./taosdumptest/tmp3 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp3 -T 8 " % binPath) # tdSql.execute("use dp2") # tdSql.query("show stables") # tdSql.checkRows(2) @@ -317,7 +317,7 @@ class TDTestCase: # #check taosdumptest/tmp4 # tdSql.execute("drop database dp2") - # os.system("%staosdump -i ./taosdumptest/tmp4 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp4 -T 8 " % binPath) # tdSql.execute("use dp2") # tdSql.query("show stables") # tdSql.checkRows(2) @@ -340,7 +340,7 @@ class TDTestCase: # #check taosdumptest/tmp5 # tdSql.execute("drop database dp2") - # os.system("%staosdump -i ./taosdumptest/tmp5 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp5 -T 8 " % binPath) # tdSql.execute("use dp2") # tdSql.query("show stables") # tdSql.checkRows(3) @@ -377,7 +377,7 @@ class TDTestCase: # tdSql.execute("drop database dp1") # tdSql.execute("drop database dp2") # tdSql.execute("drop database dp3") - # os.system("%staosdump -i ./taosdumptest/tmp6 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp6 -T 8 " % binPath) # tdSql.execute("use dp3") # tdSql.query("show databases") # tdSql.checkRows(1) @@ -394,7 +394,7 @@ class TDTestCase: # # check taosdumptest/tmp7 # tdSql.execute("drop database dp3") - # os.system("%staosdump -i ./taosdumptest/tmp7 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp7 -T 8 " % binPath) # tdSql.execute("use dp3") # tdSql.query("show databases") # tdSql.checkRows(1) @@ -411,7 +411,7 @@ class TDTestCase: # # check taosdumptest/tmp8 # tdSql.execute("drop database dp3") - # os.system("%staosdump -i ./taosdumptest/tmp8 -T 8 " % binPath) + # os.system("%s -i ./taosdumptest/tmp8 -T 8 " % binPath) # tdSql.execute("use dp3") # tdSql.query("show stables") # tdSql.checkRows(1) diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdumpTestNanoSupport.py index 81e3159346..c000b90c8e 100644 --- a/tests/pytest/tools/taosdumpTestNanoSupport.py +++ b/tests/pytest/tools/taosdumpTestNanoSupport.py @@ -35,7 +35,7 @@ class TDTestCase: else: return True - def getBuildPath(self): + def getPath(self, tool="taosdump"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -43,13 +43,14 @@ class TDTestCase: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def createdb(self, precision="ns"): tb_nums = self.numberOfTables @@ -118,12 +119,11 @@ class TDTestCase: if not os.path.exists("./taosdumptest/dumptmp3"): os.makedirs("./taosdumptest/dumptmp3") - buildPath = self.getBuildPath() - if (buildPath == ""): + binPath = self.getPath("taosdump") + if (binPath == ""): tdLog.exit("taosdump not found!") else: - tdLog.info("taosdump found in %s" % buildPath) - binPath = buildPath + "/build/bin/" + tdLog.info("taosdump found: %s" % binPath) # create nano second database @@ -132,40 +132,40 @@ class TDTestCase: # dump all data os.system( - "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) # dump part data with -S -E os.system( - '%staosdump --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % + '%s --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%staosdump --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % + '%s --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % binPath) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp2" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp2" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 510) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp3" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp3" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 900) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 1000) # check data origin_res = tdSql.getResult("select * from timedb1.st") tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump dump_res = tdSql.getResult("select * from timedb1.st") if origin_res == dump_res: tdLog.info("test nano second : dump check data pass for all data!") @@ -177,7 +177,6 @@ class TDTestCase: os.system("rm -rf ./taosdumptest/") tdSql.execute("drop database if exists timedb1") - if not os.path.exists("./taosdumptest/tmp1"): os.makedirs("./taosdumptest/dumptmp1") @@ -190,53 +189,52 @@ class TDTestCase: if not os.path.exists("./taosdumptest/dumptmp3"): os.makedirs("./taosdumptest/dumptmp3") - buildPath = self.getBuildPath() - if (buildPath == ""): + binPath = self.getPath() + if (binPath == ""): tdLog.exit("taosdump not found!") else: - tdLog.info("taosdump found in %s" % buildPath) - binPath = buildPath + "/build/bin/" + tdLog.info("taosdump found: %s" % binPath) self.createdb(precision="us") os.system( - "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%staosdump --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % + '%s --databases timedb1 -S 1625068810000000 -E 1625068860000000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%staosdump --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % + '%s --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp2" % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp3" % binPath) + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + os.system("%s -i ./taosdumptest/dumptmp2" % binPath) + os.system("%s -i ./taosdumptest/dumptmp3" % binPath) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp2" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp2" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 510) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp3" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp3" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 900) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 1000) # check data origin_res = tdSql.getResult("select * from timedb1.st") tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump dump_res = tdSql.getResult("select * from timedb1.st") if origin_res == dump_res: tdLog.info("test micro second : dump check data pass for all data!") @@ -260,56 +258,56 @@ class TDTestCase: if not os.path.exists("./taosdumptest/dumptmp3"): os.makedirs("./taosdumptest/dumptmp3") - buildPath = self.getBuildPath() - if (buildPath == ""): + binPath = self.getPath() + if (binPath == ""): tdLog.exit("taosdump not found!") else: - tdLog.info("taosdump found in %s" % buildPath) - binPath = buildPath + "/build/bin/" + tdLog.info("taosdump found: %s" % binPath) self.createdb(precision="ms") os.system( - "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % + "%s --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) os.system( - '%staosdump --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % + '%s --databases timedb1 -S 1625068810000 -E 1625068860000 -o ./taosdumptest/dumptmp2 ' % binPath) os.system( - '%staosdump --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % + '%s --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp2" % binPath) - os.system("%staosdump -i ./taosdumptest/dumptmp3" % binPath) + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + os.system("%s -i ./taosdumptest/dumptmp2" % binPath) + os.system("%s -i ./taosdumptest/dumptmp3" % binPath) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp2" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp2" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 510) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp3" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp3" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 900) tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump tdSql.query("select count(*) from timedb1.st") tdSql.checkData(0, 0, 1000) # check data origin_res = tdSql.getResult("select * from timedb1.st") tdSql.execute("drop database timedb1") - os.system("%staosdump -i ./taosdumptest/dumptmp1" % binPath) - # dump data and check for taosdump + os.system("%s -i ./taosdumptest/dumptmp1" % binPath) + # dump data and check for taosdump dump_res = tdSql.getResult("select * from timedb1.st") if origin_res == dump_res: - tdLog.info("test million second : dump check data pass for all data!") + tdLog.info( + "test million second : dump check data pass for all data!") else: tdLog.info( "test million second : dump check data failed for all data!") diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py index 7d8fc3f630..055d3085d3 100644 --- a/tests/pytest/util/dnodes-default.py +++ b/tests/pytest/util/dnodes-default.py @@ -40,7 +40,8 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", "telemetryReporting": "0", - } + } + def init(self, path): self.__init__() self.path = path @@ -72,14 +73,14 @@ class TDSimClient: cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -145,11 +146,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -198,7 +199,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self): + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -206,23 +207,22 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py index 86ef9e178e..467a19fa4d 100644 --- a/tests/pytest/util/dnodes-no-random-fail.py +++ b/tests/pytest/util/dnodes-no-random-fail.py @@ -37,7 +37,7 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", "telemetryReporting": "0", - } + } def init(self, path): self.__init__() @@ -70,14 +70,14 @@ class TDSimClient: cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -143,11 +143,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -196,7 +196,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self): + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -204,23 +204,22 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 6590f1e204..6edd5f1f6a 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -37,7 +37,7 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", "telemetryReporting": "0", - } + } def init(self, path): self.__init__() @@ -70,14 +70,14 @@ class TDSimClient: cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: tdLog.exit(cmd) - - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" cmd = "rm -rf " + self.cfgDir if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -143,11 +143,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -196,7 +196,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self): + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -204,23 +204,22 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" + tdLog.info("taosd found: %s" % binPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 427cfe349e..aa3e22ee7a 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -65,9 +65,11 @@ class TDSimClient: cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) - def os_string(self,path): - os_path = path.replace("/",os.sep) + + def os_string(self, path): + os_path = path.replace("/", os.sep) return os_path + def deploy(self): self.logDir = self.os_string("%s/sim/psim/log" % (self.path)) self.cfgDir = self.os_string("%s/sim/psim/cfg" % (self.path)) @@ -76,11 +78,11 @@ class TDSimClient: # cmd = "rm -rf " + self.logDir # if os.system(cmd) != 0: # tdLog.exit(cmd) - if os.path.exists(self.logDir): + if os.path.exists(self.logDir): try: shutil.rmtree(self.logDir) - except: - tdLog.exit("del %s failed"%self.logDir) + except BaseException: + tdLog.exit("del %s failed" % self.logDir) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: # tdLog.exit(cmd) @@ -88,11 +90,11 @@ class TDSimClient: # cmd = "rm -rf " + self.cfgDir # if os.system(cmd) != 0: # tdLog.exit(cmd) - if os.path.exists(self.cfgDir): + if os.path.exists(self.cfgDir): try: shutil.rmtree(self.cfgDir) - except: - tdLog.exit("del %s failed"%self.cfgDir) + except BaseException: + tdLog.exit("del %s failed" % self.cfgDir) # cmd = "mkdir -p " + self.cfgDir # if os.system(cmd) != 0: # tdLog.exit(cmd) @@ -102,8 +104,8 @@ class TDSimClient: # tdLog.exit(cmd) try: pathlib.Path(self.cfgPath).touch() - except: - tdLog.exit("create %s failed"%self.cfgPath) + except BaseException: + tdLog.exit("create %s failed" % self.cfgPath) if self.testCluster: self.cfg("masterIp", "192.168.0.1") self.cfg("secondIp", "192.168.0.2") @@ -123,36 +125,36 @@ class TDDnode: self.testCluster = False self.valgrind = 0 self.cfgDict = { - "numOfLogLines":"100000000", - "mnodeEqualVnodeNum":"0", - "walLevel":"2", - "fsync":"1000", - "statusInterval":"1", - "numOfMnodes":"3", - "numOfThreadsPerCore":"2.0", - "monitor":"0", - "maxVnodeConnections":"30000", - "maxMgmtConnections":"30000", - "maxMeterConnections":"30000", - "maxShellConns":"30000", - "locale":"en_US.UTF-8", - "charset":"UTF-8", - "asyncLog":"0", - "anyIp":"0", - "telemetryReporting":"0", - "dDebugFlag":"135", - "tsdbDebugFlag":"135", - "mDebugFlag":"135", - "sdbDebugFlag":"135", - "rpcDebugFlag":"135", - "tmrDebugFlag":"131", - "cDebugFlag":"135", - "httpDebugFlag":"135", - "monitorDebugFlag":"135", - "udebugFlag":"135", - "jnidebugFlag":"135", - "qdebugFlag":"135", - "maxSQLLength":"1048576", + "numOfLogLines": "100000000", + "mnodeEqualVnodeNum": "0", + "walLevel": "2", + "fsync": "1000", + "statusInterval": "1", + "numOfMnodes": "3", + "numOfThreadsPerCore": "2.0", + "monitor": "0", + "maxVnodeConnections": "30000", + "maxMgmtConnections": "30000", + "maxMeterConnections": "30000", + "maxShellConns": "30000", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "anyIp": "0", + "telemetryReporting": "0", + "dDebugFlag": "135", + "tsdbDebugFlag": "135", + "mDebugFlag": "135", + "sdbDebugFlag": "135", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "httpDebugFlag": "135", + "monitorDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + "maxSQLLength": "1048576", "enableCoreFile": "1", } @@ -200,11 +202,11 @@ class TDDnode: if os.system(cmd) != 0: tdLog.exit(cmd) - os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.dataDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.logDir, exist_ok=True) # like "mkdir -p" - os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" + os.makedirs(self.cfgDir, exist_ok=True) # like "mkdir -p" cmd = "touch " + self.cfgPath if os.system(cmd) != 0: @@ -227,16 +229,16 @@ class TDDnode: isFirstDir = 1 if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) - for key,value in updatecfgDict[0][0].items(): - if value == 'dataDir' : + for key, value in updatecfgDict[0][0].items(): + if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') - self.cfg(value,key) + self.cfg(value, key) isFirstDir = 0 else: - self.cfg(value,key) + self.cfg(value, key) else: - self.addExtraCfg(key,value) + self.addExtraCfg(key, value) for key, value in self.cfgDict.items(): self.cfg(key, value) @@ -245,8 +247,7 @@ class TDDnode: "dnode:%d is deployed and configured by %s" % (self.index, self.cfgPath)) - def getBuildPath(self, tool="taosd"): - buildPath = "" + def getPath(self, tool="taosd"): selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -254,24 +255,28 @@ class TDDnode: else: projPath = selfPath[:selfPath.find("tests")] + paths = [] for root, dirs, files in os.walk(projPath): if ((tool) in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + paths.append(os.path.join(root, tool)) break - return buildPath + return paths[0] def start(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) + tdLog.info("taosd found: %s" % binPath) - binPath = buildPath + "/build/bin/taosd" - taosadapterBinPath = buildPath + "/build/bin/taosadapter" + taosadapterBinPath = self.getPath("taosadapter") + if (taosadapterBinPath == ""): + tdLog.exit("taosAdapter not found!") + else: + tdLog.info("taosAdapter found: %s" % taosadapterBinPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -288,7 +293,7 @@ class TDDnode: print(cmd) taosadapterCmd = "nohup %s --opentsdb_telnet.enable=true > /dev/null 2>&1 & " % ( - taosadapterBinPath) + taosadapterBinPath) tdLog.info(taosadapterCmd) if os.system(taosadapterCmd) != 0: tdLog.exit(taosadapterCmd) @@ -301,18 +306,22 @@ class TDDnode: if self.valgrind == 0: time.sleep(0.1) key = 'from offline to online' - bkey = bytes(key,encoding="utf8") + bkey = bytes(key, encoding="utf8") logFile = self.logDir + "/taosdlog.0" i = 0 while not os.path.exists(logFile): sleep(0.1) i += 1 - if i>50: + if i > 50: break - popen = subprocess.Popen('tail -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + popen = subprocess.Popen( + 'tail -f ' + logFile, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) pid = popen.pid # print('Popen.pid:' + str(pid)) - timeout = time.time() + 60*2 + timeout = time.time() + 60 * 2 while True: line = popen.stdout.readline().strip() if bkey in line: @@ -322,30 +331,35 @@ class TDDnode: tdLog.exit('wait too long for taosd start') tdLog.debug("the dnode:%d has been started." % (self.index)) else: - tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + tdLog.debug( + "wait 10 seconds for the dnode:%d to start." % + (self.index)) time.sleep(10) - # time.sleep(5) + def startWin(self): - buildPath = self.getBuildPath("taosd.exe") + binPath = self.getPath("taosd.exe") - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd.exe not found!") else: - tdLog.info("taosd.exe found in %s" % buildPath) + tdLog.info("taosd.exe found: %s" % bnPath) - binPath = buildPath + "/build/bin/taosd.exe" - taosadapterBinPath = buildPath + "/build/bin/taosadapter.exe" + taosadapterBinPath = self.getPath("taosadapter.exe") + if (taosadapterBinPath == ""): + tdLog.exit("taosAdapter.exe not found!") + else: + tdLog.info("taosAdapter.exe found in %s" % taosadapterBuildPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) cmd = "mintty -h never -w hide %s -c %s" % ( - binPath, self.cfgDir) + binPath, self.cfgDir) taosadapterCmd = "mintty -h never -w hide %s " % ( - taosadapterBinPath) + taosadapterBinPath) if os.system(taosadapterCmd) != 0: tdLog.exit(taosadapterCmd) @@ -357,18 +371,22 @@ class TDDnode: if self.valgrind == 0: time.sleep(0.1) key = 'from offline to online' - bkey = bytes(key,encoding="utf8") + bkey = bytes(key, encoding="utf8") logFile = self.logDir + "/taosdlog.0" i = 0 while not os.path.exists(logFile): sleep(0.1) i += 1 - if i>50: + if i > 50: break - popen = subprocess.Popen('tail -n +0 -f ' + logFile, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + popen = subprocess.Popen( + 'tail -n +0 -f ' + logFile, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True) pid = popen.pid # print('Popen.pid:' + str(pid)) - timeout = time.time() + 60*2 + timeout = time.time() + 60 * 2 while True: line = popen.stdout.readline().strip() if bkey in line: @@ -378,19 +396,24 @@ class TDDnode: tdLog.exit('wait too long for taosd start') tdLog.debug("the dnode:%d has been started." % (self.index)) else: - tdLog.debug("wait 10 seconds for the dnode:%d to start." % (self.index)) + tdLog.debug( + "wait 10 seconds for the dnode:%d to start." % + (self.index)) time.sleep(10) - + def startWithoutSleep(self): - buildPath = self.getBuildPath() + binPath = self.getPath() - if (buildPath == ""): + if (binPath == ""): tdLog.exit("taosd not found!") else: - tdLog.info("taosd found in %s" % buildPath) + tdLog.info("taosd found: %s" % binPath) - binPath = buildPath + "/build/bin/taosd" - taosadapterBinPath = buildPath + "/build/bin/taosadapter" + taosadapterBinPath = self.getPath("taosadapter") + if (taosadapterBinPath == ""): + tdLog.exit("taosAdapter not found!") + else: + tdLog.info("taosAdapter found: %s" % taosadapterBinPath) if self.deployed == 0: tdLog.exit("dnode:%d is not deployed" % (self.index)) @@ -420,14 +443,14 @@ class TDDnode: taosadapterPsCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % taosadapterToBeKilled taosadapterProcessID = subprocess.check_output( - taosadapterPsCmd, shell=True).decode("utf-8") + taosadapterPsCmd, shell=True).decode("utf-8") while(taosadapterProcessID): taosadapterKillCmd = "kill -INT %s > /dev/null 2>&1" % taosadapterProcessID os.system(taosadapterKillCmd) time.sleep(1) taosadapterProcessID = subprocess.check_output( - taosadapterPsCmd, shell=True).decode("utf-8") + taosadapterPsCmd, shell=True).decode("utf-8") if self.valgrind == 0: toBeKilled = "taosd" @@ -599,7 +622,7 @@ class TDDnodes: def startWin(self, index): self.check(index) self.dnodes[index - 1].startWin() - + def startWithoutSleep(self, index): self.check(index) self.dnodes[index - 1].startWithoutSleep() diff --git a/tests/system-test/5-taos-tools/basic.py b/tests/system-test/5-taos-tools/basic.py index a05cf39001..f2f273b9a7 100644 --- a/tests/system-test/5-taos-tools/basic.py +++ b/tests/system-test/5-taos-tools/basic.py @@ -23,33 +23,34 @@ import subprocess class TDTestCase: def caseDescription(self): ''' - case1: [TD-11977] start taosdump without taosd + case1: [TD-11977] start taosdump without taosd case1: [TD-11977] start taosBenchmark without taosd case1: [TD-11977] start taosAdaptor without taosd - ''' + ''' return - + def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) def run(self): - tdSql.prepare() - - tools = ["taosdump", "taosBenchmark", "taosAdaptor"] + tdSql.prepare() + + tools = ["taosdump", "taosBenchmark", "taosadapter"] tdDnodes.stop(1) - for tool in tools: - path = tdDnodes.dnodes[1].getBuildPath(tool) + for tool in tools: + path = tdDnodes.dnodes[1].getPath(tool) try: - path += "/build/bin/" print(f"{path}{tool}") if tool == "taosBenchmark": - os.system(f"{path}{tool} -y") + os.system(f"{path}{tool} -y") + elif tool == "taosadapter": + os.system(f"pkill -9 {tool}") else: os.system(f"{path}{tool}") - except: + except BaseException: pass def stop(self): -- GitLab