feat: taosbenchmark support partial col 9284147 for 2.6 (#17457)

* feat: taosbenchmark support partial col 9284147 for 2.6

* fix: free cfgs

* fix: better handle command line and json memory

* test: fix two taosbenchmark test cases

* feat: update taos-tools cc973e0

* test: fix 5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py

taos-tools @ cc973e0c

-Subproject commit 217a267b5365b7a4089eb6ee2d4b2ceedfc4df66
+Subproject commit cc973e0c2999a739fa2d06c25ab5adb2a7537c8f

tests/pytest/query/query1970YearsAf.py

@@ -25,8 +25,8 @@ from util.cases import *
 from util.dnodes import *
 from util.dnodes import TDDnode
 
-class TDTestCase:
 
+class TDTestCase:
     def __init__(self):
         self.path = ""
 
@@ -68,7 +68,7 @@ class TDTestCase:
             "cachelast": 0,
             "quorum": 1,
             "fsync": 3000,
-            "update": 0
+            "update": 0,
         }
 
         # set stable schema
@@ -99,8 +99,7 @@ class TDTestCase:
                 {"type": "TINYINT", "count": 2},
                 {"type": "BOOL", "count": 2},
                 {"type": "NCHAR", "len": 3, "count": 1},
-                {"type": "BINARY", "len": 8, "count": 1}
-
+                {"type": "BINARY", "len": 8, "count": 1},
             ],
             "tags": [
                 {"type": "INT", "count": 2},
@@ -111,17 +110,14 @@ class TDTestCase:
                 {"type": "TINYINT", "count": 2},
                 {"type": "BOOL", "count": 2},
                 {"type": "NCHAR", "len": 3, "count": 1},
-                {"type": "BINARY", "len": 8, "count": 1}
-            ]
+                {"type": "BINARY", "len": 8, "count": 1},
+            ],
         }
 
         # create different stables like stable1 and add to list super_tables
         super_tables = []
         super_tables.append(stable1)
-        database = {
-            "dbinfo": dbinfo,
-            "super_tables": super_tables
-        }
+        database = {"dbinfo": dbinfo, "super_tables": super_tables}
 
         cfgdir = self.getCfgDir()
         create_table = {
@@ -137,33 +133,60 @@ class TDTestCase:
             "confirm_parameter_prompt": "no",
             "insert_interval": 0,
             "num_of_records_per_req": 100,
-            "databases": [database]
+            "databases": [database],
         }
         return create_table
 
+    def getPath(self, tool="taosBenchmark"):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if "community" in selfPath:
+            projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
+        elif "/tests/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tests/")]
+        else:
+            tdLog.exit("path: %s is not supported" % selfPath)
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if (tool) in files:
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if "packaging" not in rootRealPath:
+                    paths.append(os.path.join(root, tool))
+                    break
+        if len(paths) == 0:
+            return ""
+        return paths[0]
+
     def createinsertfile(self):
         create_table = self.creatcfg()
         date = datetime.datetime.now().strftime("%Y%m%d%H%M")
         file_create_table = f"/tmp/insert_{date}.json"
 
-        with open(file_create_table, 'w') as f:
+        with open(file_create_table, "w") as f:
             json.dump(create_table, f)
         return file_create_table
 
     def inserttable(self, filepath):
-        create_table_cmd = f"taosdemo -f {filepath}  > /dev/null 2>&1"
+        binPath = self.getPath("taosBenchmark")
+        if binPath == "":
+            tdLog.exit("taosBenchmark not found!")
+        else:
+            tdLog.info("taosBenchmark found in %s" % binPath)
+
+        create_table_cmd = "%s -f %s  > /dev/null 2>&1" % (binPath, filepath)
         _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8")
 
     def sqlsquery(self):
         # stable query
-        tdSql.query(
-            "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(43200)
 
-        tdSql.query(
-            "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(6800)
 
         tdSql.query(
@@ -172,14 +195,10 @@ class TDTestCase:
         tdSql.checkRows(3590)
 
         # child-tables query
-        tdSql.query(
-            "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from t0 where t0.ts < '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(4320)
 
-        tdSql.query(
-            "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(680)
 
         tdSql.query(
@@ -192,9 +211,7 @@ class TDTestCase:
         )
         tdSql.checkRows(680)
 
-        tdSql.query(
-            "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
-        )
+        tdSql.query("select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' ")
         tdSql.checkRows(679)
 
         tdSql.query(
@@ -203,9 +220,7 @@ class TDTestCase:
         tdSql.checkRows(43200)
 
         # query with timestamp in 'where ...'
-        tdSql.query(
-            "select * from stb2 where stb2.ts > -28800000 "
-        )
+        tdSql.query("select * from stb2 where stb2.ts > -28800000 ")
         tdSql.checkRows(6790)
 
         tdSql.query(
@@ -219,14 +234,16 @@ class TDTestCase:
         tdSql.checkRows(3590)
 
     def run(self):
-        s = 'reset query cache'
+        s = "reset query cache"
         tdSql.execute(s)
-        s = 'create database if not exists db'
+        s = "create database if not exists db"
         tdSql.execute(s)
-        s = 'use db'
+        s = "use db"
         tdSql.execute(s)
 
-        tdLog.info("==========step1:create table stable and child table,then insert data automatically")
+        tdLog.info(
+            "==========step1:create table stable and child table,then insert data automatically"
+        )
         insertfile = self.createinsertfile()
         self.inserttable(insertfile)
 

tests/system-test/5-taos-tools/taosbenchmark/taosdemoTestInsertWithJson.py

@@ -24,36 +24,44 @@ class TDTestCase:
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 
-    def getBuildPath(self):
+    def getPath(self, tool="taosBenchmark"):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
-        if ("community" in selfPath):
-            projPath = selfPath[:selfPath.find("community")]
+        if "community" in selfPath:
+            projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
+        elif "/tests/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tests/")]
         else:
-            projPath = selfPath[:selfPath.find("tests")]
+            tdLog.exit("path: %s is not supported" % selfPath)
 
+        paths = []
         for root, dirs, files in os.walk(projPath):
-            if ("taosd" in files):
+            if (tool) in files:
                 rootRealPath = os.path.dirname(os.path.realpath(root))
-                if ("packaging" not in rootRealPath):
-                    buildPath = root[:len(root)-len("/build/bin")]
+                if "packaging" not in rootRealPath:
+                    paths.append(os.path.join(root, tool))
                     break
-        return buildPath
+        if len(paths) == 0:
+            return ""
+        return paths[0]
 
     def run(self):
-        buildPath = self.getBuildPath()
-        if (buildPath == ""):
-            tdLog.exit("taosd not found!")
+        binPath = self.getPath()
+        if binPath == "":
+            tdLog.exit("taosBenchmark not found!")
         else:
-            tdLog.info("taosd found in %s" % buildPath)
-        binPath = buildPath+ "/build/bin/"
+            tdLog.info("taosBenchmark found in %s" % binPath)
 
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert*_res.txt*")
-        os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename )         
+        os.system("rm -rf 5-taos-tools/taosbenchmark/%s.sql" % testcaseFilename)
 
         # insert: create one  or mutiple tables per sql and insert multiple rows per sql
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tnt1r.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 11)
@@ -69,7 +77,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 2000)
 
         # restful connector insert data
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insertRestful.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 10)
@@ -84,19 +92,19 @@ class TDTestCase:
         tdSql.query("select count(*) from stb1")
         tdSql.checkData(0, 0, 200)
 
-        # default values json files 
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath)
+        # default values json files
+        tdSql.execute("drop database if exists db")
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-default.json -y " % binPath)
         tdSql.query("show databases;")
         for i in range(tdSql.queryRows):
-            if tdSql.queryResult[i][0] == 'db':
-                tdSql.checkData(i, 2, 100) 
-                tdSql.checkData(i, 4, 1)     
-                tdSql.checkData(i, 6, 10)       
-                tdSql.checkData(i, 16, 'ms')           
-    
+            if tdSql.queryResult[i][0] == "db":
+                tdSql.checkData(i, 2, 100)
+                tdSql.checkData(i, 4, 1)
+                tdSql.checkData(i, 6, 10)
+                tdSql.checkData(i, 16, "ms")
+
         # insert: create  mutiple tables per sql and insert one rows per sql .
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath)
+        os.system("%s -f 5-taos-tools/taosbenchmark/insert-1s1tntmr.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 10)
@@ -113,7 +121,9 @@ class TDTestCase:
 
         # insert: using parament "insert_interval to controls spped  of insert.
         # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。
-        os.system("%staosBenchmark -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath)
+        os.system(
+            "%s -f 5-taos-tools/taosbenchmark/insert-interval-speed.json -y" % binPath
+        )
         tdSql.execute("use db")
         tdSql.query("show stables")
         tdSql.checkData(0, 4, 10)
@@ -131,11 +141,6 @@ class TDTestCase:
         # rm useless files
         os.system("rm -rf ./insert*_res.txt*")
 
-
-        
-        
-        
-
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)