fix: taosbenchmark data gen refactor (#20059)

* fix: taosbenchmark data gen refactor

* fix: update taos-tools e22e5e2

* fix: update taos-tools cccc353

* fix: update taos-tools 10a211f

* fix: update taos-tools e54a926

* fix: update taos-tools 340b0f4

* fix: update taos-tools 6afed7c

* fix: update taos-tools cc6db40

* fix: update taos-tools 634399d

* fix: update taos-tools e0104dc

* fix: update taos-tools bc11ff3

* fix: update taos-tools b23e170

* fix: update taos-tools a62f774

* fix: clean up and print

* fix: update taos-tools a880c81

* test: update 5-taos-tools/taosdump/taosdumpTestInspect.py for 3.0

* fix: add sml_tags_json_array to accelerate sml json test

* fix: taosbenchmark json data generating refine 611dc09

* fix: update taos-tools ad2f7c8

* fix: coverity scan issues

* fix: udpate taos-tools d259385

* fix: update taos-tools be929ab

* fix: update taos-tools 63e63d6

* fix: update taos-tools 2a38de1

* fix: update taos-tools 0d4c001

* fix: update taos-tools dce7de5

* fix: update taos-tools 82b96df

* fix: update taos-tools 401cf6a

* fix: update taos-tools 6f692b0:wq

* fix: update taos-tools 2942ca0

* fix: update taos-tools 61cbfd2

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 634399d
+        GIT_TAG 61cbfd2
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py

@@ -11,24 +11,20 @@
 
 # -*- coding: utf-8 -*-
 
-import sys
 import os
 from util.log import *
 from util.cases import *
 from util.sql import *
 from util.dnodes import *
-import subprocess
 
 
 class TDTestCase:
     def caseDescription(self):
-        '''
+        """
         case1<sdsang>: [TD-14544] taosdump data inspect
-        '''
-        return
+        """
 
     def init(self, conn, logSql, replicaVar=1):
-        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
         self.tmpdir = "tmp"
@@ -36,44 +32,56 @@ class TDTestCase:
     def getPath(self, tool="taosdump"):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
-        if ("community" in selfPath):
-            projPath = selfPath[:selfPath.find("community")]
+        if "community" in selfPath:
+            projPath = selfPath[: selfPath.find("community")]
+        elif "src" in selfPath:
+            projPath = selfPath[: selfPath.find("src")]
+        elif "/tools/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tools/")]
+        elif "/tests/" in selfPath:
+            projPath = selfPath[: selfPath.find("/tests/")]
         else:
-            projPath = selfPath[:selfPath.find("tests")]
+            tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath))
+            projPath = "/usr/local/taos/bin"
 
         paths = []
-        for root, dirs, files in os.walk(projPath):
-            if ((tool) in files):
+        for root, dummy, files in os.walk(projPath):
+            if (tool) in files:
                 rootRealPath = os.path.dirname(os.path.realpath(root))
-                if ("packaging" not in rootRealPath):
+                if "packaging" not in rootRealPath:
                     paths.append(os.path.join(root, tool))
                     break
-        if (len(paths) == 0):
+        if len(paths) == 0:
             return ""
         return paths[0]
 
     def run(self):
-        tdSql.prepare(replica=f"{self.replicaVar}")
+        tdSql.prepare()
 
         tdSql.execute("drop database if exists db")
-        tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")
+        tdSql.execute("create database db  keep 3649 ")
 
         tdSql.execute("use db")
         tdSql.execute(
-            "create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)")
+            "create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)"
+        )
         tdSql.execute(
-            "create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)")
+            "create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)"
+        )
         tdSql.execute(
-            "insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)")
+            "insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)"
+        )
         tdSql.execute(
-            "create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)")
+            "create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
+        )
         tdSql.execute(
-            "insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)")
+            "insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)"
+        )
 
-#        sys.exit(1)
+        #        sys.exit(1)
 
-        binPath = self.getPath("taosdump")
-        if (binPath == ""):
+        binPath = self.getPath()
+        if binPath == "":
             tdLog.exit("taosdump not found!")
         else:
             tdLog.info("taosdump found in %s" % binPath)
@@ -85,35 +93,73 @@ class TDTestCase:
             os.system("rm -rf %s" % self.tmpdir)
             os.makedirs(self.tmpdir)
 
-        os.system(
-            "%s --databases db -o %s -T 1" %
-            (binPath, self.tmpdir))
+        os.system("%s --databases db -o %s -T 1" % (binPath, self.tmpdir))
 
-#        sys.exit(1)
+        #        sys.exit(1)
 
-        taosdumpInspectCmd = "%s -I %s/*.avro* -s | grep 'Schema:'|wc -l" % (
-            binPath, self.tmpdir)
-        schemaTimes = subprocess.check_output(
-            taosdumpInspectCmd, shell=True).decode("utf-8")
+        taosdumpInspectCmd = "%s -I %s/taosdump.*/*.avro* -s | grep 'Schema:'|wc -l" % (
+            binPath,
+            self.tmpdir,
+        )
+        schemaTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode(
+            "utf-8"
+        )
         print("schema found times: %d" % int(schemaTimes))
 
-        if (int(schemaTimes) != 3):
+        if int(schemaTimes) != 1:
             caller = inspect.getframeinfo(inspect.stack()[0][0])
             tdLog.exit(
-                "%s(%d) failed: expected schema found times 3, actual %d" %
-                (caller.filename, caller.lineno, int(schemaTimes)))
+                "%s(%d) failed: expected schema found times 1, actual %d"
+                % (caller.filename, caller.lineno, int(schemaTimes))
+            )
+
+        taosdumpInspectCmd = (
+            "%s -I %s/taosdump*/data*/*.avro* -s | grep 'Schema:'|wc -l"
+            % (binPath, self.tmpdir)
+        )
+        schemaTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode(
+            "utf-8"
+        )
+        print("schema found times: %d" % int(schemaTimes))
 
-        taosdumpInspectCmd = "%s -I %s/*.avro* | grep '=== Records:'|wc -l" % (
-            binPath, self.tmpdir)
-        recordsTimes = subprocess.check_output(
-            taosdumpInspectCmd, shell=True).decode("utf-8")
+        if int(schemaTimes) != 2:
+            caller = inspect.getframeinfo(inspect.stack()[0][0])
+            tdLog.exit(
+                "%s(%d) failed: expected schema found times 2, actual %d"
+                % (caller.filename, caller.lineno, int(schemaTimes))
+            )
+
+        taosdumpInspectCmd = (
+            "%s -I %s/taosdump*/*.avro* | grep '=== Records:'|wc -l"
+            % (binPath, self.tmpdir)
+        )
+        recordsTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode(
+            "utf-8"
+        )
+        print("records found times: %d" % int(recordsTimes))
+
+        if int(recordsTimes) != 1:
+            caller = inspect.getframeinfo(inspect.stack()[0][0])
+            tdLog.exit(
+                "%s(%d) failed: expected records found times 1, actual %d"
+                % (caller.filename, caller.lineno, int(recordsTimes))
+            )
+
+        taosdumpInspectCmd = (
+            "%s -I %s/taosdump*/data*/*.avro* | grep '=== Records:'|wc -l"
+            % (binPath, self.tmpdir)
+        )
+        recordsTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode(
+            "utf-8"
+        )
         print("records found times: %d" % int(recordsTimes))
 
-        if (int(recordsTimes) != 3):
+        if int(recordsTimes) != 2:
             caller = inspect.getframeinfo(inspect.stack()[0][0])
             tdLog.exit(
-                "%s(%d) failed: expected records found times 3, actual %d" %
-                (caller.filename, caller.lineno, int(recordsTimes)))
+                "%s(%d) failed: expected records found times 2, actual %d"
+                % (caller.filename, caller.lineno, int(recordsTimes))
+            )
 
     def stop(self):
         tdSql.close()