[TD-14544]<feature>: taosdump data inspect (#11273)

* [TD-14544]<feature>: taosdump data inspect

for 2.4

* [TS-14544]<feature>: taosdump data inspect

add test case

for 2.4

* add inspect case to parallel test

* adjust py module path

taos-tools @ eb3a9146

-Subproject commit e8108de4dd12e82f7d5896282227da7557f8dac2
+Subproject commit eb3a9146828cb37da435d48c073d6e7a4f39e80a

tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+import subprocess
+
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1<sdsang>: [TD-14544] taosdump data inspect
+        '''
+        return
+
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self.tmpdir = "tmp"
+
+    def getPath(self, tool="taosdump"):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        paths = []
+        for root, dirs, files in os.walk(projPath):
+            if ((tool) in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    paths.append(os.path.join(root, tool))
+                    break
+        if (len(paths) == 0):
+            return ""
+        return paths[0]
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("drop database if exists db")
+        tdSql.execute("create database db  days 11 keep 3649 blocks 8 ")
+
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table st(ts timestamp, c1 INT) tags(ntag INT)")
+        tdSql.execute("create table t1 using st tags(1)")
+        tdSql.execute("insert into t1 values(1640000000000, 1)")
+        tdSql.execute("create table t2 using st tags(2147483647)")
+        tdSql.execute("insert into t2 values(1640000000000, 2147483647)")
+        tdSql.execute("create table t3 using st tags(-2147483647)")
+        tdSql.execute("insert into t3 values(1640000000000, -2147483647)")
+        tdSql.execute("create table t4 using st tags(NULL)")
+        tdSql.execute("insert into t4 values(1640000000000, NULL)")
+
+#        sys.exit(1)
+
+        binPath = self.getPath("taosdump")
+        if (binPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % binPath)
+
+        if not os.path.exists(self.tmpdir):
+            os.makedirs(self.tmpdir)
+        else:
+            print("directory exists")
+            os.system("rm -rf %s" % self.tmpdir)
+            os.makedirs(self.tmpdir)
+
+        os.system(
+            "%s --databases db -o %s -T 1" %
+            (binPath, self.tmpdir))
+
+#        sys.exit(1)
+
+        taosdumpInspectCmd = "%s -I %s/*.avro* -s | grep 'Schema:'|wc -l" % (
+            binPath, self.tmpdir)
+        schemaTimes = subprocess.check_output(
+            taosdumpInspectCmd, shell=True).decode("utf-8")
+        print("schema found times: %d" % int(schemaTimes))
+
+        if (int(schemaTimes) != 5):
+            caller = inspect.getframeinfo(inspect.stack()[0][0])
+            tdLog.exit(
+                "%s(%d) failed: expected schema found times 5, actual %d" %
+                (caller.filename, caller.lineno, int(schemaTimes)))
+
+        taosdumpInspectCmd = "%s -I %s/*.avro* | grep '=== Records:'|wc -l" % (
+            binPath, self.tmpdir)
+        recordsTimes = subprocess.check_output(
+            taosdumpInspectCmd, shell=True).decode("utf-8")
+        print("records found times: %d" % int(recordsTimes))
+
+        if (int(recordsTimes) != 5):
+            caller = inspect.getframeinfo(inspect.stack()[0][0])
+            tdLog.exit(
+                "%s(%d) failed: expected records found times 5, actual %d" %
+                (caller.filename, caller.lineno, int(recordsTimes)))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/develop-test/fulltest-tools.sh

@@ -11,6 +11,7 @@ python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py
 python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py
 python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py
 python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py
+python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeInspect.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/limit_offset_json.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py
@@ -22,5 +23,5 @@ python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_interlace.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_telnet_alltypes.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/subscripe_json.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py
-python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py 
+python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py
 python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py

tests/develop-test/test.py

@@ -13,22 +13,22 @@
 # pip install src/connector/python/
 
 # -*- coding: utf-8 -*-
-import sys
-import getopt
-import subprocess
-import time
-from distutils.log import warn as printf
 from fabric2 import Connection
+from distutils.log import warn as printf
+import time
+import subprocess
+import getopt
+import sys
+
 sys.path.append("../pytest")
 from util.log import *
 from util.dnodes import *
 from util.cases import *
-
 import taos
 
 
 if __name__ == "__main__":
-    
+
     fileName = "all"
     deployPath = ""
     masterIp = ""
@@ -55,7 +55,7 @@ if __name__ == "__main__":
             tdLog.printNoPrefix('-w taos on windows')
             sys.exit(0)
 
-        if key in ['-r', '--restart']: 
+        if key in ['-r', '--restart']:
             restart = True
 
         if key in ['-f', '--file']:
@@ -117,7 +117,7 @@ if __name__ == "__main__":
             time.sleep(2)
 
         tdLog.info('stop All dnodes')
-    
+
     if masterIp == "":
         host = '127.0.0.1'
     else:
@@ -129,11 +129,11 @@ if __name__ == "__main__":
         tdLog.info("Procedures for testing self-deployment")
         td_clinet = TDSimClient("C:\\TDengine")
         td_clinet.deploy()
-        remote_conn = Connection("root@%s"%host)
+        remote_conn = Connection("root@%s" % host)
         with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
             remote_conn.run("python3 ./test.py")
         conn = taos.connect(
-            host="%s"%(host),
+            host="%s" % (host),
             config=td_clinet.cfgDir)
         tdCases.runOneWindows(conn, fileName)
     else:
@@ -146,23 +146,21 @@ if __name__ == "__main__":
         try:
             if key_word in open(fileName).read():
                 is_test_framework = 1
-        except:
+        except BaseException:
             pass
         if is_test_framework:
             moduleName = fileName.replace(".py", "").replace("/", ".")
             uModule = importlib.import_module(moduleName)
             try:
                 ucase = uModule.TDTestCase()
-                tdDnodes.deploy(1,ucase.updatecfgDict)
-            except :
-                tdDnodes.deploy(1,{})
+                tdDnodes.deploy(1, ucase.updatecfgDict)
+            except BaseException:
+                tdDnodes.deploy(1, {})
         else:
             pass
-            tdDnodes.deploy(1,{})
+            tdDnodes.deploy(1, {})
         tdDnodes.start(1)
 
-
-
         tdCases.logSql(logSql)
 
         if testCluster:
@@ -179,18 +177,20 @@ if __name__ == "__main__":
             if fileName == "all":
                 tdCases.runAllLinux(conn)
             else:
-                tdCases.runOneWindows(conn, fileName)
+                tdCases.runOneLinux(conn, fileName)
         if restart:
             if fileName == "all":
                 tdLog.info("not need to query ")
-            else:    
+            else:
                 sp = fileName.rsplit(".", 1)
                 if len(sp) == 2 and sp[1] == "py":
                     tdDnodes.stopAll()
                     tdDnodes.start(1)
-                    time.sleep(1)            
-                    conn = taos.connect( host, config=tdDnodes.getSimCfgPath())
-                    tdLog.info("Procedures for tdengine deployed in %s" % (host))
+                    time.sleep(1)
+                    conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
+                    tdLog.info(
+                        "Procedures for tdengine deployed in %s" %
+                        (host))
                     tdLog.info("query test after taosd restart")
                     tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
                 else:

tests/parallel_test/cases.task

@@ -749,3 +749,4 @@
 3,,develop-test,python3 ./test.py -f 1-insert/uppercase_in_stmt.py
 3,,develop-test,python3 ./test.py -f 2-query/ts_shortcut.py
 3,,develop-test,python3 ./test.py -f 2-query/TD-13414.py
+3,,develop-test,python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestInspect.py