From c2113c68a9028b1ec60b376f976df7b7b06a0843 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 7 Apr 2022 13:36:33 +0800 Subject: [PATCH] [TD-14544]: taosdump data inspect (#11273) * [TD-14544]: taosdump data inspect for 2.4 * [TS-14544]: taosdump data inspect add test case for 2.4 * add inspect case to parallel test * adjust py module path --- src/kit/taos-tools | 2 +- .../taosdump/taosdumpTestInspect.py | 123 ++++++++++++++++++ tests/develop-test/fulltest-tools.sh | 3 +- tests/develop-test/test.py | 46 +++---- tests/parallel_test/cases.task | 1 + 5 files changed, 150 insertions(+), 25 deletions(-) create mode 100644 tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py diff --git a/src/kit/taos-tools b/src/kit/taos-tools index e8108de4dd..eb3a914682 160000 --- a/src/kit/taos-tools +++ b/src/kit/taos-tools @@ -1 +1 @@ -Subproject commit e8108de4dd12e82f7d5896282227da7557f8dac2 +Subproject commit eb3a9146828cb37da435d48c073d6e7a4f39e80a diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py new file mode 100644 index 0000000000..a3c7cf8691 --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-14544] taosdump data inspect + ''' + return + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.tmpdir = "tmp" + + def getPath(self, tool="taosdump"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + return "" + return paths[0] + + def run(self): + tdSql.prepare() + + tdSql.execute("drop database if exists db") + tdSql.execute("create database db days 11 keep 3649 blocks 8 ") + + tdSql.execute("use db") + tdSql.execute( + "create table st(ts timestamp, c1 INT) tags(ntag INT)") + tdSql.execute("create table t1 using st tags(1)") + tdSql.execute("insert into t1 values(1640000000000, 1)") + tdSql.execute("create table t2 using st tags(2147483647)") + tdSql.execute("insert into t2 values(1640000000000, 2147483647)") + tdSql.execute("create table t3 using st tags(-2147483647)") + tdSql.execute("insert into t3 values(1640000000000, -2147483647)") + tdSql.execute("create table t4 using st tags(NULL)") + tdSql.execute("insert into t4 values(1640000000000, NULL)") + +# sys.exit(1) + + binPath = self.getPath("taosdump") + if (binPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % binPath) + + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + else: + print("directory exists") + os.system("rm -rf %s" % self.tmpdir) + os.makedirs(self.tmpdir) + + os.system( + "%s --databases db -o %s -T 1" % + (binPath, self.tmpdir)) + +# sys.exit(1) + + taosdumpInspectCmd = "%s -I %s/*.avro* -s | grep 'Schema:'|wc -l" % ( + binPath, self.tmpdir) + schemaTimes = subprocess.check_output( + taosdumpInspectCmd, shell=True).decode("utf-8") + print("schema found times: %d" % int(schemaTimes)) + + if (int(schemaTimes) != 5): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected schema found times 5, actual %d" % + (caller.filename, caller.lineno, int(schemaTimes))) + + taosdumpInspectCmd = "%s -I %s/*.avro* | grep '=== Records:'|wc -l" % ( + binPath, self.tmpdir) + recordsTimes = subprocess.check_output( + taosdumpInspectCmd, shell=True).decode("utf-8") + print("records found times: %d" % int(recordsTimes)) + + if (int(recordsTimes) != 5): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected records found times 5, actual %d" % + (caller.filename, caller.lineno, int(recordsTimes))) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/develop-test/fulltest-tools.sh b/tests/develop-test/fulltest-tools.sh index 959d269ebf..22d28a1b07 100755 --- a/tests/develop-test/fulltest-tools.sh +++ b/tests/develop-test/fulltest-tools.sh @@ -11,6 +11,7 @@ python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py +python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestTypeInspect.py python3 ./test.py -f 5-taos-tools/taosbenchmark/limit_offset_json.py python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py @@ -22,5 +23,5 @@ python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_interlace.py python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_telnet_alltypes.py python3 ./test.py -f 5-taos-tools/taosbenchmark/subscripe_json.py python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py -python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py diff --git a/tests/develop-test/test.py b/tests/develop-test/test.py index b39b95c903..ab22cfb23a 100644 --- a/tests/develop-test/test.py +++ b/tests/develop-test/test.py @@ -13,22 +13,22 @@ # pip install src/connector/python/ # -*- coding: utf-8 -*- -import sys -import getopt -import subprocess -import time -from distutils.log import warn as printf from fabric2 import Connection +from distutils.log import warn as printf +import time +import subprocess +import getopt +import sys + sys.path.append("../pytest") from util.log import * from util.dnodes import * from util.cases import * - import taos if __name__ == "__main__": - + fileName = "all" deployPath = "" masterIp = "" @@ -55,7 +55,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-w taos on windows') sys.exit(0) - if key in ['-r', '--restart']: + if key in ['-r', '--restart']: restart = True if key in ['-f', '--file']: @@ -117,7 +117,7 @@ if __name__ == "__main__": time.sleep(2) tdLog.info('stop All dnodes') - + if masterIp == "": host = '127.0.0.1' else: @@ -129,11 +129,11 @@ if __name__ == "__main__": tdLog.info("Procedures for testing self-deployment") td_clinet = TDSimClient("C:\\TDengine") td_clinet.deploy() - remote_conn = Connection("root@%s"%host) + remote_conn = Connection("root@%s" % host) with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): remote_conn.run("python3 ./test.py") conn = taos.connect( - host="%s"%(host), + host="%s" % (host), config=td_clinet.cfgDir) tdCases.runOneWindows(conn, fileName) else: @@ -146,23 +146,21 @@ if __name__ == "__main__": try: if key_word in open(fileName).read(): is_test_framework = 1 - except: + except BaseException: pass if is_test_framework: moduleName = fileName.replace(".py", "").replace("/", ".") uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, ucase.updatecfgDict) + except BaseException: + tdDnodes.deploy(1, {}) else: pass - tdDnodes.deploy(1,{}) + tdDnodes.deploy(1, {}) tdDnodes.start(1) - - tdCases.logSql(logSql) if testCluster: @@ -179,18 +177,20 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneWindows(conn, fileName) + tdCases.runOneLinux(conn, fileName) if restart: if fileName == "all": tdLog.info("not need to query ") - else: + else: sp = fileName.rsplit(".", 1) if len(sp) == 2 and sp[1] == "py": tdDnodes.stopAll() tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - tdLog.info("Procedures for tdengine deployed in %s" % (host)) + time.sleep(1) + conn = taos.connect(host, config=tdDnodes.getSimCfgPath()) + tdLog.info( + "Procedures for tdengine deployed in %s" % + (host)) tdLog.info("query test after taosd restart") tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 66fbbdb1cc..f7b616ab1d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -749,3 +749,4 @@ 3,,develop-test,python3 ./test.py -f 1-insert/uppercase_in_stmt.py 3,,develop-test,python3 ./test.py -f 2-query/ts_shortcut.py 3,,develop-test,python3 ./test.py -f 2-query/TD-13414.py +3,,develop-test,python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestInspect.py -- GitLab