From b8bed3f14bf5d705b523f6285aca1be1b50156ec Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 7 Apr 2022 13:26:24 +0800 Subject: [PATCH] [TD-14544]: taosdump data inspect (#11272) * [TD-14544]: taosdump data inspect * [TS-14544]: taosdump data inspect add test case * add inspect case to parallel test --- src/kit/taos-tools | 2 +- .../taosdump/taosdumpTestInspect.py | 123 ++++++++++++++++++ tests/parallel_test/cases.task | 1 + tests/pytest/tools/taosdemoTest.py | 2 + tests/pytest/tools/taosdemoTestInterlace.py | 2 + tests/pytest/tools/taosdemoTestLimitOffset.py | 2 + tests/pytest/tools/taosdemoTestQuery.py | 2 + tests/pytest/tools/taosdemoTestSampleData.py | 2 + tests/pytest/tools/taosdemoTestTblAlt.py | 2 + tests/pytest/tools/taosdemoTestWithJson.py | 2 + .../pytest/tools/taosdemoTestWithoutMetric.py | 2 + tests/pytest/tools/taosdemoTestdatatype.py | 2 + tests/pytest/tools/taosdumpTest.py | 2 + tests/pytest/tools/taosdumpTest2.py | 2 + tests/pytest/tools/taosdumpTest3.py | 2 + tests/pytest/tools/taosdumpTestBenchmark.py | 2 + tests/pytest/tools/taosdumpTestNanoSupport.py | 2 + 17 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py diff --git a/src/kit/taos-tools b/src/kit/taos-tools index e8108de4dd..eb3a914682 160000 --- a/src/kit/taos-tools +++ b/src/kit/taos-tools @@ -1 +1 @@ -Subproject commit e8108de4dd12e82f7d5896282227da7557f8dac2 +Subproject commit eb3a9146828cb37da435d48c073d6e7a4f39e80a diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py new file mode 100644 index 0000000000..bb70b1caea --- /dev/null +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -0,0 +1,123 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-14544] taosdump data inspect + ''' + return + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.tmpdir = "tmp" + + def getPath(self, tool="taosdump"): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + paths = [] + for root, dirs, files in os.walk(projPath): + if ((tool) in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + paths.append(os.path.join(root, tool)) + break + if (len(paths) == 0): + return "" + return paths[0] + + def run(self): + tdSql.prepare() + + tdSql.execute("drop database if exists db") + tdSql.execute("create database db days 11 keep 3649 blocks 8 ") + + tdSql.execute("use db") + tdSql.execute( + "create table st(ts timestamp, c1 INT) tags(ntag INT)") + tdSql.execute("create table t1 using st tags(1)") + tdSql.execute("insert into t1 values(1640000000000, 1)") + tdSql.execute("create table t2 using st tags(2147483647)") + tdSql.execute("insert into t2 values(1640000000000, 2147483647)") + tdSql.execute("create table t3 using st tags(-2147483647)") + tdSql.execute("insert into t3 values(1640000000000, -2147483647)") + tdSql.execute("create table t4 using st tags(NULL)") + tdSql.execute("insert into t4 values(1640000000000, NULL)") + +# sys.exit(1) + + binPath = self.getPath("taosdump") + if (binPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % binPath) + + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + else: + print("directory exists") + os.system("rm -rf %s" % self.tmpdir) + os.makedirs(self.tmpdir) + + os.system( + "%s --databases db -o %s -T 1" % + (binPath, self.tmpdir)) + +# sys.exit(1) + + taosdumpInspectCmd = "%s -I %s/*.avro* -s | grep 'Schema:'|wc -l" % ( + binPath, self.tmpdir) + schemaTimes = subprocess.check_output( + taosdumpInspectCmd, shell=True).decode("utf-8") + print("schema found times: %d" % int(schemaTimes)) + + if (int(schemaTimes) != 5): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected schema found times 5, actual %d" % + (caller.filename, caller.lineno, int(schemaTimes))) + + taosdumpInspectCmd = "%s -I %s/*.avro* | grep '=== Records:'|wc -l" % ( + binPath, self.tmpdir) + recordsTimes = subprocess.check_output( + taosdumpInspectCmd, shell=True).decode("utf-8") + print("records found times: %d" % int(recordsTimes)) + + if (int(recordsTimes) != 5): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected records found times 5, actual %d" % + (caller.filename, caller.lineno, int(recordsTimes))) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 3b3ed8505b..490a0a17aa 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -807,3 +807,4 @@ 2,,develop-test,python3 ./test.py -f 2-query/function_hll.py 1,,develop-test,python3 ./test.py -f 2-query/function_state.py 1,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +3,,develop-test,python3 ./test.py -f 5-taos-tools/taosdump/taosdumpTestInspect.py diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 3de3961faf..801cfe680d 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -42,6 +42,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestInterlace.py b/tests/pytest/tools/taosdemoTestInterlace.py index 8c2f0bd0ee..328ef73ae8 100644 --- a/tests/pytest/tools/taosdemoTestInterlace.py +++ b/tests/pytest/tools/taosdemoTestInterlace.py @@ -40,6 +40,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestLimitOffset.py b/tests/pytest/tools/taosdemoTestLimitOffset.py index b5aa8d75b9..150a5fed47 100644 --- a/tests/pytest/tools/taosdemoTestLimitOffset.py +++ b/tests/pytest/tools/taosdemoTestLimitOffset.py @@ -42,6 +42,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestQuery.py b/tests/pytest/tools/taosdemoTestQuery.py index 4d30835ec0..a97230d222 100644 --- a/tests/pytest/tools/taosdemoTestQuery.py +++ b/tests/pytest/tools/taosdemoTestQuery.py @@ -44,6 +44,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestSampleData.py b/tests/pytest/tools/taosdemoTestSampleData.py index 5f1d6a28bf..0800f3decf 100644 --- a/tests/pytest/tools/taosdemoTestSampleData.py +++ b/tests/pytest/tools/taosdemoTestSampleData.py @@ -42,6 +42,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestTblAlt.py b/tests/pytest/tools/taosdemoTestTblAlt.py index 6abdaa7a80..46a2de1f97 100644 --- a/tests/pytest/tools/taosdemoTestTblAlt.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -44,6 +44,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def insertDataAndAlterTable(self, threadID): diff --git a/tests/pytest/tools/taosdemoTestWithJson.py b/tests/pytest/tools/taosdemoTestWithJson.py index 7bf076ea4b..87b99ef31c 100644 --- a/tests/pytest/tools/taosdemoTestWithJson.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -39,6 +39,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestWithoutMetric.py b/tests/pytest/tools/taosdemoTestWithoutMetric.py index c41656c508..0ab49ada6b 100644 --- a/tests/pytest/tools/taosdemoTestWithoutMetric.py +++ b/tests/pytest/tools/taosdemoTestWithoutMetric.py @@ -42,6 +42,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdemoTestdatatype.py b/tests/pytest/tools/taosdemoTestdatatype.py index c27d65ee1c..c221206365 100644 --- a/tests/pytest/tools/taosdemoTestdatatype.py +++ b/tests/pytest/tools/taosdemoTestdatatype.py @@ -42,6 +42,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index a92c242119..2155556776 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -50,6 +50,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index 53c5109830..952f68b477 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -47,6 +47,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def generateString(self, length): diff --git a/tests/pytest/tools/taosdumpTest3.py b/tests/pytest/tools/taosdumpTest3.py index e91540fdd1..3994ad0323 100644 --- a/tests/pytest/tools/taosdumpTest3.py +++ b/tests/pytest/tools/taosdumpTest3.py @@ -50,6 +50,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def run(self): diff --git a/tests/pytest/tools/taosdumpTestBenchmark.py b/tests/pytest/tools/taosdumpTestBenchmark.py index 75238b7f73..d3ac7347b3 100644 --- a/tests/pytest/tools/taosdumpTestBenchmark.py +++ b/tests/pytest/tools/taosdumpTestBenchmark.py @@ -50,6 +50,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def insert_data(self, tbname, ts_start, count): diff --git a/tests/pytest/tools/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdumpTestNanoSupport.py index cbcf57db0a..146beb90e5 100644 --- a/tests/pytest/tools/taosdumpTestNanoSupport.py +++ b/tests/pytest/tools/taosdumpTestNanoSupport.py @@ -50,6 +50,8 @@ class TDTestCase: if ("packaging" not in rootRealPath): paths.append(os.path.join(root, tool)) break + if (len(paths) == 0): + return "" return paths[0] def createdb(self, precision="ns"): -- GitLab