Feature/sangshuduo/td 3317 taosdemo interlace (#5476)

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

check offset 0.

* [TD-3316] <fix>: add testcase for taosdemo limit and offset.

fix sample file import bug.

* [TD-3316] <fix>: add test case for limit and offset. fix sample data issue.

* [TD-3327] <fix>: fix taosdemo segfault when import data from sample data file.

* [TD-3317] <feature>: make taosdemo support interlace mode.

json parameter rows_per_tbl support.

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode.

refactor

* [TD-3317] <feature>: support interlace mode insertion.

refactor.

* [TD-3317] <feature>: support interlace mode insertion.

change json file.

* [TD-3317] <feature>: support interlace mode insertion.

fix multithread create table regression.

* [TD-3317] <feature>: support interlace mode insertion.

working but not perfect.

* [TD-3317] <feature>: support interlace mode insertion.

rename lowaTest with taosdemoTestWithJson
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

src/kit/taosdemo/insert-interlace.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 5000,
+    "rows_per_tbl": 50,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 10,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 200,
+            "multi_thread_write_one_tbl": "no",
+            "rows_per_tbl": 20,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}

tests/pytest/fulltest.sh

@@ -231,12 +231,13 @@ python3 test.py -f query/queryInterval.py
 python3 test.py -f query/queryFillTest.py
 
 # tools
-python3 test.py -f tools/lowaTest.py
 python3 test.py -f tools/taosdemoTest.py
 python3 test.py -f tools/taosdemoTestWithoutMetric.py
+python3 test.py -f tools/taosdemoTestWithJson.py
 python3 test.py -f tools/taosdemoTestLimitOffset.py
 python3 test.py -f tools/taosdumpTest.py
 python3 test.py -f tools/taosdemoTest2.py
+python3 test.py -f tools/taosdemoTestSampleData.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py
@@ -291,6 +292,5 @@ python3 ./test.py -f insert/boundary2.py
 python3 ./test.py -f alter/alter_debugFlag.py
 python3 ./test.py -f query/queryBetweenAnd.py
 python3 ./test.py -f tag_lite/alter_tag.py
-python3 test.py -f tools/taosdemoTestSampleData.py
 
-#======================p4-end===============
+#======================p4-end===============
\ No newline at end of file

tests/pytest/tools/insert-interlace.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 5000,
+    "rows_per_tbl": 50,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 9,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 250,
+            "multi_thread_write_one_tbl": "no",
+            "rows_per_tbl": 80,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}],
+            "tags": [{"type": "INT", "count":1}]
+        }]
+    }]
+}

tests/pytest/tools/insert-tblimit-tboffset0.json

@@ -44,7 +44,6 @@
             "childtable_offset": 0,
             "multi_thread_write_one_tbl": "no",
             "number_of_tbl_in_one_sql": 0,
-            "rows_per_tbl": 100,
             "max_sql_len": 1024000,
             "disorder_ratio": 0,
             "disorder_range": 1000,

tests/pytest/tools/insert-tblimit1-tboffset.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 1000,
+            "childtable_limit": 1,
+            "childtable_offset": 50,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}],
+            "tags": [{"type": "TINYINT", "count":1}]
+        }]
+    }]
+}

tests/pytest/tools/insert.json

 {
     "filetype":"insert",
     "cfgdir": "/etc/taos",
-	"host": "127.0.0.1",
-	"port": 6030,
-	"user": "root",
-	"password": "taosdata",
-	"thread_count": 1,
-	"databases": [{
-		"dbinfo": {
-			"name": "db01",
-			"replica": 1,
-			"days": 10,
-			"cache": 16,
-			"blocks": 8,
-			"precision": "ms",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "databases": [{
+        "dbinfo": {
+            "name": "db01",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
             "update": 0,
-			"maxtablesPerVnode": 1000
-		},
-		"super_tables": [{
-			"name": "stb01",
-			"childtable_count": 100,
-			"childtable_prefix": "stb01_",
-			"auto_create_table": "no",
-			"data_source": "rand",
-			"insert_mode": "taosc",
-			"insert_rate": 0,
-			"insert_rows": 1000,
-			"timestamp_step": 1000,
-			"start_timestamp": "2020-10-01 00:00:00.000",
-			"sample_format": "csv",
-			"sample_file": "/home/data/sample.csv",
-			"tags_file": "",
-			"columns": [{
-				"type": "SMALLINT"
-			}, {
-				"type": "BOOL"
-			}, {
-				"type": "BINARY",
-				"len": 6
-			}],
-			"tags": [{
-				"type": "INT"
-			},{
-				"type": "BINARY",
-				"len": 4
-			}]
-		}]
-	}]
+            "maxtablesPerVnode": 1000
+        },
+        "super_tables": [{
+            "name": "stb01",
+            "childtable_count": 100,
+            "childtable_prefix": "stb01_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rate": 0,
+            "insert_rows": 1000,
+            "timestamp_step": 1000,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "/home/data/sample.csv",
+            "tags_file": "",
+            "columns": [{
+                "type": "SMALLINT"
+            }, {
+                "type": "BOOL"
+            }, {
+                "type": "BINARY",
+                "len": 6
+            }],
+            "tags": [{
+                "type": "INT"
+            },{
+                "type": "BINARY",
+                "len": 4
+            }]
+        }]
+    }]
 }

tests/pytest/tools/taosdemoTestInterlace.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.numberOfTables = 10000
+        self.numberOfRecords = 100
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+        os.system("%staosdemo -f tools/insert-interlace.json" % binPath)
+
+        tdSql.execute("use db")
+        tdSql.query("select count(tbname) from db.stb")
+        tdSql.checkData(0, 0, 100)
+        tdSql.query("select count(*) from db.stb")
+        tdSql.checkData(0, 0, 33000)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemoTestLimitOffset.py

@@ -68,6 +68,15 @@ class TDTestCase:
         tdSql.query("select count(*) from db.stb")
         tdSql.checkData(0, 0, 20000)
 
+        os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath)
+
+        tdSql.execute("reset query cache")
+        tdSql.execute("use db")
+        tdSql.query("select count(tbname) from db.stb")
+        tdSql.checkData(0, 0, 100)
+        tdSql.query("select count(*) from db.stb")
+        tdSql.checkData(0, 0, 1000)
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)