[TD-5872]<fix>: taosdemo stmt improve. (#7251)

* [TD-5872]<fix>: taosdemo stmt improve.

* refactor stmt functions.

* [TD-5872]<fix>: taosdemo stmt csv perf improve.

* rand func back to early impl.

* fix windows/mac compile error.

* fix empty tag sample.

* [TD-5873]<test>add stmt’performance taosdemo testcase
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>
Co-authored-by: tomchon <haoran920c@163.com>

tests/pytest/tools/taosdemoAllTest/1174-tag.csv

+1
+2
+3
\ No newline at end of file

tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json

+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "192.168.1.103",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 10,
+	"thread_count_create_tbl": 1,
+    "result_file": "1174.out",
+    "confirm_parameter_prompt": "no",
+	"num_of_records_per_req": 51,
+    "databases": [
+        {
+            "dbinfo": {
+			"name": "gdse",
+			"drop": "yes"
+		},
+		"super_tables": [{
+			"name": "model_1174",
+            "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "model_1174_",
+			"auto_create_table": "no",
+			"batch_create_tbl_num": 0,
+			"data_source": "sample",
+			"insert_mode": "stmt",
+			"insert_rate": 0,
+			"insert_rows": 2592000,
+			"interlace_rows": 1,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 0,
+			"max_sql_len": 1048576,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 1000,
+			"start_timestamp": "2021-05-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "tools/taosdemoAllTest/1174.csv",
+			"tags_file": "tools/taosdemoAllTest/1174-tag.csv",
+			"columns": [{"type": "FLOAT", "count": 109}, {"type": "INT", "count": 4}, {"type": "FLOAT", "count": 8}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 5}, {"type": "INT", "count": 47}, {"type": "BOOL", "count": 103}, {"type": "INT", "count": 2}, {"type": "TIMESTAMP", "count": 3}, {"type": "BOOL", "count": 28}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 6}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 7}, {"type": "BOOL", "count": 7}, {"type": "FLOAT", "count": 2}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 3}, {"type": "INT", "count": 3}, {"type": "BOOL", "count": 1}],
+			"tags": [{"type": "INT", "count": 1}]
+		}]
+	}]
+}

tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json

+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "192.168.1.103",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 10,
+	"thread_count_create_tbl": 1,
+    "result_file": "1174.out",
+    "confirm_parameter_prompt": "no",
+	"num_of_records_per_req": 51,
+    "databases": [
+        {
+            "dbinfo": {
+			"name": "gdse",
+			"drop": "yes"
+		},
+		"super_tables": [{
+			"name": "model_1174",
+            "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "model_1174_",
+			"auto_create_table": "no",
+			"batch_create_tbl_num": 0,
+			"data_source": "sample",
+			"insert_mode": "taosc",
+			"insert_rate": 0,
+			"insert_rows": 2592000,
+			"interlace_rows": 1,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 0,
+			"max_sql_len": 1048576,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 1000,
+			"start_timestamp": "2021-05-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "tools/taosdemoAllTest/1174.csv",
+			"tags_file": "tools/taosdemoAllTest/1174-tag.csv",
+			"columns": [{"type": "FLOAT", "count": 109}, {"type": "INT", "count": 4}, {"type": "FLOAT", "count": 8}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 5}, {"type": "INT", "count": 47}, {"type": "BOOL", "count": 103}, {"type": "INT", "count": 2}, {"type": "TIMESTAMP", "count": 3}, {"type": "BOOL", "count": 28}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 6}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 7}, {"type": "BOOL", "count": 7}, {"type": "FLOAT", "count": 2}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 3}, {"type": "INT", "count": 3}, {"type": "BOOL", "count": 1}],
+			"tags": [{"type": "INT", "count": 1}]
+		}]
+	}]
+}

tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json

+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "192.168.1.103",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 10,
+	"thread_count_create_tbl": 1,
+    "result_file": "1174.out",
+    "confirm_parameter_prompt": "no",
+	"num_of_records_per_req": 51,
+    "databases": [
+        {
+            "dbinfo": {
+			"name": "gdse",
+			"drop": "yes"
+		},
+		"super_tables": [{
+			"name": "model_1174",
+            "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "model_1174_",
+			"auto_create_table": "no",
+			"batch_create_tbl_num": 0,
+			"data_source": "rand",
+			"insert_mode": "stmt",
+			"insert_rate": 0,
+			"insert_rows": 259200,
+			"interlace_rows": 1,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 0,
+			"max_sql_len": 1048576,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 1000,
+			"start_timestamp": "2021-05-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "tools/taosdemoAllTest/1174.csv",
+			"tags_file": "tools/taosdemoAllTest/1174-tag.csv",
+			"columns": [{"type": "FLOAT", "count": 109}, {"type": "INT", "count": 4}, {"type": "FLOAT", "count": 8}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 5}, {"type": "INT", "count": 47}, {"type": "BOOL", "count": 103}, {"type": "INT", "count": 2}, {"type": "TIMESTAMP", "count": 3}, {"type": "BOOL", "count": 28}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 6}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 7}, {"type": "BOOL", "count": 7}, {"type": "FLOAT", "count": 2}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 3}, {"type": "INT", "count": 3}, {"type": "BOOL", "count": 1}],
+			"tags": [{"type": "INT", "count": 1}]
+		}]
+	}]
+}

tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json

+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "192.168.1.103",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 10,
+	"thread_count_create_tbl": 1,
+    "result_file": "1174.out",
+    "confirm_parameter_prompt": "no",
+	"num_of_records_per_req": 51,
+    "databases": [
+        {
+            "dbinfo": {
+			"name": "gdse",
+			"drop": "yes"
+		},
+		"super_tables": [{
+			"name": "model_1174",
+            "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "model_1174_",
+			"auto_create_table": "no",
+			"batch_create_tbl_num": 0,
+			"data_source": "sample",
+			"insert_mode": "stmt",
+			"insert_rate": 0,
+			"insert_rows": 259200,
+			"interlace_rows": 1,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 0,
+			"max_sql_len": 1048576,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 1000,
+			"start_timestamp": "2021-05-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "tools/taosdemoAllTest/1174.csv",
+			"tags_file": "tools/taosdemoAllTest/1174-tag.csv",
+			"columns": [{"type": "FLOAT", "count": 109}, {"type": "INT", "count": 4}, {"type": "FLOAT", "count": 8}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 5}, {"type": "INT", "count": 47}, {"type": "BOOL", "count": 103}, {"type": "INT", "count": 2}, {"type": "TIMESTAMP", "count": 3}, {"type": "BOOL", "count": 28}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 6}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 7}, {"type": "BOOL", "count": 7}, {"type": "FLOAT", "count": 2}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 3}, {"type": "INT", "count": 3}, {"type": "BOOL", "count": 1}],
+			"tags": [{"type": "INT", "count": 1}]
+		}]
+	}]
+}

tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json

+{
+	"filetype": "insert",
+	"cfgdir": "/etc/taos",
+	"host": "192.168.1.103",
+	"port": 6030,
+	"user": "root",
+	"password": "taosdata",
+	"thread_count": 10,
+	"thread_count_create_tbl": 1,
+    "result_file": "1174.out",
+    "confirm_parameter_prompt": "no",
+	"num_of_records_per_req": 51,
+    "databases": [
+        {
+            "dbinfo": {
+			"name": "gdse",
+			"drop": "yes"
+		},
+		"super_tables": [{
+			"name": "model_1174",
+            "child_table_exists":"no",
+			"childtable_count": 1,
+			"childtable_prefix": "model_1174_",
+			"auto_create_table": "no",
+			"batch_create_tbl_num": 0,
+			"data_source": "sample",
+			"insert_mode": "taosc",
+			"insert_rate": 0,
+			"insert_rows": 259200,
+			"interlace_rows": 1,
+			"multi_thread_write_one_tbl": "no",
+			"number_of_tbl_in_one_sql": 0,
+			"max_sql_len": 1048576,
+			"disorder_ratio": 0,
+			"disorder_range": 1000,
+			"timestamp_step": 1000,
+			"start_timestamp": "2021-05-01 00:00:00.000",
+			"sample_format": "csv",
+			"sample_file": "tools/taosdemoAllTest/1174.csv",
+			"tags_file": "tools/taosdemoAllTest/1174-tag.csv",
+			"columns": [{"type": "FLOAT", "count": 109}, {"type": "INT", "count": 4}, {"type": "FLOAT", "count": 8}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 5}, {"type": "INT", "count": 47}, {"type": "BOOL", "count": 103}, {"type": "INT", "count": 2}, {"type": "TIMESTAMP", "count": 3}, {"type": "BOOL", "count": 28}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 6}, {"type": "INT", "count": 1}, {"type": "FLOAT", "count": 7}, {"type": "BOOL", "count": 7}, {"type": "FLOAT", "count": 2}, {"type": "INT", "count": 3}, {"type": "FLOAT", "count": 3}, {"type": "INT", "count": 3}, {"type": "BOOL", "count": 1}],
+			"tags": [{"type": "INT", "count": 1}]
+		}]
+	}]
+}

tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmtPerformance.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        # tdSql.init(conn.cursor(), logSql)
+        
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+        
+    def run(self):
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+
+        # insert: create one  or mutiple tables per sql and insert multiple rows per sql 
+        os.system("%staosdemo -f tools/taosdemoAllTest/stmt/1174-small-stmt-random.json -y " % binPath)
+        # sleep(60)
+
+        # os.system("%staosdemo -f tools/taosdemoAllTest/stmt/1174-small-taosc.json -y " % binPath)
+        # sleep(60)
+        # os.system("%staosdemo -f tools/taosdemoAllTest/stmt/1174-small-stmt.json -y " % binPath)
+        # sleep(60)
+        # os.system("%staosdemo -f tools/taosdemoAllTest/stmt/1174-large-taosc.json -y " % binPath)
+        # sleep(60)
+        # os.system("%staosdemo -f tools/taosdemoAllTest/stmt/1174-large-stmt.json -y " % binPath)
+
+        # tdSql.execute("use db")
+        # tdSql.query("select count (tbname) from stb0")
+        # tdSql.checkData(0, 0, 1000)
+        # tdSql.query("select count (tbname) from stb1")
+        # tdSql.checkData(0, 0, 1000)
+        # tdSql.query("select count(*) from stb00_0")
+        # tdSql.checkData(0, 0, 100)
+        # tdSql.query("select count(*) from stb0")
+        # tdSql.checkData(0, 0, 100000)
+        # tdSql.query("select count(*) from stb01_1")
+        # tdSql.checkData(0, 0, 200)
+        # tdSql.query("select count(*) from stb1")
+        # tdSql.checkData(0, 0, 200000)        
+
+
+ 
+
+        testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf ./insert_res.txt")
+        os.system("rm -rf tools/taosdemoAllTest/%s.sql" % testcaseFilename )     
+        
+        
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())