Merge pull request #21326 from taosdata/test/TD-21161

Test/td 21161

tests/parallel_test/cases.task

@@ -589,6 +589,32 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_benchmark.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -R
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py

tests/system-test/1-insert/rowlength64k.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 10,
+    "thread_count_create_tbl": 10,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 10,
+    "num_of_records_per_req": 1,
+    "max_sql_len": 102400000,
+    "databases": [{
+        "dbinfo": {
+            "name": "json_test",
+            "drop": "yes",
+            "replica": 1,
+            "precision": "ms",
+            "keep": 36500,
+            "minRows": 100,
+            "maxRows": 4096
+        },
+        "super_tables": [{
+            "name": "stb_old",
+            "child_table_exists":"no",
+            "childtable_count": 1,
+            "childtable_prefix": "stb_old_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 10,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "sample",
+            "sample_file": "./tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT","count":1000}, {"type": "BINARY", "len": 16, "count":20}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },{
+            "name": "stb_new",
+            "child_table_exists":"no",
+            "childtable_count": 1,
+            "childtable_prefix": "stb_new_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 10,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT","count":4000}, {"type": "BINARY", "len": 16, "count":90}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":3}]
+        },{
+            "name": "stb_mix",
+            "child_table_exists":"no",
+            "childtable_count": 1,
+            "childtable_prefix": "stb_mix_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 10,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT","count":500},{"type": "SMALLINT","count":500},{"type": "TINYINT","count":500},{"type": "DOUBLE","count":500},{"type": "FLOAT","count":500},{"type": "BOOL","count":500},{"type": "BIGINT","count":500},{"type": "NCHAR","len": 20,"count":300},{"type": "BINARY","len": 34,"count":290},{"type": "BINARY","len": 101,"count":1}],
+            "tags": [{"type": "INT", "count":3}, {"type": "NCHAR", "len": 10, "count":1}]
+        },{
+            "name": "stb_excel",
+            "child_table_exists":"no",
+            "childtable_count": 1,
+            "childtable_prefix": "stb_excel_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 10,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT","count":500},{"type": "SMALLINT","count":500},{"type": "SMALLINT","count":500},{"type": "DOUBLE","count":500},{"type": "FLOAT","count":500},{"type": "BOOL","count":500},{"type": "BIGINT","count":500},{"type": "NCHAR","len": 30,"count":300},{"type": "BINARY","len": 47,"count":290},{"type": "BINARY","len": 211,"count":1}],
+            "tags": [{"type": "INT", "count":3}, {"type": "NCHAR", "len": 10, "count":1}]
+        }]
+    }]
+}
\ No newline at end of file

tests/system-test/1-insert/rowlength64k_1.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from util.cases import tdCases
+from .rowlength64k import *
+
+class TDTestCase(TDTestCase):
+
+                        
+    def run(self):
+        tdSql.prepare()
+        
+        startTime_all = time.time() 
+        #self.run_1() 
+        # self.run_2() 
+        self.run_3() 
+        #self.run_4() 
+        
+        endTime_all = time.time()
+        print("total time %ds" % (endTime_all - startTime_all))  
+
+        
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/rowlength64k_2.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from util.cases import tdCases
+from .rowlength64k import *
+
+class TDTestCase(TDTestCase):
+
+                        
+    def run(self):
+        tdSql.prepare()
+        
+        startTime_all = time.time() 
+        # self.run_1() 
+        # self.run_2() 
+        # self.run_3() 
+        self.run_4() 
+        #self.run_5() 
+        # self.run_5() 
+        # self.run_6() 
+        # self.run_7() 
+        
+        endTime_all = time.time()
+        print("total time %ds" % (endTime_all - startTime_all))  
+
+        
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/rowlength64k_3.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from util.cases import tdCases
+from .rowlength64k import *
+
+class TDTestCase(TDTestCase):
+
+                        
+    def run(self):
+        tdSql.prepare()
+        
+        startTime_all = time.time() 
+        # self.run_1() 
+        # self.run_2() 
+        # self.run_3() 
+        # self.run_4() 
+        self.run_6() 
+        self.run_7() 
+        
+        endTime_all = time.time()
+        print("total time %ds" % (endTime_all - startTime_all))  
+
+        
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/rowlength64k_4.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+from util.cases import tdCases
+from .rowlength64k import *
+
+class TDTestCase(TDTestCase):
+
+                        
+    def run(self):
+        tdSql.prepare()
+        
+        startTime_all = time.time() 
+        # self.run_1() 
+        # self.run_2() 
+        # self.run_3() 
+        #self.run_4() 
+        self.run_5() 
+        # self.run_5() 
+        # self.run_6() 
+        # self.run_7() 
+        
+        endTime_all = time.time()
+        print("total time %ds" % (endTime_all - startTime_all))  
+
+        
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/1-insert/rowlength64k_benchmark.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import os
+import time
+import taos
+import subprocess
+import string
+from faker import Faker
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+from util.dnodes import *
+
+class TDTestCase:
+    updatecfgDict = {'maxSQLLength':1048576,'debugFlag': 143 ,"querySmaOptimize":1}
+    
+    def init(self, conn, logSql, replicaVar):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.testcasePath = os.path.split(__file__)[0]
+        self.testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename))
+        
+        now = time.time()
+        self.ts = int(round(now * 1000))
+        self.num = 100
+    
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        tdSql.prepare()
+        
+        startTime_all = time.time()  
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+
+        #-N:regular table  -d:database name   -t:table num  -n:rows num per table  -l:col num  -y:force
+        #regular old && new
+        startTime = time.time()
+        os.system("%staosBenchmark -N -d regular_old -t 1 -n 10 -l 1023 -y" % binPath)
+        tdSql.execute("use regular_old")
+        tdSql.query("show tables;")
+        tdSql.checkRows(1)
+        tdSql.query("select * from meters;")
+        tdSql.checkCols(1024)
+        tdSql.query("describe meters;")
+        tdSql.checkRows(1024)
+
+        os.system("%staosBenchmark -N -d regular_new -t 1 -n 10 -l 4095 -y" % binPath)
+        tdSql.execute("use regular_new")
+        tdSql.query("show tables;")
+        tdSql.checkRows(1)
+        tdSql.query("select * from meters;")
+        tdSql.checkCols(4096)
+        tdSql.query("describe meters;")
+        tdSql.checkRows(4096)
+
+        #super table  -d:database name   -t:table num  -n:rows num per table  -l:col num  -y:force
+        os.system("%staosBenchmark -d super_old -t 1 -n 10 -l 1021 -y" % binPath)
+        tdSql.execute("use super_old")
+        tdSql.query("show tables;")
+        tdSql.checkRows(1)
+        tdSql.query("select * from meters;")
+        tdSql.checkCols(1024)
+        tdSql.query("select * from d0;")
+        tdSql.checkCols(1022)
+        tdSql.query("describe meters;")
+        tdSql.checkRows(1024)
+        tdSql.query("describe d0;")
+        tdSql.checkRows(1024)
+
+        os.system("%staosBenchmark -d super_new -t 1 -n 10 -l 4093 -y" % binPath)
+        tdSql.execute("use super_new")
+        tdSql.query("show tables;")
+        tdSql.checkRows(1)
+        tdSql.query("select * from meters;")
+        tdSql.checkCols(4096)
+        tdSql.query("select * from d0;")
+        tdSql.checkCols(4094)
+        tdSql.query("describe meters;")
+        tdSql.checkRows(4096)
+        tdSql.query("describe d0;")
+        tdSql.checkRows(4096)
+        tdSql.execute("create table stb_new1_1 using meters tags(1,2)")
+        tdSql.query("select * from stb_new1_1")
+        tdSql.checkCols(4094)
+        tdSql.query("describe stb_new1_1;")
+        tdSql.checkRows(4096)
+
+        # insert: create one  or mutiple tables per sql and insert multiple rows per sql 
+        os.system("%staosBenchmark -f %s/rowlength64k.json -y " % (binPath,self.testcasePath))
+        tdSql.execute("use json_test")
+        tdSql.query("select count (tbname) from stb_old")
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select * from stb_old")
+        tdSql.checkRows(10)
+        tdSql.checkCols(1024)
+            
+        tdSql.query("select count (tbname) from stb_new")
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select * from stb_new")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4096)
+        tdSql.query("describe stb_new;")
+        tdSql.checkRows(4096)
+        tdSql.query("select * from stb_new_0")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4091)
+        tdSql.query("describe stb_new_0;")
+        tdSql.checkRows(4096)
+        tdSql.execute("create table stb_new1_1 using stb_new tags(1,2,3,4,5)")
+        tdSql.query("select * from stb_new1_1")
+        tdSql.checkCols(4091)
+        tdSql.query("describe stb_new1_1;")
+        tdSql.checkRows(4096)
+
+        tdSql.query("select count (tbname) from stb_mix")
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select * from stb_mix")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4096)
+        tdSql.query("describe stb_mix;")
+        tdSql.checkRows(4096)
+        tdSql.query("select * from stb_mix_0")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4092)
+        tdSql.query("describe stb_mix_0;")
+        tdSql.checkRows(4096)
+
+        tdSql.query("select count (tbname) from stb_excel")
+        tdSql.checkData(0, 0, 10)
+
+        tdSql.query("select * from stb_excel")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4096)
+        tdSql.query("describe stb_excel;")
+        tdSql.checkRows(4096)
+        tdSql.query("select * from stb_excel_0")
+        tdSql.checkRows(10)
+        tdSql.checkCols(4092)
+        tdSql.query("describe stb_excel_0;")
+        tdSql.checkRows(4096)
+        endTime = time.time()
+        print("total time %ds" % (endTime - startTime))      
+
+        endTime_all = time.time()
+        print("total time %ds" % (endTime_all - startTime_all))        
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+        
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file