Merge pull request #6350 from taosdata/test/TD-4475

Test/td 4475

tests/pytest/manualTest/manual_alter_block.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+##TODO: auto test version is currently unsupported, need to come up with 
+#       an auto test version in the future
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+        
+    def run(self):
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"  
+
+        #alter cache block to 3, then check alter
+        tdSql.execute('alter database db blocks 3')
+        tdSql.query('show databases')
+        tdSql.checkData(0,9,3)
+
+        #run taosdemo to occupy all cache, need to manually check memory consumption
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath) 
+        input("please check memory usage for taosd. After checking, press enter")
+
+        #alter cache block to 8, then check alter
+        tdSql.execute('alter database db blocks 8')
+        tdSql.query('show databases')
+        tdSql.checkData(0,9,8)
+
+        #run taosdemo to occupy all cache, need to manually check memory consumption
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block2.json" % binPath) 
+        input("please check memory usage for taosd. After checking, press enter")
+
+        ##expected result the peak memory consumption should increase by around 80MB = 5 blocks of cache
+
+        ##test results
+        #2021/06/02 before:2621700K     after: 2703640K memory usage increased by 80MB = 5 block
+        #           confirm with the change in block.       Baosheng Chang
+
+    def stop(self):
+        tdSql.close()
+        tdLog.debug("%s alter block manual check finish" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/manualTest/manual_alter_comp.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+
+##TODO: auto test version is currently unsupported, need to come up with 
+#       an auto test version in the future
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def getRootPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+        print(selfPath)
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+            print(projPath)
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+            print("test" + projPath)
+
+        for root, dirs, files in os.walk(projPath):
+            if ('data' in dirs and 'sim' in root):
+                rootPath = root
+
+        return rootPath
+        
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+        
+        
+
+    def run(self):
+        dnodePath = self.getRootPath()
+        os.system(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        tdSql.prepare()
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"  
+
+        #comp is at 14
+        #check disk usage when comp=2
+        tdSql.query('show databases')
+        tdSql.execute('alter database db blocks 3') # minimize the data in cache
+        tdSql.checkData(0,14,2)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data/vnode")
+        print('comp = 2')
+        input("please check disk usage for taosd. After checking, press enter")
+        
+        #removing all data file
+        os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran
+        input("please check if the pervious data is being deleted. Then, press enter")
+
+        #check disk usage when comp=0
+        tdSql.prepare()
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,2)
+        tdSql.execute('alter database db comp 0')
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,0)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data")
+        print('comp = 0')
+        input("please check disk usage for taosd. After checking, press enter")
+
+        #removing all data file
+        os.system(f'sudo rm -rf {dnodePath}/data/* {dnodePath}/log/*')
+        #print(f'rm -rf {dnodePath}/data/* {dnodePath}/log/*') #for showing the command ran
+        input("please check if the pervious data is being deleted. Then, press enter")
+
+        #check disk usage when comp=1
+        tdSql.prepare()
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,2)
+        tdSql.execute('alter database db comp 1')
+        tdSql.query('show databases')
+        tdSql.checkData(0,14,1)
+        os.system("%staosdemo -f tools/taosdemoAllTest/manual_block1_comp.json" % binPath)
+        print("default location is at /home/bryan/Documents/Github/TDengine/sim/dnode1/data")
+        print('comp = 1')
+        input("please check disk usage for taosd. After checking, press enter")
+
+        ##test result
+        #   2021/06/02 comp=2:13M   comp=1:57M comp=0:399M. Test past
+        #       each row entered is identical       Tester - Baosheng Chang
+
+    def stop(self):
+        tdSql.close()
+        tdLog.debug("%s alter block manual check finish" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",                
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 3,
+            "precision": "ms",
+            "keep": 3650,
+            "minRows": 1000,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",   
+            "childtable_count": 500,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "no",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "sample",       
+            "insert_mode": "taosc",      
+            "insert_rows": 10000,       
+            "childtable_limit": 10,      
+            "childtable_offset":100,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1,         
+            "start_timestamp": "2019-10-01 00:00:00.000",  
+            "sample_format": "csv",       
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT", "count":3}, {"type": "DOUBLE", "count":3}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}, {"type": "BOOL"}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}
\ No newline at end of file

tests/pytest/tools/taosdemoAllTest/manual_block2.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,                   
+    "thread_count_create_tbl": 4,        
+    "result_file": "./insert_res.txt",   
+    "confirm_parameter_prompt": "no",    
+    "insert_interval": 0,         
+    "interlace_rows": 100,        
+    "num_of_records_per_req": 32766,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",                
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 3650,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"yes",   
+            "childtable_count": 500,   
+            "childtable_prefix": "stb_", 
+            "auto_create_table": "no",   
+            "batch_create_tbl_num": 20,   
+            "data_source": "rand",       
+            "insert_mode": "taosc",      
+            "insert_rows": 100000,       
+            "childtable_limit": 500,      
+            "childtable_offset":0,     
+            "interlace_rows": 0,         
+            "insert_interval":0,         
+            "max_sql_len": 1024000,      
+            "disorder_ratio": 0,         
+            "disorder_range": 1000,      
+            "timestamp_step": 1,         
+            "start_timestamp": "now",  
+            "sample_format": "csv",       
+            "sample_file": "./sample.csv",   
+            "tags_file": "",              
+            "columns": [{"type": "INT"}],
+            "tags": [{"type": "TINYINT", "count":2}]
+        }]
+    }]
+}
\ No newline at end of file