[TD-4840]<test>: add testcase of compressing tsdb meta data

tests/pytest/tsdb/insertDataDb1.json

@@ -35,13 +35,13 @@
         "super_tables": [{
             "name": "stb0",
             "child_table_exists":"no",
-            "childtable_count": 100000,
+            "childtable_count": 1000,
             "childtable_prefix": "stb00_",
             "auto_create_table": "no",
             "batch_create_tbl_num": 1000,
             "data_source": "rand",
             "insert_mode": "taosc",
-            "insert_rows": 100,
+            "insert_rows": 1000,
             "childtable_limit": 0,
             "childtable_offset":0,
             "interlace_rows": 0,
@@ -60,7 +60,7 @@
         {
             "name": "stb1",
             "child_table_exists":"no",
-            "childtable_count": 100000,
+            "childtable_count": 10000,
             "childtable_prefix": "stb01_",
             "auto_create_table": "no",
             "batch_create_tbl_num": 1000,

tests/pytest/tsdb/insertDataDb2.json

@@ -38,10 +38,10 @@
             "childtable_count": 200000,
             "childtable_prefix": "stb0_",
             "auto_create_table": "no",
-            "batch_create_tbl_num": 100,
+            "batch_create_tbl_num": 1000,
             "data_source": "rand",
             "insert_mode": "taosc",
-            "insert_rows": 2000,
+            "insert_rows": 0,
             "childtable_limit": 0,
             "childtable_offset":0,
             "interlace_rows": 0,
@@ -60,7 +60,7 @@
         {
             "name": "stb1",
             "child_table_exists":"no",
-            "childtable_count": 500000,
+            "childtable_count": 2,
             "childtable_prefix": "stb1_",
             "auto_create_table": "no",
             "batch_create_tbl_num": 1000,

tests/pytest/tsdb/sdbComp.py → tests/pytest/tsdb/tsdbComp.py

@@ -20,13 +20,13 @@ from util.cases import *
 from util.sql import *
 from util.dnodes import *
 import subprocess
-
+from random import choice
 
 class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
-        
+
     def getBuildPath(self):
         global selfPath
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -63,7 +63,7 @@ class TDTestCase:
         tdSql.execute("drop database if exists db1") 
         os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath)
         tdSql.execute("drop table if exists db2.stb0") 
-        os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath)
+        os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath)
         # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
         # print(query_pid1)
         tdSql.execute("use db2")
@@ -80,12 +80,54 @@ class TDTestCase:
         tdSql.execute("alter table stb2_0 drop column col1")
         tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
 
-        # stop taosd and compact wal file
+        # create  db utest
+
+        
+        dataType= [ "tinyint", "smallint",  "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned",  "bigint unsigned"] 
+
+        tdSql.execute("drop database if exists utest") 
+        tdSql.execute("create database utest keep 3650")
+        tdSql.execute("use utest")
+        tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+                    col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''')
+        
+        # rowNum1 = 13
+        # for i in range(rowNum1):
+        #     columnName= "col" + str(i+1)
+        #     tdSql.execute("alter table test drop column %s ;"  % columnName ) 
+
+        rowNum2= 988
+        for i in range(rowNum2):
+            tdSql.execute("alter table test add column col%d  %s ;" %( i+14, choice(dataType))     ) 
+
+        rowNum3= 988
+        for i in range(rowNum3):
+            tdSql.execute("alter table test drop column col%d   ;" %( i+14)     )    
+
+
+        self.rowNum = 1
+        self.rowNum2 = 100
+        self.rowNum3 = 20
+        self.ts = 1537146000000
+        self.ts1 = 1537146000000000
+        self.ts2 = 1597146000000
+        # tdSql.execute("create table test1 using test tags('beijing', 10)")
+        # tdSql.execute("create table test2 using test tags('tianjing', 20)")
+        # tdSql.execute("create table test3 using test tags('shanghai', 20)")   
+        
+        for j in range(self.rowNum2):
+            tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
+            for i in range(self.rowNum):
+                tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" 
+                        % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
+        
+        for j in range(self.rowNum2):    
+            tdSql.execute("drop table if exists test%d" % (j+1))
+
+        
+        # stop taosd and restart taosd
         tdDnodes.stop(1)
         sleep(10)
-        # assert os.path.exists(walFilePath) , "%s is not generated, compact didn't  take effect " % walFilePath    
-
-        # use new wal file to start  taosd 
         tdDnodes.start(1)
         sleep(5)
         tdSql.execute("reset query cache")
@@ -104,7 +146,11 @@ class TDTestCase:
         tdSql.checkData(0, 0, 2)
         tdSql.query("select count(*) from stb2_0")
         tdSql.checkData(0, 0, 2)
-   
+
+        tdSql.execute("use utest")
+        tdSql.query("select count (tbname) from test")
+        tdSql.checkData(0, 0, 1) 
+
         # delete useless file
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")