test:modify testcase of multi-Thread creating tables

tests/system-test/1-insert/insertWithMoreVgroup.py

@@ -13,7 +13,7 @@
 
 import sys
 import os
-import threading
+import threading as thd
 import multiprocessing as mp
 from numpy.lib.function_base import insert
 import taos
@@ -66,58 +66,13 @@ class TDTestCase:
         # self.create_tables();
         self.ts = 1500000000000
 
-
-    # run case   
-    def run(self):
-
-        # #  test base case
-        # self.test_case1()
-        # tdLog.debug(" LIMIT test_case1 ............ [OK]")
-
-        # test case
-        # self.test_case2()
-        # tdLog.debug(" LIMIT test_case2 ............ [OK]")
-
-        # # test case
-        # self.test_case3()
-        # tdLog.debug(" LIMIT test_case3 ............ [OK]")
-
-
-        # test qnode
-        self.test_case4()
-        tdLog.debug(" LIMIT test_case3 ............ [OK]")
-
-
     # stop 
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)
 
-    # --------------- case  -------------------
 
-    # create tables
-    def create_tables(self,dbname,stbname,count):
-        tdSql.execute("use %s" %dbname)
-        tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname)
-        pre_create = "create table"
-        sql = pre_create
-        tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
-        # print(time.time())
-        exeStartTime=time.time()
-        for i in range(count):
-            sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1)
-            if i >0 and i%3000 == 0:
-                tdSql.execute(sql)
-                sql = pre_create
-        # print(time.time())
-        # end sql        
-        if sql != pre_create:
-            tdSql.execute(sql)
-        exeEndTime=time.time()
-        spendTime=exeEndTime-exeStartTime
-        speedCreate=count/spendTime
-        tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
-        return
+    # --------------- case  -------------------
 
     def newcur(self,host,cfg):
         user = "root"
@@ -128,28 +83,23 @@ class TDTestCase:
         print(cur)
         return cur
 
-    def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop):
-        host = "localhost"
+    # create tables
+    def create_tables(self,host,dbname,stbname,count):
         buildPath = self.getBuildPath()
         config = buildPath+ "../sim/dnode1/cfg/"
         
         tsql=self.newcur(host,config)
-        tsql.execute("drop database if exists %s"%dbname)
-        tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
         tsql.execute("use %s" %dbname)
-        tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname)
 
         pre_create = "create table"
         sql = pre_create
-        tcountStop=int(tcountStop)
-        tcountStart=int(tcountStart)
-        count=tcountStop-tcountStart
+        count=int(count)
 
         tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
         # print(time.time())
         exeStartTime=time.time()
         # print(type(tcountStop),type(tcountStart))
-        for i in range(tcountStart,tcountStop):
+        for i in range(0,count):
             sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1)
             if i >0 and i%20000 == 0:
                 # print(sql)
@@ -166,11 +116,78 @@ class TDTestCase:
         # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
         return
 
+    def mutiThread_create_tables(self,host,dbname,stbname,vgroups,threadNumbers,count):
+        buildPath = self.getBuildPath()
+        config = buildPath+ "../sim/dnode1/cfg/"
+        
+        tsql=self.newcur(host,config)
+        tdLog.debug("create database %s"%dbname)
+        tsql.execute("drop database if exists %s"%dbname)
+        tsql.execute("create database %s vgroups %d"%(dbname,vgroups))
+        tsql.execute("use %s" %dbname)
+        count=int(count)
+        threads = []
+        for i in range(threadNumbers):
+            tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
+            threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) 
+        start_time = time.time()
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+        end_time = time.time()
+        spendTime=end_time-start_time
+        speedCreate=count/spendTime
+        tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate))
+        
+        return
+
+    # def create_tables(self,host,dbname,stbname,vgroups,tcountStart,tcountStop):
 
 
     # insert data
-    def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount):
-        tdSql.execute("use %s" %dbname)
+    def insert_data(self, host, dbname, stbname, ts_start,rowCount):
+        buildPath = self.getBuildPath()
+        config = buildPath+ "../sim/dnode1/cfg/"
+        
+        tsql=self.newcur(host,config)
+        tdLog.debug("ready to inser data")
+
+        tsql.execute("use %s" %dbname)
+        pre_insert = "insert into "
+        sql = pre_insert
+        tcount=int(tcount)
+        allRows=tcount*rowCount
+        tdLog.debug("doing insert data into stable-index:%s rows:%d ..."%(stbname, allRows))
+        exeStartTime=time.time()
+        for i in range(0,tcount):
+            sql += " %s_%d values "%(stbname,i)
+            for j in range(rowCount):
+                sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j)
+                if j >0 and j%5000 == 0:
+                    # print(sql)
+                    tdSql.execute(sql)
+                    sql = "insert into %s_%d values " %(stbname,i)
+        # end sql        
+        if sql != pre_insert:
+            # print(sql)
+            tdSql.execute(sql)
+        exeEndTime=time.time()
+        spendTime=exeEndTime-exeStartTime
+        speedInsert=allRows/spendTime
+        # tdLog.debug("spent %.2fs to INSERT  %d rows , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
+
+        tdLog.debug("INSERT TABLE DATA ............ [OK]")
+        return
+
+    def mutiThread_insert_data(self, host, dbname, stbname, threadNumbers, ts_start, tcountStart,tcountStop,rowCount):
+        buildPath = self.getBuildPath()
+        config = buildPath+ "../sim/dnode1/cfg/"
+        
+        tsql=self.newcur(host,config)
+        tdLog.debug("ready to inser data")
+
+        tsql.execute("use %s" %dbname)
         pre_insert = "insert into "
         sql = pre_insert
         tcount=tcountStop-tcountStart
@@ -195,8 +212,30 @@ class TDTestCase:
         # tdLog.debug("spent %.2fs to INSERT  %d rows , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert))
 
         tdLog.debug("INSERT TABLE DATA ............ [OK]")
+
+
+        buildPath = self.getBuildPath()
+        config = buildPath+ "../sim/dnode1/cfg/"
+        
+        tsql=self.newcur(host,config)
+        tsql.execute("use %s" %dbname)
+        count=int(count)
+        threads = []
+        for i in range(threadNumbers):
+            tsql.execute("create stable %s%d(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%(stbname,i))
+            threads.append(thd.Thread(target=self.create_tables, args=(host, dbname, stbname+"%d"%i, count,))) 
+        start_time = time.time()
+        for tr in threads:
+            tr.start()
+        for tr in threads:
+            tr.join()
+        end_time = time.time()
+        spendTime=end_time-start_time
+        speedCreate=count/spendTime
+        tdLog.debug("spent %.2fs to create %d stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,threadNumbers,threadNumbers*count,speedCreate))
         return
 
+
     def taosBench(self,jsonFile):
         buildPath = self.getBuildPath()
         if (buildPath == ""):
@@ -247,47 +286,15 @@ class TDTestCase:
         return
     # test case1 base 
     def test_case1(self):
-        tdLog.debug("-----create database and tables test------- ")
-
-        #create database and tables;
-        count=50000
-        vgroups=1
-        threads = []
-        threadNumbers=2
-        for i in range(threadNumbers):
-            threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) 
-        start_time = time.time()
-        for tr in threads:
-            tr.start()
-        for tr in threads:
-            tr.join()
-        end_time = time.time()
-        spendTime=end_time-start_time
-        speedCreate=count/spendTime
-        tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
-        # self.new_create_tables("db1", "stb1", 15*10000)
-        # self.new_create_tables("db1", "stb1", 15*10000)
-
-        # tdSql.execute("create database db4 vgroups 4")
-        # self.create_tables("db4", "stb4", 30*10000)
-
-        # tdSql.execute("create database db6 vgroups 6")
-        # self.create_tables("db6", "stb6", 30*10000)
-
-        # tdSql.execute("create database db8 vgroups 8")
-        # self.create_tables("db8", "stb8", 30*10000)
-
-        # tdSql.execute("create database db12 vgroups 12")
-        # self.create_tables("db12", "stb12", 30*10000)
-
-        # tdSql.execute("create database db16 vgroups 16")
-        # self.create_tables("db16", "stb16", 30*10000)
+        tdLog.debug("-----create database and muti-thread create tables test------- ")
+        #host,dbname,stbname,vgroups,threadNumbers,tcountStart,tcountStop
+        self.mutiThread_create_tables(host="localhost",dbname="db2",stbname="stb2", vgroups=1, threadNumbers=5, count=10000)
         return 
 
     # test case2 base:insert data
     def test_case2(self):
 
-        tdLog.debug("-----insert data test------- ")
+        tdLog.debug("-----muti-thread insert data test------- ")
         # drop database
         tdSql.execute("drop database if exists db1")
         tdSql.execute("drop database if exists db4")
@@ -301,28 +308,6 @@ class TDTestCase:
         tdSql.execute("create database db1 vgroups 1")
         self.create_tables("db1", "stb1", 1*100)
         self.insert_data("db1", "stb1", self.ts, 1*50,1*10000)
-        
-
-        tdSql.execute("create database db4 vgroups 4")
-        self.create_tables("db4", "stb4", 1*100)
-        self.insert_data("db4", "stb4", self.ts, 1*100,1*10000)
-
-        tdSql.execute("create database db6 vgroups 6")
-        self.create_tables("db6", "stb6", 1*100)
-        self.insert_data("db6", "stb6", self.ts, 1*100,1*10000)
-
-        tdSql.execute("create database db8 vgroups 8")
-        self.create_tables("db8", "stb8", 1*100)
-        self.insert_data("db8", "stb8", self.ts, 1*100,1*10000)
-
-        tdSql.execute("create database db12 vgroups 12")
-        self.create_tables("db12", "stb12", 1*100)
-        self.insert_data("db12", "stb12", self.ts, 1*100,1*10000)
-
-        tdSql.execute("create database db16 vgroups 16")
-        self.create_tables("db16", "stb16", 1*100)
-        self.insert_data("db16", "stb16", self.ts, 1*100,1*10000)
-                
         return
 
     def test_case3(self):
@@ -378,6 +363,27 @@ class TDTestCase:
         # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000)
         # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000)
 
+    # run case   
+    def run(self):
+
+        # #  test base case
+        # self.test_case1()
+        # tdLog.debug(" LIMIT test_case1 ............ [OK]")
+
+        # test case
+        # self.test_case2()
+        # tdLog.debug(" LIMIT test_case2 ............ [OK]")
+
+        # test case
+        self.test_case3()
+        tdLog.debug(" LIMIT test_case3 ............ [OK]")
+
+
+        # # test qnode
+        # self.test_case4()
+        # tdLog.debug(" LIMIT test_case3 ............ [OK]")
+
+
         return 
 #
 # add case with filename