[TD-1693][TD-1991]<test>improve mixing query case

tests/pytest/concurrent_inquiry.py

@@ -16,112 +16,186 @@ import sys
 import json
 import time
 import random
-# query sql
-query_sql = [
-# first supertable
-"select count(*) from test.meters ;",
-"select count(*) from test.meters where t3 > 2;",
-"select count(*) from test.meters where ts <> '2020-05-13 10:00:00.002';",
-"select count(*) from test.meters where t7 like 'taos_1%';",
-"select count(*) from test.meters where t7 like '_____2';",
-"select count(*) from test.meters where t8 like '%思%';",
-"select count(*) from test.meters interval(1n) order by ts desc;",
-#"select max(c0) from test.meters group by tbname",
-"select first(ts)  from test.meters where t5 >5000 and t5<5100;",
-"select last(ts)  from test.meters where t5 >5000 and t5<5100;",
-"select last_row(*)  from test.meters;",
-"select twa(c1) from test.t1 where ts > 1500000001000  and ts < 1500000101000" ,
-"select avg(c1) from test.meters where t5 >5000 and t5<5100;",
-"select bottom(c1, 2) from test.t1;",
-"select diff(c1) from test.t1;",
-"select leastsquares(c1, 1, 1) from test.t1  ;",
-"select max(c1) from test.meters where t5 >5000 and t5<5100;",
-"select min(c1) from test.meters where t5 >5000 and t5<5100;",
-"select c1 + c2 + c1 / c5 + c4 + c2 from test.t1;",
-"select percentile(c1, 50) from test.t1;",
-"select spread(c1) from test.t1  ;",
-"select stddev(c1) from test.t1;",
-"select sum(c1) from test.meters where t5 >5000 and t5<5100;",
-"select top(c1, 2) from test.meters where t5 >5000 and t5<5100;"
-"select twa(c4) from test.t1 where ts > 1500000001000  and ts < 1500000101000" ,
-"select avg(c4) from test.meters where t5 >5000 and t5<5100;",
-"select bottom(c4, 2) from test.t1 where t5 >5000 and t5<5100;",
-"select diff(c4) from test.t1 where t5 >5000 and t5<5100;",
-"select leastsquares(c4, 1, 1) from test.t1  ;",
-"select max(c4) from test.meters where t5 >5000 and t5<5100;",
-"select min(c4) from test.meters where t5 >5000 and t5<5100;",
-"select c5 + c2  + c4 / c5 + c4 + c2 from test.t1 ;",
-"select percentile(c5, 50) from test.t1;",
-"select spread(c5) from test.t1  ;",
-"select stddev(c5) from test.t1 where t5 >5000 and t5<5100;",
-"select sum(c5) from test.meters where t5 >5000 and t5<5100;",
-"select top(c5, 2) from test.meters where t5 >5000 and t5<5100;",
-#all vnode
-"select count(*) from test.meters where t5 >5000 and t5<5100",       
-"select max(c0),avg(c1) from test.meters where t5 >5000 and t5<5100",
-"select sum(c5),avg(c1) from test.meters where t5 >5000 and t5<5100",
-"select max(c0),min(c5) from test.meters where t5 >5000 and t5<5100",
-"select min(c0),avg(c5) from test.meters where t5 >5000 and t5<5100",
-# second supertable
-"select count(*) from test.meters1 where t3 > 2;",
-"select count(*) from test.meters1 where ts <> '2020-05-13 10:00:00.002';",
-"select count(*) from test.meters where t7 like 'taos_1%';",
-"select count(*) from test.meters where t7 like '_____2';",
-"select count(*) from test.meters where t8 like '%思%';",
-"select count(*) from test.meters1 interval(1n) order by ts desc;",
-#"select max(c0) from test.meters1 group by tbname",
-"select first(ts)  from test.meters1 where t5 >5000 and t5<5100;",
-"select last(ts)  from test.meters1 where t5 >5000 and t5<5100;",
-"select last_row(*)  from test.meters1 ;",
-"select twa(c1) from test.m1 where ts > 1500000001000  and ts < 1500000101000" ,
-"select avg(c1) from test.meters1 where t5 >5000 and t5<5100;",
-"select bottom(c1, 2) from test.m1 where t5 >5000 and t5<5100;",
-"select diff(c1) from test.m1 ;",
-"select leastsquares(c1, 1, 1) from test.m1  ;",
-"select max(c1) from test.meters1 where t5 >5000 and t5<5100;",
-"select min(c1) from test.meters1 where t5 >5000 and t5<5100;",
-"select c1 + c2 + c1 / c0 + c2 from test.m1 ;",
-"select percentile(c1, 50) from test.m1;",
-"select spread(c1) from test.m1  ;",
-"select stddev(c1) from test.m1;",
-"select sum(c1) from test.meters1 where t5 >5000 and t5<5100;",
-"select top(c1, 2) from test.meters1 where t5 >5000 and t5<5100;",
-"select twa(c5) from test.m1 where ts > 1500000001000  and ts < 1500000101000" ,
-"select avg(c5) from test.meters1 where t5 >5000 and t5<5100;",
-"select bottom(c5, 2) from test.m1;",
-"select diff(c5) from test.m1;",
-"select leastsquares(c5, 1, 1) from test.m1  ;",
-"select max(c5) from test.meters1 where t5 >5000 and t5<5100;",
-"select min(c5) from test.meters1 where t5 >5000 and t5<5100;",
-"select c5 + c2  + c4 / c5  + c0 from test.m1;",
-"select percentile(c4, 50) from test.m1;",
-"select spread(c4) from test.m1  ;",
-"select stddev(c4) from test.m1;",
-"select sum(c4) from test.meters1 where t5 >5100 and t5<5300;",
-"select top(c4, 2) from test.meters1 where t5 >5100 and t5<5300;",
-"select count(*) from test.meters1 where t5 >5100 and t5<5300",
-#all vnode
-"select count(*) from test.meters1 where t5 >5100 and t5<5300",       
-"select max(c0),avg(c1) from test.meters1 where t5 >5000 and t5<5100",
-"select sum(c5),avg(c1) from test.meters1 where t5 >5000 and t5<5100",
-"select max(c0),min(c5) from test.meters1 where t5 >5000 and t5<5100",
-"select min(c0),avg(c5) from test.meters1 where t5 >5000 and t5<5100",
-#join 
-# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t5 = meters1.t5",
-# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t7 = meters1.t7",
-# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8",
-# "select meters.ts,meters1.c2 from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8"
+import requests
+from requests.auth import HTTPBasicAuth
+func_list=['avg','count','twa','sum','stddev','leastsquares','min',
+'max','first','last','top','bottom','percentile','apercentile',
+'last_row','diff','spread']
+condition_list=[
+    "where _c0 > now -10d ",
+    'interval(10s)',
+    'limit 10',
+    'group by',
+    'order by',
+    'fill(null)'
+    
 ]
-
+where_list = ['_c0>now-10d',' <50'," like \'%a%\'"]
 class ConcurrentInquiry:
-    def initConnection(self):  
-        self.numOfTherads = 50
+    def __init__(self,n_Therads=25,r_Therads=25):  
+        self.n_numOfTherads = n_Therads
+        self.r_numOfTherads = r_Therads
         self.ts=1500000001000
-        
+        self.dbname='test'
+        self.stb_list=[]
+        self.subtb_list=[]
+        self.stb_stru_list=[]
+        self.subtb_stru_list=[]
+        self.stb_tag_list=[]
+        self.subtb_tag_list=[]
     def SetThreadsNum(self,num):
         self.numOfTherads=num
-    def query_thread(self,threadID):
-        host = "10.211.55.14"
+    def ret_fcol(self,cl,sql):                     #返回结果的第一列
+        cl.execute(sql)
+        fcol_list=[]
+        for data in cl:
+            fcol_list.append(data[0])
+        return fcol_list
+    def r_stb_list(self,cl):
+        sql='show '+self.dbname+'.stables'
+        self.stb_list=self.ret_fcol(cl,sql)
+    def r_subtb_list(self,cl,stablename):
+        sql='select tbname from '+self.dbname+'.'+stablename+' limit 2;'
+        self.subtb_list+=self.ret_fcol(cl,sql)
+    def cal_struct(self,cl,tbname):
+        tb=[]
+        tag=[]
+        sql='describe '+self.dbname+'.'+tbname+';'
+        cl.execute(sql)
+        for data in cl:
+            if data[3]:
+                tag.append(data[0])
+            else:
+                tb.append(data[0])
+        return tb,tag
+    def r_stb_stru(self,cl):
+        for i in self.stb_list:
+            tb,tag=self.cal_struct(cl,i)
+            self.stb_stru_list.append(tb)
+            self.stb_tag_list.append(tag)
+    def r_subtb_stru(self,cl):
+        for i in self.subtb_list:
+            tb,tag=self.cal_struct(cl,i)
+            self.subtb_stru_list.append(tb)
+            self.subtb_tag_list.append(tag)
+    def get_full(self):
+        host = "127.0.0.1"
+        user = "root"
+        password = "taosdata"
+        conn = taos.connect(
+            host,
+            user,
+            password,
+            )
+        cl = conn.cursor()
+        self.r_stb_list(cl)
+        for i in self.stb_list:
+            self.r_subtb_list(cl,i)
+        self.r_stb_stru(cl)
+        self.r_subtb_stru(cl)
+        #print(self.stb_list,self.subtb_list,self.stb_stru_list,self.stb_tag_list,self.subtb_stru_list,self.subtb_tag_list)
+        cl.close()
+        conn.close()   
+    def con_where(self,tlist):
+        l=[]
+        for i in range(random.randint(0,len(tlist))):
+            c = random.choice(where_list)
+            if c == '_c0>now-10d':
+                l.append(c)
+            else:
+                l.append(random.choice(tlist)+c)
+        return 'where '+random.choice([' and ',' or ']).join(l)
+    def con_interval(self,tlist):
+        return random.choice(['interval(10s)','interval(10d)','interval(1n)'])
+    def con_limit(self,tlist):
+        return random.choice(['limit 10','limit 10 offset 10','slimit 10','slimit 10 offset 10','limit 10 slimit 10','limit 10 offset 5 slimit 5 soffset 10'])
+    def con_fill(self,tlist):
+        return random.choice(['fill(null)','fill(prev)','fill(none)','fill(LINEAR)'])
+    def con_group(self,tlist):
+        return 'group by '+random.choice(tlist)
+    def con_order(self,tlist):
+        return 'order by '+random.choice(tlist)
+    def gen_query_sql(self):
+        tbi=random.randint(0,len(self.subtb_list)+len(self.stb_list))
+        tbname=''
+        col_list=[]
+        tag_list=[]
+        is_stb=0
+        if tbi>len(self.stb_list) :
+            tbi=tbi-len(self.stb_list)
+            tbname=self.subtb_list[tbi-1]
+            col_list=self.subtb_stru_list[tbi-1]
+            tag_list=self.subtb_tag_list[tbi-1]
+        else:
+            tbname=self.stb_list[tbi-1]
+            col_list=self.stb_stru_list[tbi-1]
+            tag_list=self.stb_tag_list[tbi-1]
+            is_stb=1
+        tlist=col_list+tag_list
+        con_rand=random.randint(0,len(condition_list))
+        func_rand=random.randint(0,len(func_list))
+        col_rand=random.randint(0,len(col_list))
+        tag_rand=random.randint(0,len(tag_list))
+        t_rand=random.randint(0,len(tlist))
+        sql='select '                                           #select 
+        random.shuffle(col_list)
+        random.shuffle(func_list)
+        sel_col_list=[]
+        col_rand=random.randint(0,len(col_list))
+        for i,j in zip(col_list[0:col_rand],func_list):
+            if j == 'leastsquares':
+                sel_col_list.append(j+'('+i+',1,1)')
+            elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile':
+                sel_col_list.append(j+'('+i+',1)')
+            else:
+                sel_col_list.append(j+'('+i+')')
+        sql=sql+','.join(sel_col_list)+' from '+random.choice(self.stb_list+self.subtb_list)+' '                        #select col & func
+        con_func=[self.con_where,self.con_interval,self.con_limit,self.con_group,self.con_order,self.con_fill]
+        sel_con=random.sample(con_func,random.randint(0,len(con_func)))
+        sel_con_list=[]
+        for i in sel_con:
+            sel_con_list.append(i(tlist))
+        sql+=' '.join(sel_con_list)                                       # condition
+        print(sql)
+        return sql
+    def rest_query(self,sql):
+        host = "127.0.0.1"
+        user = "root"
+        password = "taosdata"
+        port =6041
+        url = "http://{}:{}/rest/sql".format(host, port )
+        try:
+            r = requests.post(url, 
+                data = 'use test',
+                auth = HTTPBasicAuth('root', 'taosdata'))  
+            r = requests.post(url, 
+                data = sql,
+                auth = HTTPBasicAuth('root', 'taosdata'))         
+        except:
+            print("REST API Failure (TODO: more info here)")
+            raise
+        rj = r.json()
+        if ('status' not in rj):
+            raise RuntimeError("No status in REST response")
+
+        if rj['status'] == 'error':  # clearly reported error
+            if ('code' not in rj):  # error without code
+                raise RuntimeError("REST error return without code")
+            errno = rj['code']  # May need to massage this in the future
+            # print("Raising programming error with REST return: {}".format(rj))
+            raise taos.error.ProgrammingError(
+                rj['desc'], errno)  # todo: check existance of 'desc'
+
+        if rj['status'] != 'succ':  # better be this
+            raise RuntimeError(
+                "Unexpected REST return status: {}".format(
+                    rj['status']))
+
+        nRows = rj['rows'] if ('rows' in rj) else 0
+        
+        return nRows
+    def query_thread_n(self,threadID):
+        host = "127.0.0.1"
         user = "root"
         password = "taosdata"
         conn = taos.connect(
@@ -135,35 +209,59 @@ class ConcurrentInquiry:
         print("Thread %d: starting" % threadID)
         
         while True:
-            ran_query_sql=query_sql
-            random.shuffle(ran_query_sql)
-            for i in ran_query_sql:
-                print("Thread %d : %s"% (threadID,i))
+            
                 try:
+                    sql=self.gen_query_sql()
+                    print("sql is ",sql)
                     start = time.time()
-                    cl.execute(i)
-                    cl.fetchall
+                    cl.execute(sql)
+                    cl.fetchall()
                     end = time.time()
                     print("time cost :",end-start)
                 except Exception as e:
                     print(
                 "Failure thread%d, sql: %s,exception: %s" %
-                (threadID, str(i),str(e)))
-                    exit(-1)
+                (threadID, str(sql),str(e)))
+                    #exit(-1)
                     
                 
-            print("Thread %d: finishing" % threadID)
+        print("Thread %d: finishing" % threadID)
           
-        
+    def query_thread_r(self,threadID):
+        print("Thread %d: starting" % threadID)
+        while True:
+                try:
+                    sql=self.gen_query_sql()
+                    print("sql is ",sql)
+                    start = time.time()
+                    self.rest_query(sql)
+                    end = time.time()
+                    print("time cost :",end-start)
+                except Exception as e:
+                    print(
+                "Failure thread%d, sql: %s,exception: %s" %
+                (threadID, str(sql),str(e)))
+                    #exit(-1)
+                    
+                
+        print("Thread %d: finishing" % threadID)    
 
     def run(self):
-            
+        print(self.n_numOfTherads,self.r_numOfTherads)  
         threads = []
-        for i in range(self.numOfTherads):
-            thread = threading.Thread(target=self.query_thread, args=(i,))
+        for i in range(self.n_numOfTherads):
+            thread = threading.Thread(target=self.query_thread_n, args=(i,))
             threads.append(thread)
             thread.start()  
-        
-q = ConcurrentInquiry()
-q.initConnection()
+        for i in range(self.r_numOfTherads):
+        # for i in range(1):
+            thread = threading.Thread(target=self.query_thread_r, args=(i,))
+            threads.append(thread)
+            thread.start()
+if len(sys.argv)>1:
+    q = ConcurrentInquiry(n_Therads=sys.argv[1],r_Therads=sys.argv[2])
+else:
+    q = ConcurrentInquiry()
+q.get_full()
+#q.gen_query_sql()
 q.run()

tests/pytest/cq.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+import threading
+import taos
+import sys
+import json
+import time
+import random
+# query sql
+query_sql = [
+# first supertable
+"select count(*) from test.meters ;",
+"select count(*) from test.meters where t3 > 2;",
+"select count(*) from test.meters where ts <> '2020-05-13 10:00:00.002';",
+"select count(*) from test.meters where t7 like 'taos_1%';",
+"select count(*) from test.meters where t7 like '_____2';",
+"select count(*) from test.meters where t8 like '%思%';",
+"select count(*) from test.meters interval(1n) order by ts desc;",
+#"select max(c0) from test.meters group by tbname",
+"select first(ts)  from test.meters where t5 >5000 and t5<5100;",
+"select last(ts)  from test.meters where t5 >5000 and t5<5100;",
+"select last_row(*)  from test.meters;",
+"select twa(c1) from test.t1 where ts > 1500000001000  and ts < 1500000101000" ,
+"select avg(c1) from test.meters where t5 >5000 and t5<5100;",
+"select bottom(c1, 2) from test.t1;",
+"select diff(c1) from test.t1;",
+"select leastsquares(c1, 1, 1) from test.t1  ;",
+"select max(c1) from test.meters where t5 >5000 and t5<5100;",
+"select min(c1) from test.meters where t5 >5000 and t5<5100;",
+"select c1 + c2 + c1 / c5 + c4 + c2 from test.t1;",
+"select percentile(c1, 50) from test.t1;",
+"select spread(c1) from test.t1  ;",
+"select stddev(c1) from test.t1;",
+"select sum(c1) from test.meters where t5 >5000 and t5<5100;",
+"select top(c1, 2) from test.meters where t5 >5000 and t5<5100;"
+"select twa(c4) from test.t1 where ts > 1500000001000  and ts < 1500000101000" ,
+"select avg(c4) from test.meters where t5 >5000 and t5<5100;",
+"select bottom(c4, 2) from test.t1 where t5 >5000 and t5<5100;",
+"select diff(c4) from test.t1 where t5 >5000 and t5<5100;",
+"select leastsquares(c4, 1, 1) from test.t1  ;",
+"select max(c4) from test.meters where t5 >5000 and t5<5100;",
+"select min(c4) from test.meters where t5 >5000 and t5<5100;",
+"select c5 + c2  + c4 / c5 + c4 + c2 from test.t1 ;",
+"select percentile(c5, 50) from test.t1;",
+"select spread(c5) from test.t1  ;",
+"select stddev(c5) from test.t1 where t5 >5000 and t5<5100;",
+"select sum(c5) from test.meters where t5 >5000 and t5<5100;",
+"select top(c5, 2) from test.meters where t5 >5000 and t5<5100;",
+#all vnode
+"select count(*) from test.meters where t5 >5000 and t5<5100",       
+"select max(c0),avg(c1) from test.meters where t5 >5000 and t5<5100",
+"select sum(c5),avg(c1) from test.meters where t5 >5000 and t5<5100",
+"select max(c0),min(c5) from test.meters where t5 >5000 and t5<5100",
+"select min(c0),avg(c5) from test.meters where t5 >5000 and t5<5100",
+# second supertable
+"select count(*) from test.meters1 where t3 > 2;",
+"select count(*) from test.meters1 where ts <> '2020-05-13 10:00:00.002';",
+"select count(*) from test.meters where t7 like 'taos_1%';",
+"select count(*) from test.meters where t7 like '_____2';",
+"select count(*) from test.meters where t8 like '%思%';",
+"select count(*) from test.meters1 interval(1n) order by ts desc;",
+#"select max(c0) from test.meters1 group by tbname",
+"select first(ts)  from test.meters1 where t5 >5000 and t5<5100;",
+"select last(ts)  from test.meters1 where t5 >5000 and t5<5100;",
+"select last_row(*)  from test.meters1 ;",
+"select twa(c1) from test.m1 where ts > 1500000001000  and ts < 1500000101000" ,
+"select avg(c1) from test.meters1 where t5 >5000 and t5<5100;",
+"select bottom(c1, 2) from test.m1 where t5 >5000 and t5<5100;",
+"select diff(c1) from test.m1 ;",
+"select leastsquares(c1, 1, 1) from test.m1  ;",
+"select max(c1) from test.meters1 where t5 >5000 and t5<5100;",
+"select min(c1) from test.meters1 where t5 >5000 and t5<5100;",
+"select c1 + c2 + c1 / c0 + c2 from test.m1 ;",
+"select percentile(c1, 50) from test.m1;",
+"select spread(c1) from test.m1  ;",
+"select stddev(c1) from test.m1;",
+"select sum(c1) from test.meters1 where t5 >5000 and t5<5100;",
+"select top(c1, 2) from test.meters1 where t5 >5000 and t5<5100;",
+"select twa(c5) from test.m1 where ts > 1500000001000  and ts < 1500000101000" ,
+"select avg(c5) from test.meters1 where t5 >5000 and t5<5100;",
+"select bottom(c5, 2) from test.m1;",
+"select diff(c5) from test.m1;",
+"select leastsquares(c5, 1, 1) from test.m1  ;",
+"select max(c5) from test.meters1 where t5 >5000 and t5<5100;",
+"select min(c5) from test.meters1 where t5 >5000 and t5<5100;",
+"select c5 + c2  + c4 / c5  + c0 from test.m1;",
+"select percentile(c4, 50) from test.m1;",
+"select spread(c4) from test.m1  ;",
+"select stddev(c4) from test.m1;",
+"select sum(c4) from test.meters1 where t5 >5100 and t5<5300;",
+"select top(c4, 2) from test.meters1 where t5 >5100 and t5<5300;",
+"select count(*) from test.meters1 where t5 >5100 and t5<5300",
+#all vnode
+"select count(*) from test.meters1 where t5 >5100 and t5<5300",       
+"select max(c0),avg(c1) from test.meters1 where t5 >5000 and t5<5100",
+"select sum(c5),avg(c1) from test.meters1 where t5 >5000 and t5<5100",
+"select max(c0),min(c5) from test.meters1 where t5 >5000 and t5<5100",
+"select min(c0),avg(c5) from test.meters1 where t5 >5000 and t5<5100",
+#join 
+# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t5 = meters1.t5",
+# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t7 = meters1.t7",
+# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8",
+# "select meters.ts,meters1.c2 from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8"
+]
+
+class ConcurrentInquiry:
+    def initConnection(self):  
+        self.numOfTherads = 50
+        self.ts=1500000001000
+        
+    def SetThreadsNum(self,num):
+        self.numOfTherads=num
+    def query_thread(self,threadID):
+        host = "10.211.55.14"
+        user = "root"
+        password = "taosdata"
+        conn = taos.connect(
+            host,
+            user,
+            password,
+            )
+        cl = conn.cursor()
+        cl.execute("use test;")
+        
+        print("Thread %d: starting" % threadID)
+        
+        while True:
+            ran_query_sql=query_sql
+            random.shuffle(ran_query_sql)
+            for i in ran_query_sql:
+                print("Thread %d : %s"% (threadID,i))
+                try:
+                    start = time.time()
+                    cl.execute(i)
+                    cl.fetchall()
+                    end = time.time()
+                    print("time cost :",end-start)
+                except Exception as e:
+                    print(
+                "Failure thread%d, sql: %s,exception: %s" %
+                (threadID, str(i),str(e)))
+                    exit(-1)
+                    
+                
+            print("Thread %d: finishing" % threadID)
+          
+        
+
+    def run(self):
+            
+        threads = []
+        for i in range(self.numOfTherads):
+            thread = threading.Thread(target=self.query_thread, args=(i,))
+            threads.append(thread)
+            thread.start()  
+        
+q = ConcurrentInquiry()
+q.initConnection()
+q.run()