From ffa51bb99089a32c1168c70f12281b90ac4acb8b Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Tue, 8 Dec 2020 15:51:00 +0800
Subject: [PATCH] [TD-2114]<test>: Imporve performance test script

---
 tests/perftest-scripts/perftest-query.sh      |  13 +-
 .../pytest/insert/insertFromCSVPerformance.py | 131 ++++++++++++
 tests/pytest/query/queryPerformance.py        | 195 ++++++++++--------
 tests/pytest/query/unionAllTest.py            |   2 +-
 tests/pytest/tools/taosdemoPerformance.py     |  93 +++++++++
 5 files changed, 346 insertions(+), 88 deletions(-)
 create mode 100644 tests/pytest/insert/insertFromCSVPerformance.py
 create mode 100644 tests/pytest/tools/taosdemoPerformance.py

diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh
index 51bb9b36c3..b96daa5464 100755
--- a/tests/perftest-scripts/perftest-query.sh
+++ b/tests/perftest-scripts/perftest-query.sh
@@ -65,7 +65,18 @@ function runQueryPerfTest {
 	echoInfo "Run Performance Test"
 	cd $WORK_DIR/TDengine/tests/pytest
 	
-	python3 query/queryPerformance.py 0 | tee -a $PERFORMANCE_TEST_REPORT
+	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+
+	yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -t 1000 -n 1000 > taosdemoperf.txt
+
+	CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'`
+	INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'`
+	REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'`
+	
+	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT
+	[ -f taosdemoperf.txt ] && rm taosdemoperf.txt
 }
 
 
diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py
new file mode 100644
index 0000000000..84fd1d7cca
--- /dev/null
+++ b/tests/pytest/insert/insertFromCSVPerformance.py
@@ -0,0 +1,131 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class insertFromCSVPerformace:
+    def __init__(self, commitID, dbName, stbName, branchName):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.branchName = branchName
+        self.ts = 1500074556514
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)
+        
+    def writeCSV(self):
+        with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
+            writer = csv.writer(csvFile, dialect='excel')
+            for i in range(1000000):
+                newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000)
+                d = datetime.datetime.fromtimestamp(newTimestamp / 1000)
+                dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
+                writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)])
+    
+    def removCSVHeader(self):
+        data = pd.read_csv("ordered.csv")
+        data = data.drop([0])
+        data.to_csv("ordered.csv", header = False, index = False)
+    
+    def createTables(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName)
+        cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName))
+
+        cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+        cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+
+        cursor.close()
+    
+    def run(self):
+        cursor = self.conn.cursor()
+        cursor.execute("use %s" % self.dbName)
+        print("==================== CSV insert performance ====================")
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t1 file 'outoforder.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t1")            
+        out_of_order_time = (float) (totalTime / 10)
+        print("Out of Order - Insert time: %f" % out_of_order_time)                      
+        
+        totalTime = 0
+        for i in range(10):
+            cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
+            startTime = time.time()
+            cursor.execute("insert into t2 file 'ordered.csv'")
+            totalTime += time.time() - startTime
+            cursor.execute("drop table if exists t2")
+
+        in_order_time = (float) (totalTime / 10)
+        print("In order - Insert time: %f" % in_order_time)
+        cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID))
+        
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='csv_insert',
+        type=str,
+        help='Database name to be created (default: csv_insert)')
+    parser.add_argument(
+        '-b',
+        '--branch-name',
+        action='store',
+        default='develop',
+        type=str,
+        help='branch name (default: develop)')
+    
+    args = parser.parse_args()
+    perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name)
+
+    perftest.createTables()
+    perftest.run()
\ No newline at end of file
diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py
index 72af38450c..720ae745cb 100644
--- a/tests/pytest/query/queryPerformance.py
+++ b/tests/pytest/query/queryPerformance.py
@@ -16,10 +16,16 @@ import sys
 import os
 import taos
 import time
+import argparse
 
 
 class taosdemoQueryPerformace:
-    def initConnection(self):
+    def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix):
+        self.clearCache = clearCache
+        self.commitID = commitID
+        self.dbName = dbName
+        self.stbName = stbName
+        self.tbPerfix = tbPerfix
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -30,92 +36,109 @@ class taosdemoQueryPerformace:
             self.password,
             self.config)
 
+    def createPerfTables(self):
+        cursor = self.conn.cursor()
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName)
+
+        sql = "select count(*) from test.meters"
+        tableid = 1
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters"
+        tableid = 2
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select count(*) from test.meters where loc='beijing'"
+        tableid = 3
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10"
+        tableid = 4
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)"
+        tableid = 5
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select last_row(*) from meters"
+        tableid = 6
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select * from meters"
+        tableid = 7
+        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'"
+        tableid = 8
+        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+
+        cursor.close()
 
     def query(self): 
-        cursor = self.conn.cursor()                  
-        cursor.execute("use test")
-
-        totalTime = 0
-        for i in range(100):       
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")  
-            startTime = time.time()            
-            cursor.execute("select count(*) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):  
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select count(*) from test.meters where loc='beijing'")
-            totalTime += time.time() - startTime
-        print("query time for: select count(*) from test.meters where loc='beijing' %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required              
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.meters where areaid=10 %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):                
-                # root permission is required
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from test.t10 interval(10s) %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select last_row(*) from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required           
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select * from meters")
-            totalTime += time.time() - startTime
-        print("query time for: select * from meters %f seconds" % (totalTime / 100))
-
-        totalTime = 0
-        for i in range(100):
-            if(sys.argv[1] == '1'):
-                # root permission is required        
-                os.system("echo 3 > /proc/sys/vm/drop_caches")
-            startTime = time.time()
-            cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'")
-            totalTime += time.time() - startTime
-        print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100))
+        cursor = self.conn.cursor() 
+        print("==================== query performance ====================")
+
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName)      
+
+        for data in cursor:
+            table_name = data[0]
+            query_id = data[1]
+            sql = data[2]            
+            
+            totalTime = 0            
+            cursor2 = self.conn.cursor()
+            cursor2.execute("use test")
+            for i in range(100):       
+                if(self.clearCache == True):
+                    # root permission is required
+                    os.system("echo 3 > /proc/sys/vm/drop_caches")                
+                
+                startTime = time.time()         
+                cursor2.execute(sql)
+                totalTime += time.time() - startTime
+            cursor2.close()    
+            print("query time for: %s %f seconds" % (sql, totalTime / 100))
+                        
+            cursor3 = self.conn.cursor()
+            cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID))
+
+        cursor3.close()
+        cursor.close()
 
 if __name__ == '__main__':
-    perftest = taosdemoQueryPerformace()
-    perftest.initConnection()
-    perftest.query()
\ No newline at end of file
+    parser = argparse.ArgumentParser()    
+    parser.add_argument(
+        '-r',
+        '--remove-cache',                
+        action='store_true',
+        default=False,
+        help='clear cache before query (default: False)')
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',
+        default='null',
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--stable-name',
+        action='store',
+        default='query_tb',
+        type=str,
+        help='table name to be created (default: query_tb)')
+    parser.add_argument(
+        '-p',
+        '--table-perfix',
+        action='store',
+        default='q',
+        type=str,
+        help='table name perfix (default: q)')
+    
+    args = parser.parse_args()
+    perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix)
+    perftest.createPerfTables()
+    perftest.query()
diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py
index bb4fb95de6..1b69c8ac4d 100644
--- a/tests/pytest/query/unionAllTest.py
+++ b/tests/pytest/query/unionAllTest.py
@@ -52,7 +52,7 @@ class TDTestCase:
         tdSql.checkRows(5)
 
         sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1
-            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5''''
+            union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5' limit 1'''
         tdSql.query(sql)
         tdSql.checkRows(6)
 
diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py
new file mode 100644
index 0000000000..6b6296e61a
--- /dev/null
+++ b/tests/pytest/tools/taosdemoPerformance.py
@@ -0,0 +1,93 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+import time
+import datetime
+import csv
+import random
+import pandas as pd
+import argparse
+import os.path
+
+class taosdemoPerformace:
+    def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond):
+        self.commitID = commitID
+        self.dbName = dbName
+        self.createTableTime = createTableTime
+        self.insertRecordsTime = insertRecordsTime
+        self.recordsPerSecond = recordsPerSecond        
+        self.host = "127.0.0.1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taosperf"
+        self.conn = taos.connect(
+            self.host,
+            self.user,
+            self.password,
+            self.config)    
+    
+    def createTablesAndStoreData(self):
+        cursor = self.conn.cursor()
+                    
+        cursor.execute("create database if not exists %s" % self.dbName)
+        cursor.execute("use %s" % self.dbName)
+        cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))")
+        print("==================== taosdemo performance ====================")
+        print("create tables time: %f" % self.createTableTime)
+        print("insert records time: %f" % self.insertRecordsTime)
+        print("records per second: %f" % self.recordsPerSecond)
+        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID))
+        cursor.execute("drop database if exists taosdemo_insert_test")
+
+        cursor.close()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-c',
+        '--commit-id',
+        action='store',        
+        type=str,
+        help='git commit id (default: null)')
+    parser.add_argument(
+        '-d',
+        '--database-name',
+        action='store',
+        default='perf',
+        type=str,
+        help='Database name to be created (default: perf)')
+    parser.add_argument(
+        '-t',
+        '--create-table',
+        action='store',        
+        type=float,
+        help='create table time')
+    parser.add_argument(
+        '-i',
+        '--insert-records',
+        action='store',        
+        type=float,
+        help='insert records time')
+    parser.add_argument(
+        '-r',
+        '---records-per-second',
+        action='store',        
+        type=float,
+        help='records per request')
+    
+    args = parser.parse_args()
+
+    perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second)    
+    perftest.createTablesAndStoreData()
\ No newline at end of file
-- 
GitLab