################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. # # This file is proprietary and confidential to TAOS Technologies. # No part of this file may be reproduced, stored, transmitted, # disclosed or used in any form or by any means other than as # expressly provided by the written permission from Jianhui Tao # ################################################################### # -*- coding: utf-8 -*- import taos import pandas as pd import argparse import os.path import json class taosdemoPerformace: def __init__(self, commitID, dbName): self.commitID = commitID self.dbName = dbName self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" self.config = "/etc/perf" self.conn = taos.connect( self.host, self.user, self.password, self.config) self.insertDB = "insertDB"; def generateJson(self): db = { "name": "%s" % self.insertDB, "drop": "yes", "replica": 1 } stb = { "name": "meters", "child_table_exists":"no", "childtable_count": 10000, "childtable_prefix": "stb_", "auto_create_table": "no", "data_source": "rand", "batch_create_tbl_num": 10, "insert_mode": "taosc", "insert_rows": 100000, "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1, "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", "columns": [ {"type": "INT", "count": 4} ], "tags": [ {"type": "INT", "count":1}, {"type": "BINARY", "len": 16} ] } stables = [] stables.append(stb) db = { "dbinfo": db, "super_tables": stables } insert_data = { "filetype": "insert", "cfgdir": "/etc/perf", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", "thread_count": 10, "thread_count_create_tbl": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 30000, "databases": [db] } insert_json_file = f"/tmp/insert.json" with open(insert_json_file, 'w') as f: json.dump(insert_data, f) return insert_json_file def getCMDOutput(self, cmd): cmd = os.popen(cmd) output = cmd.read() cmd.close() return output def insertData(self): os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") self.commitID = self.getCMDOutput("git rev-parse --short HEAD") delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") self.avgDelay = delay[:-4] delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") self.maxDelay = delay[:-4] delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") self.minDelay = delay[:-3] os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") def createTablesAndStoreData(self): cursor = self.conn.cursor() cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("use %s" % self.dbName) cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") print("create tables time: %f" % float(self.createTableTime)) print("insert records time: %f" % float(self.insertRecordsTime)) print("records per second: %f" % float(self.recordsPerSecond)) print("avg delay: %f" % float(self.avgDelay)) print("max delay: %f" % float(self.maxDelay)) print("min delay: %f" % float(self.minDelay)) cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( '-c', '--commit-id', action='store', type=str, help='git commit id (default: null)') parser.add_argument( '-d', '--database-name', action='store', default='perf', type=str, help='Database name to be created (default: perf)') args = parser.parse_args() perftest = taosdemoPerformace(args.commit_id, args.database_name) perftest.insertData() perftest.createTablesAndStoreData()