################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. # # This file is proprietary and confidential to TAOS Technologies. # No part of this file may be reproduced, stored, transmitted, # disclosed or used in any form or by any means other than as # expressly provided by the written permission from Jianhui Tao # ################################################################### # -*- coding: utf-8 -*- import sys import os import time import datetime import inspect import psutil import shutil import json from util.log import * from multiprocessing import cpu_count # TODO: fully test the function. Handle exceptions. # Handle json format not accepted by taosdemo class TDTaosdemoCfg: def __init__(self): self.insert_cfg = { "filetype": "insert", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", "thread_count": cpu_count(), "thread_count_create_tbl": cpu_count(), "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, "num_of_records_per_req": 32766, "max_sql_len": 32766, "databases": None } self.db = { "name": 'db', "drop": 'yes', "replica": 1, "days": 10, "cache": 16, "blocks": 6, "precision": "ms", "keep": 3650, "minRows": 100, "maxRows": 4096, "comp": 2, "walLevel": 1, "cachelast": 0, "quorum": 1, "fsync": 3000, "update": 0 } self.query_cfg = { "filetype": "query", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", "confirm_parameter_prompt": "no", "databases": "db", "query_times": 2, "query_mode": "taosc", "specified_table_query": None, "super_table_query": None } self.table_query = { "query_interval": 1, "concurrent": 3, "sqls": None } self.stable_query = { "stblname": "stb", "query_interval": 1, "threads": 3, "sqls": None } self.sub_cfg = { "filetype": "subscribe", "cfgdir": "/etc/taos", "host": "127.0.0.1", "port": 6030, "user": "root", "password": "taosdata", "databases": "db", "confirm_parameter_prompt": "no", "specified_table_query": None, "super_table_query": None } self.table_sub = { "concurrent": 1, "mode": "sync", "interval": 10000, "restart": "yes", "keepProgress": "yes", "sqls": None } self.stable_sub = { "stblname": "stb", "threads": 1, "mode": "sync", "interval": 10000, "restart": "yes", "keepProgress": "yes", "sqls": None } self.stbs = [] self.stb_template = { "name": "stb", "child_table_exists": "no", "childtable_count": 100, "childtable_prefix": "stb_", "auto_create_table": "no", "batch_create_tbl_num": 5, "data_source": "rand", "insert_mode": "taosc", "insert_rows": 100, "childtable_limit": 10, "childtable_offset": 0, "interlace_rows": 0, "insert_interval": 0, "max_sql_len": 32766, "disorder_ratio": 0, "disorder_range": 1000, "timestamp_step": 1, "start_timestamp": "2020-10-01 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", "columns": [{"type": "INT", "count": 1}], "tags": [{"type": "BIGINT", "count": 1}] } self.tb_query_sql = [] self.tb_query_sql_template = { "sql": "select last_row(*) from stb_0 ", "result": "temp/query_res0.txt" } self.stb_query_sql = [] self.stb_query_sql_template = { "sql": "select last_row(ts) from xxxx", "result": "temp/query_res2.txt" } self.tb_sub_sql = [] self.tb_sub_sql_template = { "sql": "select * from stb_0 ;", "result": "temp/subscribe_res0.txt" } self.stb_sub_sql = [] self.stb_sub_sql_template = { "sql": "select * from xxxx where ts > '2021-02-25 11:35:00.000' ;", "result": "temp/subscribe_res1.txt" } # The following functions are import functions for different dicts and lists # except import_sql, all other import functions will a dict and overwrite the origional dict # dict_in: the dict used to overwrite the target def import_insert_cfg(self, dict_in): self.insert_cfg = dict_in def import_db(self, dict_in): self.db = dict_in def import_stbs(self, dict_in): self.stbs = dict_in def import_query_cfg(self, dict_in): self.query_cfg = dict_in def import_table_query(self, dict_in): self.table_query = dict_in def import_stable_query(self, dict_in): self.stable_query = dict_in def import_sub_cfg(self, dict_in): self.sub_cfg = dict_in def import_table_sub(self, dict_in): self.table_sub = dict_in def import_stable_sub(self, dict_in): self.stable_sub = dict_in def import_sql(self, Sql_in, mode): """used for importing the sql later used Args: Sql_in (dict): the imported sql dict mode (str): the sql storing location within TDTaosdemoCfg format: 'fileType_tableType' fileType: query, sub tableType: table, stable """ if mode == 'query_table': self.tb_query_sql = Sql_in elif mode == 'query_stable': self.stb_query_sql = Sql_in elif mode == 'sub_table': self.tb_sub_sql = Sql_in elif mode == 'sub_stable': self.stb_sub_sql = Sql_in # import functions end # The following functions are alter functions for different dicts # Args: # key: the key that is going to be modified # value: the value of the key that is going to be modified # if key = 'databases' | "specified_table_query" | "super_table_query"|"sqls" # value will not be used def alter_insert_cfg(self, key, value): if key == 'databases': self.insert_cfg[key] = [ { 'dbinfo': self.db, 'super_tables': self.stbs } ] else: self.insert_cfg[key] = value def alter_db(self, key, value): self.db[key] = value def alter_query_tb(self, key, value): if key == "sqls": self.table_query[key] = self.tb_query_sql else: self.table_query[key] = value def alter_query_stb(self, key, value): if key == "sqls": self.stable_query[key] = self.stb_query_sql else: self.stable_query[key] = value def alter_query_cfg(self, key, value): if key == "specified_table_query": self.query_cfg["specified_table_query"] = self.table_query elif key == "super_table_query": self.query_cfg["super_table_query"] = self.stable_query else: self.table_query[key] = value def alter_sub_cfg(self, key, value): if key == "specified_table_query": self.sub_cfg["specified_table_query"] = self.table_sub elif key == "super_table_query": self.sub_cfg["super_table_query"] = self.stable_sub else: self.table_query[key] = value def alter_sub_stb(self, key, value): if key == "sqls": self.stable_sub[key] = self.stb_sub_sql else: self.stable_sub[key] = value def alter_sub_tb(self, key, value): if key == "sqls": self.table_sub[key] = self.tb_sub_sql else: self.table_sub[key] = value # alter function ends # the following functions are for handling the sql lists def append_sql_stb(self, target, value): """for appending sql dict into specific sql list Args: target (str): the target append list format: 'fileType_tableType' fileType: query, sub tableType: table, stable unique: 'insert_stbs' value (dict): the sql dict going to be appended """ if target == 'insert_stbs': self.stbs.append(value) elif target == 'query_table': self.tb_query_sql.append(value) elif target == 'query_stable': self.stb_query_sql.append(value) elif target == 'sub_table': self.tb_sub_sql.append(value) elif target == 'sub_stable': self.stb_sub_sql.append(value) def pop_sql_stb(self, target, index): """for poping a sql dict from specific sql list Args: target (str): the target append list format: 'fileType_tableType' fileType: query, sub tableType: table, stable unique: 'insert_stbs' index (int): the sql dict that is going to be popped """ if target == 'insert_stbs': self.stbs.pop(index) elif target == 'query_table': self.tb_query_sql.pop(index) elif target == 'query_stable': self.stb_query_sql.pop(index) elif target == 'sub_table': self.tb_sub_sql.pop(index) elif target == 'sub_stable': self.stb_sub_sql.pop(index) # sql list modification function end # The following functions are get functions for different dicts def get_db(self): return self.db def get_stb(self): return self.stbs def get_insert_cfg(self): return self.insert_cfg def get_query_cfg(self): return self.query_cfg def get_tb_query(self): return self.table_query def get_stb_query(self): return self.stable_query def get_sub_cfg(self): return self.sub_cfg def get_tb_sub(self): return self.table_sub def get_stb_sub(self): return self.stable_sub def get_sql(self, target): """general get function for all sql lists Args: target (str): the sql list want to get format: 'fileType_tableType' fileType: query, sub tableType: table, stable unique: 'insert_stbs' """ if target == 'query_table': return self.tb_query_sql elif target == 'query_stable': return self.stb_query_sql elif target == 'sub_table': return self.tb_sub_sql elif target == 'sub_stable': return self.stb_sub_sql def get_template(self, target): """general get function for the default sql template Args: target (str): the sql list want to get format: 'fileType_tableType' fileType: query, sub tableType: table, stable unique: 'insert_stbs' """ if target == 'insert_stbs': return self.stb_template elif target == 'query_table': return self.tb_query_sql_template elif target == 'query_stable': return self.stb_query_sql_template elif target == 'sub_table': return self.tb_sub_sql_template elif target == 'sub_stable': return self.stb_sub_sql_template else: print(f'did not find {target}') # the folloing are the file generation functions """defalut document: generator functio for generating taosdemo json file will assemble the dicts and dump the final json Args: pathName (str): the directory wanting the json file to be fileName (str): the name suffix of the json file Returns: str: [pathName]/[filetype]_[filName].json """ def generate_insert_cfg(self, pathName, fileName): cfgFileName = f'{pathName}/insert_{fileName}.json' self.alter_insert_cfg('databases', None) with open(cfgFileName, 'w') as file: json.dump(self.insert_cfg, file) return cfgFileName def generate_query_cfg(self, pathName, fileName): cfgFileName = f'{pathName}/query_{fileName}.json' self.alter_query_tb('sqls', None) self.alter_query_stb('sqls', None) self.alter_query_cfg('specified_table_query', None) self.alter_query_cfg('super_table_query', None) with open(cfgFileName, 'w') as file: json.dump(self.query_cfg, file) return cfgFileName def generate_subscribe_cfg(self, pathName, fileName): cfgFileName = f'{pathName}/subscribe_{fileName}.json' self.alter_sub_tb('sqls', None) self.alter_sub_stb('sqls', None) self.alter_sub_cfg('specified_table_query', None) self.alter_sub_cfg('super_table_query', None) with open(cfgFileName, 'w') as file: json.dump(self.sub_cfg, file) return cfgFileName # file generation functions ends def drop_cfg_file(self, fileName): os.remove(f'{fileName}') taosdemoCfg = TDTaosdemoCfg()