diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py index ee611733d077375ed390d43909873b4bdd27278f..d9b8e9ce4a7bc144839334332268ac0f09f78f0d 100644 --- a/tests/pytest/cluster/clusterEnvSetup/basic.py +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -12,141 +12,89 @@ # -*- coding: utf-8 -*- import os +import taos import random import argparse -class ClusterTestcase: +class BuildDockerCluser: + + def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion): + self.hostName = hostName + self.user = user + self.password = password + self.configDir = configDir + self.numOfNodes = numOfNodes + self.clusterVersion = clusterVersion + + def getConnection(self): + self.conn = taos.connect( + host = self.hostName, + user = self.user, + password = self.password, + config = self.configDir) + + def createDondes(self): + self.cursor = self.conn.cursor() + for i in range(2, self.numOfNodes + 1): + self.cursor.execute("create dnode tdnode%d" % i) + + def startArbitrator(self): + print("start arbitrator") + os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator") - def run(self): - os.system("./buildClusterEnv.sh -n 3 -v 2.0.14.1") - os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x") - os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10") + if self.numOfNodes < 2 or self.numOfNodes > 5: + print("the number of nodes must be between 2 and 5") + exit(0) + os.system("./buildClusterEnv.sh -n %d -v %s" % (self.numOfNodes, self.clusterVersion)) + self.getConnection() + self.createDondes() + self.startArbitrator() parser = argparse.ArgumentParser() parser.add_argument( '-H', - '--host-name', + '--host', action='store', - default='127.0.0.1', + default='tdnode1', type=str, - help='host name to be connected (default: 127.0.0.1)') -parser.add_argument( - '-S', - '--ts', - action='store', - default=1500000000000, - type=int, - help='insert data from timestamp (default: 1500000000000)') + help='host name to be connected (default: tdnode1)') parser.add_argument( - '-d', - '--db-name', + '-u', + '--user', action='store', - default='test', + default='root', type=str, - help='Database name to be created (default: test)') -parser.add_argument( - '-t', - '--number-of-native-threads', - action='store', - default=10, - type=int, - help='Number of native threads (default: 10)') -parser.add_argument( - '-T', - '--number-of-rest-threads', - action='store', - default=10, - type=int, - help='Number of rest threads (default: 10)') -parser.add_argument( - '-r', - '--number-of-records', - action='store', - default=100, - type=int, - help='Number of record to be created for each table (default: 100)') -parser.add_argument( - '-c', - '--create-table', - action='store', - default='0', - type=int, - help='whether gen data (default: 0)') + help='user (default: root)') parser.add_argument( '-p', - '--subtb-name-prefix', - action='store', - default='t', - type=str, - help='subtable-name-prefix (default: t)') -parser.add_argument( - '-P', - '--stb-name-prefix', + '--password', action='store', - default='st', + default='taosdata', type=str, - help='stable-name-prefix (default: st)') + help='password (default: taosdata)') parser.add_argument( - '-b', - '--probabilities', - action='store', - default='0.05', - type=float, - help='probabilities of join (default: 0.05)') -parser.add_argument( - '-l', - '--loop-per-thread', + '-c', + '--config-dir', action='store', - default='100', - type=int, - help='loop per thread (default: 100)') -parser.add_argument( - '-u', - '--user', - action='store', - default='root', + default='/etc/taos', type=str, - help='user name') -parser.add_argument( - '-w', - '--password', - action='store', - default='root', - type=str, - help='user name') + help='configuration directory (default: /etc/taos)') parser.add_argument( '-n', - '--number-of-tables', - action='store', - default=1000, - type=int, - help='Number of subtales per stable (default: 1000)') -parser.add_argument( - '-N', - '--number-of-stables', + '--num-of-nodes', action='store', default=2, type=int, - help='Number of stables (default: 2)') + help='number of nodes in the cluster (default: 2, min: 2, max: 5)') parser.add_argument( - '-m', - '--mix-stable-subtable', + '-v', + '--version', action='store', - default=0, - type=int, - help='0:stable & substable ,1:subtable ,2:stable (default: 0)') + default='2.0.14.1', + type=str, + help='the version of the cluster to be build, Default is 2.0.14.1') args = parser.parse_args() -q = ConcurrentInquiry( - args.ts,args.host_name,args.user,args.password,args.db_name, - args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads, - args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records, - args.mix_stable_subtable ) - -if args.create_table: - q.gen_data() -q.get_full() - -#q.gen_query_sql() -q.run() \ No newline at end of file +cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version) +cluster.run() \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh index dfeab8c094575e4bac5f4eb7d17de0af8406d901..968cdd1c1c81b9f6dba68bc2cca542038ada8606 100755 --- a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -32,7 +32,7 @@ function addTaoscfg { do touch /data/node$i/cfg/taos.cfg echo 'firstEp tdnode1:6030' > /data/node$i/cfg/taos.cfg - echo 'fqdn tdnode$i' >> /data/node$i/cfg/taos.cfg + echo 'fqdn tdnode'$i >> /data/node$i/cfg/taos.cfg echo 'arbitrator tdnode1:6042' >> /data/node$i/cfg/taos.cfg done } @@ -64,9 +64,9 @@ function prepareBuild { rm -rf $CURR_DIR/../../../../release/* fi - - if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz && ! -e TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then cd $CURR_DIR/../../../../packaging + echo "generating TDeninger packages" ./release.sh -v edge -n $VERSION >> /dev/null if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then @@ -92,26 +92,28 @@ function prepareBuild { } function clusterUp { + echo "docker compose start" - cd $DOCKER_DIR - - PARAMETERS=PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION + cd $DOCKER_DIR if [ $NUM_OF_NODES -eq 2 ]; then - $PARAMETERS docker-compose up -d + echo "create 2 dnodes" + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose up -d fi if [ $NUM_OF_NODES -eq 3 ]; then - $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml up -d + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml up -d fi if [ $NUM_OF_NODES -eq 4 ]; then - $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d fi if [ $NUM_OF_NODES -eq 5 ]; then - $PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d + PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d fi + + echo "docker compose finish" } createDIR diff --git a/tests/pytest/cluster/clusterEnvSetup/insert.json b/tests/pytest/cluster/clusterEnvSetup/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..56a64b7b8561877cb26b4ef2336ab8b98f26c02c --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/insert.json @@ -0,0 +1,55 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 1, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 2, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 1, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/cluster/clusterEnvSetup/taosdemoxWrapper.py b/tests/pytest/cluster/clusterEnvSetup/taosdemoxWrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ad0696669a7a3fc3bd40a5016eb12fdf24366cba --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/taosdemoxWrapper.py @@ -0,0 +1,142 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import random +import argparse + +class taosdemoxWrapper: + + def __init__(self, host, metadata, database, tables, threads, configDir, replica, + columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen): + self.host = host + self.metadata = metadata + self.database = database + self.tables = tables + self.threads = threads + self.configDir = configDir + self.replica = replica + self.columnType = columnType + self.columnsPerTable = columnsPerTable + self.rowsPerTable = rowsPerTable + self.disorderRatio = disorderRatio + self.disorderRange = disorderRange + self.charTypeLen = charTypeLen + + def run(self): + if self.metadata is None: + os.system("taosdemox -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y" + % (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType, + self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen)) + else: + os.system("taosdemox -f %s" % self.metadata) + + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host-name', + action='store', + default='tdnode1', + type=str, + help='host name to be connected (default: tdnode1)') +parser.add_argument( + '-f', + '--metadata', + action='store', + default=None, + type=str, + help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL') +parser.add_argument( + '-d', + '--db-name', + action='store', + default='test', + type=str, + help='Database name to be created (default: test)') +parser.add_argument( + '-t', + '--num-of-tables', + action='store', + default=10, + type=int, + help='Number of tables (default: 10000)') +parser.add_argument( + '-T', + '--num-of-threads', + action='store', + default=10, + type=int, + help='Number of rest threads (default: 10)') +parser.add_argument( + '-c', + '--config-dir', + action='store', + default='/etc/taos/', + type=str, + help='Configuration directory. (Default is /etc/taos/)') +parser.add_argument( + '-a', + '--replica', + action='store', + default=100, + type=int, + help='Set the replica parameters of the database (default: 1, min: 1, max: 3)') +parser.add_argument( + '-b', + '--column-type', + action='store', + default='int', + type=str, + help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)') +parser.add_argument( + '-l', + '--num-of-cols', + action='store', + default=10, + type=int, + help='The number of columns per record (default: 10)') +parser.add_argument( + '-n', + '--num-of-rows', + action='store', + default=1000, + type=int, + help='Number of subtales per stable (default: 1000)') +parser.add_argument( + '-O', + '--disorder-ratio', + action='store', + default=0, + type=int, + help=' (0: in order, > 0: disorder ratio, default: 0)') +parser.add_argument( + '-R', + '--disorder-range', + action='store', + default=0, + type=int, + help='Out of order datas range, ms (default: 1000)') +parser.add_argument( + '-w', + '--char-type-length', + action='store', + default=16, + type=int, + help='Out of order datas range, ms (default: 16)') + +args = parser.parse_args() +taosdemox = taosdemoxWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables, + args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols, + args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length) +taosdemox.run() \ No newline at end of file