Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
提交 e53e414f 编写于 作者: P Ping Xiao
浏览文件
操作

[TD-2846]<test>: update docker cluster scripts

上级 42d5896b
隐藏空白更改
内联 并排
Showing with 265 addition and 118 deletion
tests/pytest/cluster/clusterEnvSetup/basic.py
浏览文件 @ e53e414f
......@@ -12,141 +12,89 @@
# -*- coding: utf-8 -*-
import os
import taos
import random
import argparse
class ClusterTestcase:
class BuildDockerCluser:
def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion):
self.hostName = hostName
self.user = user
self.password = password
self.configDir = configDir
self.numOfNodes = numOfNodes
self.clusterVersion = clusterVersion
def getConnection(self):
self.conn = taos.connect(
host = self.hostName,
user = self.user,
password = self.password,
config = self.configDir)
def createDondes(self):
self.cursor = self.conn.cursor()
for i in range(2, self.numOfNodes + 1):
self.cursor.execute("create dnode tdnode%d" % i)
def startArbitrator(self):
print("start arbitrator")
os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator")
def run(self):
os.system("./buildClusterEnv.sh -n 3 -v 2.0.14.1")
os.system("yes|taosdemo -h 172.27.0.7 -n 100 -t 100 -x")
os.system("python3 ../../concurrent_inquiry.py -H 172.27.0.7 -T 4 -t 4 -l 10")
if self.numOfNodes < 2 or self.numOfNodes > 5:
print("the number of nodes must be between 2 and 5")
exit(0)
os.system("./buildClusterEnv.sh -n %d -v %s" % (self.numOfNodes, self.clusterVersion))
self.getConnection()
self.createDondes()
self.startArbitrator()
parser = argparse.ArgumentParser()
parser.add_argument(
'-H',
'--host-name',
'--host',
action='store',
default='127.0.0.1',
default='tdnode1',
type=str,
help='host name to be connected (default: 127.0.0.1)')
parser.add_argument(
'-S',
'--ts',
action='store',
default=1500000000000,
type=int,
help='insert data from timestamp (default: 1500000000000)')
help='host name to be connected (default: tdnode1)')
parser.add_argument(
'-d',
'--db-name',
'-u',
'--user',
action='store',
default='test',
default='root',
type=str,
help='Database name to be created (default: test)')
parser.add_argument(
'-t',
'--number-of-native-threads',
action='store',
default=10,
type=int,
help='Number of native threads (default: 10)')
parser.add_argument(
'-T',
'--number-of-rest-threads',
action='store',
default=10,
type=int,
help='Number of rest threads (default: 10)')
parser.add_argument(
'-r',
'--number-of-records',
action='store',
default=100,
type=int,
help='Number of record to be created for each table (default: 100)')
parser.add_argument(
'-c',
'--create-table',
action='store',
default='0',
type=int,
help='whether gen data (default: 0)')
help='user (default: root)')
parser.add_argument(
'-p',
'--subtb-name-prefix',
action='store',
default='t',
type=str,
help='subtable-name-prefix (default: t)')
parser.add_argument(
'-P',
'--stb-name-prefix',
'--password',
action='store',
default='st',
default='taosdata',
type=str,
help='stable-name-prefix (default: st)')
help='password (default: taosdata)')
parser.add_argument(
'-b',
'--probabilities',
action='store',
default='0.05',
type=float,
help='probabilities of join (default: 0.05)')
parser.add_argument(
'-l',
'--loop-per-thread',
'-c',
'--config-dir',
action='store',
default='100',
type=int,
help='loop per thread (default: 100)')
parser.add_argument(
'-u',
'--user',
action='store',
default='root',
default='/etc/taos',
type=str,
help='user name')
parser.add_argument(
'-w',
'--password',
action='store',
default='root',
type=str,
help='user name')
help='configuration directory (default: /etc/taos)')
parser.add_argument(
'-n',
'--number-of-tables',
action='store',
default=1000,
type=int,
help='Number of subtales per stable (default: 1000)')
parser.add_argument(
'-N',
'--number-of-stables',
'--num-of-nodes',
action='store',
default=2,
type=int,
help='Number of stables (default: 2)')
help='number of nodes in the cluster (default: 2, min: 2, max: 5)')
parser.add_argument(
'-m',
'--mix-stable-subtable',
'-v',
'--version',
action='store',
default=0,
type=int,
help='0:stable & substable ,1:subtable ,2:stable (default: 0)')
default='2.0.14.1',
type=str,
help='the version of the cluster to be build, Default is 2.0.14.1')
args = parser.parse_args()
q = ConcurrentInquiry(
args.ts,args.host_name,args.user,args.password,args.db_name,
args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads,
args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records,
args.mix_stable_subtable )
if args.create_table:
q.gen_data()
q.get_full()
#q.gen_query_sql()
q.run()
\ No newline at end of file
cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version)
cluster.run()
\ No newline at end of file
tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh
浏览文件 @ e53e414f
......@@ -32,7 +32,7 @@ function addTaoscfg {
do
touch /data/node$i/cfg/taos.cfg
echo 'firstEp tdnode1:6030' > /data/node$i/cfg/taos.cfg
echo 'fqdn tdnode$i' >> /data/node$i/cfg/taos.cfg
echo 'fqdn tdnode'$i >> /data/node$i/cfg/taos.cfg
echo 'arbitrator tdnode1:6042' >> /data/node$i/cfg/taos.cfg
done
}
......@@ -64,9 +64,9 @@ function prepareBuild {
rm -rf $CURR_DIR/../../../../release/*
fi
if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz && ! -e TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
cd $CURR_DIR/../../../../packaging
echo "generating TDeninger packages"
./release.sh -v edge -n $VERSION >> /dev/null
if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then
......@@ -92,26 +92,28 @@ function prepareBuild {
}
function clusterUp {
echo "docker compose start"
cd $DOCKER_DIR
PARAMETERS=PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION
cd $DOCKER_DIR
if [ $NUM_OF_NODES -eq 2 ]; then
$PARAMETERS docker-compose up -d
echo "create 2 dnodes"
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose up -d
fi
if [ $NUM_OF_NODES -eq 3 ]; then
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml up -d
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml up -d
fi
if [ $NUM_OF_NODES -eq 4 ]; then
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml up -d
fi
if [ $NUM_OF_NODES -eq 5 ]; then
$PARAMETERS docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION docker-compose -f docker-compose.yml -f node3.yml -f node4.yml -f node5.yml up -d
fi
echo "docker compose finish"
}
createDIR
......
tests/pytest/cluster/clusterEnvSetup/insert.json 0 → 100644
浏览文件 @ e53e414f
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 4,
"thread_count_create_tbl": 1,
"result_file": "./insert_res.txt",
"confirm_parameter_prompt": "no",
"databases": [{
"dbinfo": {
"name": "db",
"drop": "no",
"replica": 1,
"days": 2,
"cache": 16,
"blocks": 8,
"precision": "ms",
"keep": 365,
"minRows": 100,
"maxRows": 4096,
"comp":2,
"walLevel":1,
"quorum":1,
"fsync":3000,
"update": 0
},
"super_tables": [{
"name": "stb",
"child_table_exists":"no",
"childtable_count": 1,
"childtable_prefix": "stb_",
"auto_create_table": "no",
"data_source": "rand",
"insert_mode": "taosc",
"insert_rate": 0,
"insert_rows": 100000,
"multi_thread_write_one_tbl": "no",
"number_of_tbl_in_one_sql": 1,
"rows_per_tbl": 100,
"max_sql_len": 1024000,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 10,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"tags_file": "",
"columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
"tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
}]
}]
}
tests/pytest/cluster/clusterEnvSetup/taosdemoxWrapper.py 0 → 100644
浏览文件 @ e53e414f
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import random
import argparse
class taosdemoxWrapper:
def __init__(self, host, metadata, database, tables, threads, configDir, replica,
columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen):
self.host = host
self.metadata = metadata
self.database = database
self.tables = tables
self.threads = threads
self.configDir = configDir
self.replica = replica
self.columnType = columnType
self.columnsPerTable = columnsPerTable
self.rowsPerTable = rowsPerTable
self.disorderRatio = disorderRatio
self.disorderRange = disorderRange
self.charTypeLen = charTypeLen
def run(self):
if self.metadata is None:
os.system("taosdemox -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y"
% (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType,
self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen))
else:
os.system("taosdemox -f %s" % self.metadata)
parser = argparse.ArgumentParser()
parser.add_argument(
'-H',
'--host-name',
action='store',
default='tdnode1',
type=str,
help='host name to be connected (default: tdnode1)')
parser.add_argument(
'-f',
'--metadata',
action='store',
default=None,
type=str,
help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL')
parser.add_argument(
'-d',
'--db-name',
action='store',
default='test',
type=str,
help='Database name to be created (default: test)')
parser.add_argument(
'-t',
'--num-of-tables',
action='store',
default=10,
type=int,
help='Number of tables (default: 10000)')
parser.add_argument(
'-T',
'--num-of-threads',
action='store',
default=10,
type=int,
help='Number of rest threads (default: 10)')
parser.add_argument(
'-c',
'--config-dir',
action='store',
default='/etc/taos/',
type=str,
help='Configuration directory. (Default is /etc/taos/)')
parser.add_argument(
'-a',
'--replica',
action='store',
default=100,
type=int,
help='Set the replica parameters of the database (default: 1, min: 1, max: 3)')
parser.add_argument(
'-b',
'--column-type',
action='store',
default='int',
type=str,
help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)')
parser.add_argument(
'-l',
'--num-of-cols',
action='store',
default=10,
type=int,
help='The number of columns per record (default: 10)')
parser.add_argument(
'-n',
'--num-of-rows',
action='store',
default=1000,
type=int,
help='Number of subtales per stable (default: 1000)')
parser.add_argument(
'-O',
'--disorder-ratio',
action='store',
default=0,
type=int,
help=' (0: in order, > 0: disorder ratio, default: 0)')
parser.add_argument(
'-R',
'--disorder-range',
action='store',
default=0,
type=int,
help='Out of order datas range, ms (default: 1000)')
parser.add_argument(
'-w',
'--char-type-length',
action='store',
default=16,
type=int,
help='Out of order datas range, ms (default: 16)')
args = parser.parse_args()
taosdemox = taosdemoxWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables,
args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols,
args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length)
taosdemox.run()
\ No newline at end of file
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或