[TD-5114]<test>: add testcase of rollUpgrading

tests/pytest/manualTest/TD-5114/continueCreateDn.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+sys.path.insert(0, os.getcwd())
+from util.log import *
+from util.sql import *
+from util.dnodes import *
+import taos
+import threading
+import subprocess
+from random import choice
+ 
+class TwoClients:
+    def initConnection(self):
+        self.host = "chr03"
+        self.user = "root"
+        self.password = "taosdata"
+        self.config = "/etc/taos/"     
+        self.port =6030 
+        self.rowNum = 10
+        self.ts = 1537146000000  
+
+    def run(self):
+        
+        # new taos client
+        conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config )
+        print(conn1)
+        cur1 = conn1.cursor()
+        tdSql.init(cur1, True)
+
+        tdSql.execute("drop database if exists db3")
+
+        # insert data with taosc 
+        for i in range(10):
+            os.system("taosdemo -f manualTest/TD-5114/insertDataDb3Replica2.json -y ")
+        # # check data correct 
+        tdSql.execute("show databases")
+        tdSql.execute("use db3")
+        tdSql.query("select count (tbname) from stb0")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query("select count (*) from stb0")
+        tdSql.checkData(0, 0, 4000000)
+
+        # insert data with  python connector , if you want to use this case ,cancel note.
+        
+        # for x in range(10):
+        #     dataType= [ "tinyint", "smallint",  "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned",  "bigint unsigned"] 
+        #     tdSql.execute("drop database if exists db3") 
+        #     tdSql.execute("create database db3 keep 3650  replica 2 ")
+        #     tdSql.execute("use db3")
+        #     tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, 
+        #                 col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(3000), tag1 int)''')
+        #     rowNum2= 988
+        #     for i in range(rowNum2):
+        #         tdSql.execute("alter table test add column col%d  %s ;" %( i+14, choice(dataType))     ) 
+        #     rowNum3= 988
+        #     for i in range(rowNum3):
+        #         tdSql.execute("alter table test drop column col%d   ;" %( i+14)     )    
+        #     self.rowNum = 50
+        #     self.rowNum2 = 2000
+        #     self.ts = 1537146000000
+        #     for j in range(self.rowNum2):
+        #         tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) )
+        #         for i in range(self.rowNum):
+        #             tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" 
+        #                     % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1))
+
+        # # check data correct 
+        # tdSql.execute("show databases")
+        # tdSql.execute("use db3")
+        # tdSql.query("select count (tbname) from test")
+        # tdSql.checkData(0, 0, 200) 
+        # tdSql.query("select count (*) from test")
+        # tdSql.checkData(0, 0, 200000) 
+
+
+        # delete useless file
+        testcaseFilename = os.path.split(__file__)[-1]
+        os.system("rm -rf ./insert_res.txt")
+        os.system("rm -rf manualTest/TD-5114/%s.sql" % testcaseFilename )       
+   
+clients = TwoClients()
+clients.initConnection()
+# clients.getBuildPath()
+clients.run()
\ No newline at end of file

tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 0,
+    "num_of_records_per_req": 3000,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db3",
+            "drop": "yes",
+            "replica": 2,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb0",
+            "child_table_exists":"no",
+            "childtable_count": 20000,
+            "childtable_prefix": "stb0_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 1000,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 2000,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}

tests/pytest/manualTest/TD-5114/rollingUpgrade.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from sys import version
+from fabric import Connection
+import random
+import time
+import datetime
+import logging
+import subprocess
+import os
+import sys
+
+class Node:
+    def __init__(self, index, username, hostIP, password, version):
+        self.index = index        
+        self.username = username
+        self.hostIP = hostIP
+        # self.hostName = hostName
+        # self.homeDir = homeDir
+        self.version = version
+        self.verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % self.version
+        self.installPath = "TDengine-enterprise-server-%s" % self.version
+        # self.corePath = '/coredump'
+        self.conn = Connection("{}@{}".format(username, hostIP), connect_kwargs={"password": "{}".format(password)})    
+
+
+    def buildTaosd(self):
+        try:
+            print(self.conn)
+            # self.conn.run('echo "1234" > /home/chr/installtest/test.log')
+            self.conn.run("cd /home/chr/installtest/ && tar -xvf  %s " %self.verName)
+            self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
+        except Exception as e:
+            print("Build Taosd error for node %d " % self.index)
+            logging.exception(e)
+            pass
+
+    def rebuildTaosd(self):
+        try:
+            print(self.conn)
+            # self.conn.run('echo "1234" > /home/chr/installtest/test.log')
+            self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath)
+        except Exception as e:
+            print("Build Taosd error for node %d " % self.index)
+            logging.exception(e)
+            pass
+
+    def startTaosd(self):
+        try:
+            self.conn.run("sudo systemctl start taosd")
+        except Exception as e:
+            print("Start Taosd error for node %d " % self.index)
+            logging.exception(e)
+
+    def restartTarbi(self):
+        try:
+            self.conn.run("sudo systemctl restart tarbitratord  ")
+        except Exception as e:
+            print("Start Taosd error for node %d " % self.index)
+            logging.exception(e)
+
+    def clearData(self):
+        timeNow = datetime.datetime.now()
+        # timeYes = datetime.datetime.now() + datetime.timedelta(days=-1)
+        timStr = timeNow.strftime('%Y%m%d%H%M%S')
+        # timStr = timeNow.strftime('%Y%m%d%H%M%S')
+        try:
+            # self.conn.run("mv /var/lib/taos/ /var/lib/taos%s " % timStr)
+            self.conn.run("rm -rf  /home/chr/data/taos*")
+        except Exception as e:
+            print("rm -rf  /var/lib/taos error %d " % self.index)
+            logging.exception(e)
+
+    def stopTaosd(self):
+        try:
+            self.conn.run("sudo systemctl stop taosd")
+        except Exception as e:
+            print("Stop Taosd error for node %d " % self.index)
+            logging.exception(e)
+            
+    def restartTaosd(self):
+        try:
+            self.conn.run("sudo systemctl restart taosd")
+        except Exception as e:
+            print("Stop Taosd error for node %d " % self.index)
+            logging.exception(e)
+
+class oneNode:     
+
+    def FirestStartNode(self, id, username, IP, passwd, version):
+        # get installPackage
+        verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version
+        # installPath = "TDengine-enterprise-server-%s" % self.version
+        node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0')
+        node131.conn.run('sshpass  -p  tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id))
+        node131.conn.close() 
+        # install TDengine at 192.168.103/104/141
+        try:
+            node = Node(id, username, IP, passwd, version)
+            node.conn.run('echo "start taosd"')
+            node.buildTaosd()
+            # clear DataPath  , if need clear data 
+            node.clearData()
+            node.startTaosd()
+            if id == 103 :
+                node.restartTarbi()
+            print("start taosd ver:%s node:%d successfully " % (version,id))
+            node.conn.close() 
+
+            # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+            # assert query_pid == 1 , "node %d: start taosd  failed " % id
+        except Exception as e:
+            print("Stop Taosd error for node %d " % id)
+            logging.exception(e)
+
+    def startNode(self, id, username, IP, passwd, version):
+        # start TDengine 
+        try:
+            node = Node(id, username, IP, passwd, version)
+            node.conn.run('echo "restart taosd"')
+            # clear DataPath  , if need clear data 
+            node.clearData()
+            node.restartTaosd()
+            time.sleep(5)
+            if id == 103 :
+                node.restartTarbi()
+            print("start taosd ver:%s node:%d successfully " % (version,id))
+            node.conn.close() 
+
+            # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+            # assert query_pid == 1 , "node %d: start taosd  failed " % id
+        except Exception as e:
+            print("Stop Taosd error for node %d " % id)
+            logging.exception(e)
+
+    def firstUpgradeNode(self, id, username, IP, passwd, version):
+        # get installPackage
+        verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version
+        # installPath = "TDengine-enterprise-server-%s" % self.version
+        node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0')
+        node131.conn.run('echo "upgrade cluster"')
+        node131.conn.run('sshpass  -p  tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id))
+        node131.conn.close() 
+        # upgrade TDengine at 192.168.103/104/141
+        try:
+            node = Node(id, username, IP, passwd, version)
+            node.conn.run('echo "start taosd"')
+            node.conn.run('echo "1234" > /home/chr/test.log')
+            node.buildTaosd()
+            time.sleep(5)
+            node.startTaosd()
+            if id == 103 :
+                node.restartTarbi()
+            print("start taosd ver:%s node:%d successfully " % (version,id))
+            node.conn.close() 
+
+            # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
+            # assert query_pid == 1 , "node %d: start taosd  failed " % id
+        except Exception as e:
+            print("Stop Taosd error for node %d " % id)
+            logging.exception(e)
+    
+    def  upgradeNode(self, id, username, IP, passwd, version):
+
+        # backCluster TDengine at 192.168.103/104/141
+        try:
+            node = Node(id, username, IP, passwd, version)
+            node.conn.run('echo "rollback taos"')
+            node.rebuildTaosd()
+            time.sleep(5)
+            node.startTaosd()
+            if id == 103 :
+                node.restartTarbi()
+            print("start taosd ver:%s node:%d successfully " % (version,id))
+            node.conn.close() 
+        except Exception as e:
+            print("Stop Taosd error for node %d " % id)
+            logging.exception(e)
+
+
+# how  to use : cd TDinternal/commumity/test/pytest && python3 manualTest/rollingUpgrade.py  ,when  inserting data, we can start " python3 manualTest/rollingUpagrade.py". add example "oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')"
+
+
+# node103=oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
+# node104=oneNode().FirestStartNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
+# node141=oneNode().FirestStartNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
+
+# node103=oneNode().startNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
+# time.sleep(30)
+# node141=oneNode().startNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
+# time.sleep(30)
+# node104=oneNode().startNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
+# time.sleep(30)
+
+# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5')
+# time.sleep(30)
+# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5')
+# time.sleep(30)
+# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5')
+# time.sleep(30)
+
+# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
+# time.sleep(30)
+# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
+# time.sleep(30)
+# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
+# time.sleep(30)
+
+# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
+# time.sleep(30)
+# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
+# time.sleep(30)
+# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')
+# time.sleep(30)
+
+
+
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')
+# time.sleep(120)
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.0')
+# time.sleep(180)
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.0')
+# time.sleep(240)
+
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5')
+# time.sleep(120)
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5')
+# time.sleep(120)
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5')
+# time.sleep(180)
+
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
+# time.sleep(120)
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
+# time.sleep(120)
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
+# time.sleep(180)
+
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
+# time.sleep(180)
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
+# time.sleep(180)
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')
+
+
+# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.9')
+# time.sleep(5)
+# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.9')
+# time.sleep(5)
+# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.9')
+# time.sleep(30)
+
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10')
+# time.sleep(12)
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10')
+# time.sleep(12)
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10')
+# time.sleep(180)
+
+# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12')
+# time.sleep(120)
+# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12')
+# time.sleep(120)
+# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12')