clusterSetup.py

###################################################################
#           Copyright (c) 2016 by TAOS Technologies, Inc.
#                     All rights reserved.
#
#  This file is proprietary and confidential to TAOS Technologies.
#  No part of this file may be reproduced, stored, transmitted,
#  disclosed or used in any form or by any means other than as
#  expressly provided by the written permission from Jianhui Tao
#
###################################################################

# -*- coding: utf-8 -*-

from fabric import Connection
import random
import time
import logging

class Node:
    def __init__(self, index, username, hostIP, hostName, password, homeDir):
        self.index = index        
        self.username = username
        self.hostIP = hostIP
        self.hostName = hostName
        self.homeDir = homeDir
        self.corePath = '/coredump'
        self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)})        
    
    def buildTaosd(self):
        try:
            self.conn.cd("/root/TDinternal/community")
            self.conn.run("git checkout develop")
            self.conn.run("git pull")
            self.conn.cd("/root/TDinternal")
            self.conn.run("git checkout develop")
            self.conn.run("git pull")
            self.conn.cd("/root/TDinternal/debug")
            self.conn.run("cmake ..")
            self.conn.run("make")
            self.conn.run("make install")
        except Exception as e:
            print("Build Taosd error for node %d " % self.index)
            logging.exception(e)
            pass

    def startTaosd(self):
        try:
            self.conn.run("sudo systemctl start taosd")
        except Exception as e:
            print("Start Taosd error for node %d " % self.index)
            logging.exception(e)
        
    def stopTaosd(self):
        try:
            self.conn.run("sudo systemctl stop taosd")
        except Exception as e:
            print("Stop Taosd error for node %d " % self.index)
            logging.exception(e)
    
    def restartTaosd(self):
        try:
            self.conn.run("sudo systemctl restart taosd")
        except Exception as e:
            print("Stop Taosd error for node %d " % self.index)
            logging.exception(e)    

    def removeTaosd(self):
        try:
            self.conn.run("rmtaos")
        except Exception as e:
            print("remove taosd error for node %d " % self.index)
            logging.exception(e)
    
    def forceStopOneTaosd(self):
        try:
            self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')")
        except Exception as e:
            print("kill taosd error on node%d " % self.index)            
    
    def startOneTaosd(self):
        try:
            self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &")
        except Exception as e:
            print("start taosd error on node%d " % self.index)
            logging.exception(e)    
    
    def installTaosd(self, packagePath):
        self.conn.put(packagePath, self.homeDir)
        self.conn.cd(self.homeDir)
        self.conn.run("tar -zxf $(basename '%s')" % packagePath)
        with self.conn.cd("TDengine-enterprise-server"):
            self.conn.run("yes|./install.sh")

    def configTaosd(self, taosConfigKey, taosConfigValue):
        self.conn.run("sudo echo '%s %s' >> %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))

    def removeTaosConfig(self, taosConfigKey, taosConfigValue): 
        self.conn.run("sudo sed -in-place -e '/%s %s/d' %s" % (taosConfigKey, taosConfigValue, "/etc/taos/taos.cfg"))
    
    def configHosts(self, ip, name):
        self.conn.run("echo '%s %s' >> %s" % (ip, name, '/etc/hosts'))

    def removeData(self):
        try:
            self.conn.run("sudo rm -rf /var/lib/taos/*")
        except Exception as e:
            print("remove taosd data error for node %d " % self.index)
            logging.exception(e)
    
    def removeLog(self):
        try:
            self.conn.run("sudo rm -rf /var/log/taos/*")
        except Exception as e:
            print("remove taosd error for node %d " % self.index)
            logging.exception(e)

    def removeDataForMnode(self):
        try:
            self.conn.run("sudo rm -rf /var/lib/taos/*")
        except Exception as e:
            print("remove taosd error for node %d " % self.index)
            logging.exception(e)

    def removeDataForVnode(self, id):
        try:
            self.conn.run("sudo rm -rf /var/lib/taos/vnode%d/*.data" % id)
        except Exception as e:
            print("remove taosd error for node %d " % self.index)
            logging.exception(e)
    

    def detectCoredumpFile(self):
        try:
            result = self.conn.run("find /coredump -name 'core_*' ", hide=True)
            output = result.stdout
            print("output: %s" % output)
            return output
        except Exception as e:
            print("find coredump file error on node %d " % self.index)
            logging.exception(e)
        

class Nodes:
    def __init__(self):
        self.tdnodes = []
        self.tdnodes.append(Node(0, 'root', '192.168.17.194', 'taosdata', 'r', '/root/'))
        # self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/'))
        # self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/'))
        # self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/'))
        # self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/'))

    def stopOneNode(self, index):
        self.tdnodes[index].stopTaosd()
        self.tdnodes[index].forceStopOneTaosd()
    
    def startOneNode(self, index):
        self.tdnodes[index].startOneTaosd()
    
    def detectCoredumpFile(self, index):
        return self.tdnodes[index].detectCoredumpFile()

    def stopAllTaosd(self):
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].stopTaosd()

    def startAllTaosd(self):
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].startTaosd()                    
    
    def restartAllTaosd(self):
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].restartTaosd()       
    
    def addConfigs(self, configKey, configValue):          
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].configTaosd(configKey, configValue)        
    
    def removeConfigs(self, configKey, configValue): 
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].removeTaosConfig(configKey, configValue)  
    
    def removeAllDataFiles(self):
        for i in range(len(self.tdnodes)):
            self.tdnodes[i].removeData()

class Test:
    def __init__(self):
        self.nodes = Nodes()

    # kill taosd randomly every 10 mins
    def randomlyKillDnode(self):        
        loop = 0
        while True:                
            index = random.randint(0, 4)
            print("loop: %d, kill taosd on node%d" %(loop, index))
            self.nodes.stopOneNode(index)
            time.sleep(60)
            self.nodes.startOneNode(index)
            time.sleep(600)
            loop = loop + 1
    
    def detectCoredump(self):
        loop = 0
        while True:
            for i in range(len(self.nodes.tdnodes)):
                result = self.nodes.detectCoredumpFile(i)
                print("core file path is %s" % result)
                if result and not result.isspace():
                    self.nodes.stopAllTaosd()                    
            print("sleep for 10 mins")
            time.sleep(600)

test = Test()
test.detectCoredump()