test.py

#!/usr/bin/python
###################################################################
#           Copyright (c) 2016 by TAOS Technologies, Inc.
#                     All rights reserved.
#
#  This file is proprietary and confidential to TAOS Technologies.
#  No part of this file may be reproduced, stored, transmitted,
#  disclosed or used in any form or by any means other than as
#  expressly provided by the written permission from Jianhui Tao
#
###################################################################
# install pip
# pip install src/connector/python/

# -*- coding: utf-8 -*-
import sys
import getopt
import subprocess
import time
from distutils.log import warn as printf
from fabric2 import Connection

from util.log import *
from util.dnodes import *
from util.cases import *
from util.dockerNodes import *

import taos


if __name__ == "__main__":
    fileName = "all"
    deployPath = ""
    masterIp = ""
    testCluster = False
    valgrind = 0
    logSql = True
    stop = 0
    restart = False
    docker = False
    dataDir = "/data"
    windows = 0
    opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:d:p:m:l:scghrw', [
        'file=', 'docker=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows'])
    for key, value in opts:
        if key in ['-h', '--help']:
            tdLog.printNoPrefix(
                'A collection of test cases written using Python')
            tdLog.printNoPrefix('-f Name of test case file written by Python')
            tdLog.printNoPrefix('-d docker cluster test')
            tdLog.printNoPrefix('-p Deploy Path for Simulator')
            tdLog.printNoPrefix('-m Master Ip for Simulator')
            tdLog.printNoPrefix('-l <True:False> logSql Flag')
            tdLog.printNoPrefix('-s stop All dnodes')
            tdLog.printNoPrefix('-c Test Cluster Flag')
            tdLog.printNoPrefix('-g valgrind Test Flag')
            tdLog.printNoPrefix('-r taosd restart test')
            tdLog.printNoPrefix('-w taos on windows')            
            sys.exit(0)

        if key in ['-r', '--restart']:
            restart = True

        if key in ['-f', '--file']:
            fileName = os.path.normpath(value)
            print(fileName)
        
        if key in ['-d', '--docker']:
            fileName = os.path.normpath(value)            
            docker = True

        if key in ['-p', '--path']:
            deployPath = value

        if key in ['-m', '--master']:
            masterIp = value

        if key in ['-l', '--logSql']:
            if (value.upper() == "TRUE"):
                logSql = True
            elif (value.upper() == "FALSE"):
                logSql = False
            else:
                tdLog.printNoPrefix("logSql value %s is invalid" % logSql)
                sys.exit(0)

        if key in ['-c', '--cluster']:
            testCluster = True

        if key in ['-g', '--valgrind']:
            valgrind = 1

        if key in ['-s', '--stop']:
            stop = 1        

        if key in ['-w', '--windows']:
            windows = 1

    if (stop != 0):
        if (valgrind == 0):
            toBeKilled = "taosd"
        else:
            toBeKilled = "valgrind.bin"

        killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled

        psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
        processID = subprocess.check_output(psCmd, shell=True)

        while(processID):
            os.system(killCmd)
            time.sleep(1)
            processID = subprocess.check_output(psCmd, shell=True)

        for port in range(6030, 6041):
            usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port
            processID = subprocess.check_output(usePortPID, shell=True)

            if processID:
                killCmd = "kill -TERM %s" % processID
                os.system(killCmd)
            fuserCmd = "fuser -k -n tcp %d" % port
            os.system(fuserCmd)
        if valgrind:
            time.sleep(2)

        tdLog.info('stop All dnodes')

    if masterIp == "":
        host = '127.0.0.1'
    else:
        host = masterIp    

    tdLog.info("Procedures for tdengine deployed in %s" % (host)) 
    if docker:        
        tdCases.logSql(logSql)
        tdLog.info("Procedures for testing self-deployment")             
        is_test_framework = 0
        key_word = 'tdCases.addLinux'
        try:
            if key_word in open(fileName).read():
                is_test_framework = 1
        except BaseException:
            pass
        if is_test_framework:
            moduleName = fileName.replace(".py", "").replace(os.sep, ".")
            uModule = importlib.import_module(moduleName)
            try:
                ucase = uModule.TDTestCase()                
                numOfNodes = ucase.updatecfgDict.get('numOfNodes')                
                cluster.init(numOfNodes, dataDir)                    
                cluster.prepardBuild()
                
                startArbitrator = False
                arbitratorHost = "tdnode1"
                for i in range(numOfNodes):
                    if ucase.updatecfgDict.get('%d' % (i + 1)) != None:
                        config = dict (ucase.updatecfgDict.get('%d' % (i + 1)))
                        print(config)
                        for key, value in config.items():
                            if key == "arbitrator":
                                startArbitrator = True
                                arbitratorHost = value
                            cluster.cfg(key, value, i + 1)                      
                cluster.run()
                if startArbitrator:
                    hostname=value.split(":")[0]
                    cluster.startArbitrator(hostname)
                conn = cluster.conn
            except Exception as e:
                print(e.args)
                print(str(e))
                exit(1)
        tdCases.runOneLinux(conn, fileName)
    elif windows:
        tdCases.logSql(logSql)
        tdLog.info("Procedures for testing self-deployment")
        td_clinet = TDSimClient("C:\\TDengine")
        td_clinet.deploy()
        if masterIp == "" or masterIp == "localhost":
            tdDnodes.init(deployPath)
            tdDnodes.setTestCluster(testCluster)
            tdDnodes.setValgrind(valgrind)
            tdDnodes.stopAll()
            is_test_framework = 0
            key_word = 'tdCases.addWindows'
            try:
                if key_word in open(fileName).read():
                    is_test_framework = 1
            except:
                pass
            if is_test_framework:
                moduleName = fileName.replace(".py", "").replace(os.sep, ".")
                uModule = importlib.import_module(moduleName)
                try:
                    ucase = uModule.TDTestCase()
                    tdDnodes.deploy(1,ucase.updatecfgDict)
                except :
                    tdDnodes.deploy(1,{})
            else:
                pass
                tdDnodes.deploy(1,{})
            tdDnodes.startWin(1)
        else:
            remote_conn = Connection("root@%s"%host)
            with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'):
                remote_conn.run("python3 ./test.py")
        conn = taos.connect(
            host="%s" % (host),
            config=td_clinet.cfgDir)
        tdCases.runOneWindows(conn, fileName)
        tdCases.logSql(logSql)

    else:
        tdDnodes.init(deployPath)
        tdDnodes.setTestCluster(testCluster)
        tdDnodes.setValgrind(valgrind)
        tdDnodes.stopAll()
        is_test_framework = 0
        key_word = 'tdCases.addLinux'
        try:
            if key_word in open(fileName).read():
                is_test_framework = 1
        except BaseException:
            pass
        if is_test_framework:
            moduleName = fileName.replace(".py", "").replace(os.sep, ".")
            uModule = importlib.import_module(moduleName)
            try:
                ucase = uModule.TDTestCase()
                tdDnodes.deploy(1, ucase.updatecfgDict)
            except BaseException:
                tdDnodes.deploy(1, {})
        else:
            pass
            tdDnodes.deploy(1, {})
        tdDnodes.start(1)

        tdCases.logSql(logSql)

        if testCluster:
            tdLog.info("Procedures for testing cluster")
            if fileName == "all":
                tdCases.runAllCluster()
            else:
                tdCases.runOneCluster(fileName)
        else:
            tdLog.info("Procedures for testing self-deployment")
            conn = taos.connect(
                host,
                config=tdDnodes.getSimCfgPath())
            if fileName == "all":
                tdCases.runAllLinux(conn)
            else:
                tdCases.runOneLinux(conn, fileName)
        if restart:
            if fileName == "all":
                tdLog.info("not need to query ")
            else:
                sp = fileName.rsplit(".", 1)
                if len(sp) == 2 and sp[1] == "py":
                    tdDnodes.stopAll()
                    tdDnodes.start(1)
                    time.sleep(1)
                    conn = taos.connect(host, config=tdDnodes.getSimCfgPath())
                    tdLog.info(
                        "Procedures for tdengine deployed in %s" %
                        (host))
                    tdLog.info("query test after taosd restart")
                    tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py")
                else:
                    tdLog.info("not need to query")
    conn.close()