Merge pull request #6030 from taosdata/xiaoping/add_test_case2

[TD-3663]<test>: add test case for docker cluster

tests/pytest/cluster/clusterEnvSetup/basic.py

-###################################################################
-#           Copyright (c) 2016 by TAOS Technologies, Inc.
-#                     All rights reserved.
-#
-#  This file is proprietary and confidential to TAOS Technologies.
-#  No part of this file may be reproduced, stored, transmitted,
-#  disclosed or used in any form or by any means other than as
-#  expressly provided by the written permission from Jianhui Tao
-#
-###################################################################
-
-# -*- coding: utf-8 -*-
-
-import os
-import taos
-import random
-import argparse
-
-class BuildDockerCluser:
-
-    def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion, dockerDir, removeFlag):
-        self.hostName = hostName
-        self.user = user
-        self.password = password
-        self.configDir = configDir
-        self.numOfNodes = numOfNodes
-        self.clusterVersion = clusterVersion
-        self.dockerDir = dockerDir
-        self.removeFlag = removeFlag
-
-    def getConnection(self):
-        self.conn = taos.connect(
-            host = self.hostName,
-            user = self.user,
-            password = self.password,
-            config = self.configDir)
-
-    def createDondes(self):
-        self.cursor = self.conn.cursor()        
-        for i in range(2, self.numOfNodes + 1):            
-            self.cursor.execute("create dnode tdnode%d" % i)
-    
-    def startArbitrator(self):
-        print("start arbitrator")
-        os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator")
-
-    def run(self):        
-        if self.numOfNodes < 2 or self.numOfNodes > 10:
-            print("the number of nodes must be between 2 and 10")
-            exit(0)   
-        print("remove Flag value %s" % self.removeFlag)     
-        if self.removeFlag == False:
-            os.system("./cleanClusterEnv.sh -d %s" % self.dockerDir)
-        os.system("./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.clusterVersion, self.dockerDir))
-        self.getConnection()
-        self.createDondes()
-        self.startArbitrator()
-
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    '-H',
-    '--host',
-    action='store',
-    default='tdnode1',
-    type=str,
-    help='host name to be connected (default: tdnode1)')
-parser.add_argument(
-    '-u',
-    '--user',
-    action='store',
-    default='root',
-    type=str,
-    help='user (default: root)')
-parser.add_argument(
-    '-p',
-    '--password',
-    action='store',
-    default='taosdata',
-    type=str,
-    help='password (default: taosdata)')
-parser.add_argument(
-    '-c',
-    '--config-dir',
-    action='store',
-    default='/etc/taos',
-    type=str,
-    help='configuration directory (default: /etc/taos)')
-parser.add_argument(
-    '-n',
-    '--num-of-nodes',
-    action='store',
-    default=2,
-    type=int,
-    help='number of nodes in the cluster (default: 2, min: 2, max: 5)')
-parser.add_argument(
-    '-v',
-    '--version',
-    action='store',
-    default='2.0.18.1',
-    type=str,
-    help='the version of the cluster to be build, Default is 2.0.17.1')
-parser.add_argument(
-    '-d',
-    '--docker-dir',
-    action='store',
-    default='/data',
-    type=str,
-    help='the data dir for docker, default is /data')
-parser.add_argument(
-    '--flag',
-    action='store_true',        
-    help='remove docker containers flag, default: True')
-
-args = parser.parse_args()
-cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version, args.docker_dir, args.flag)
-cluster.run()
-
-# usage 1: python3 basic.py -n 2 --flag   (flag is True)
-# usage 2: python3 basic.py -n 2 (flag should be False when it is not specified)
\ No newline at end of file

tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh

-#!/bin/bash
-echo "Executing cleanClusterEnv.sh"
-CURR_DIR=`pwd`
-
-if [ $# != 2 ]; then 
-  echo "argument list need input : "  
-  echo "  -d docker dir" 
-  exit 1
-fi
-
-DOCKER_DIR=
-while getopts "d:" arg
-do
-  case $arg in
-    d)
-      DOCKER_DIR=$OPTARG
-      ;;    
-    ?)
-      echo "unkonwn argument"
-      ;;
-  esac
-done
-
-function removeDockerContainers {
-    cd $DOCKER_DIR
-    docker-compose down --remove-orphans
-}
-
-function cleanEnv {
-  echo "Clean up docker environment"    
-  for i in {1..10}
-  do    
-    rm -rf $DOCKER_DIR/node$i/data/*    
-    rm -rf $DOCKER_DIR/node$i/log/*
-  done
-}
-
-removeDockerContainers
-cleanEnv
\ No newline at end of file

tests/pytest/cluster/clusterEnvSetup/node4.yml

-version: '3.7'
-
-services: 
-  td2.0-node4:
-     build:
-       context: .
-       args:
-         - PACKAGE=${PACKAGE}
-         - TARBITRATORPKG=${TARBITRATORPKG}
-         - EXTRACTDIR=${DIR}
-         - EXTRACTDIR2=${DIR2}
-         - DATADIR=${DATADIR}
-     image: 'tdengine:${VERSION}'
-     container_name: 'tdnode4'     
-     cap_add:
-       - ALL
-     stdin_open: true
-     tty: true
-     environment:
-       TZ: "Asia/Shanghai"
-     command: >
-       sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && 
-       echo $TZ > /etc/timezone &&
-       mkdir /coredump &&
-       echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
-       sysctl -p &&
-       exec my-main-application"
-     extra_hosts:
-      - "tdnode2:172.27.0.8"
-      - "tdnode3:172.27.0.9"
-      - "tdnode4:172.27.0.10"
-      - "tdnode5:172.27.0.11"
-      - "tdnode6:172.27.0.12"
-      - "tdnode7:172.27.0.13"
-      - "tdnode8:172.27.0.14"
-      - "tdnode9:172.27.0.15"
-      - "tdnode10:172.27.0.16"
-     volumes: 
-       # bind data directory
-       - type: bind
-         source: ${DATADIR}/node4/data
-         target: /var/lib/taos
-       # bind log directory
-       - type: bind
-         source: ${DATADIR}/node4/log
-         target: /var/log/taos
-       # bind configuration
-       - type: bind
-         source: ${DATADIR}/node4/cfg
-         target: /etc/taos
-       # bind core dump path 
-       - type: bind
-         source: ${DATADIR}/node4/core
-         target: /coredump
-       - type: bind
-         source: ${DATADIR}
-         target: /root
-     hostname: tdnode4
-     networks:
-         taos_update_net:
-             ipv4_address: 172.27.0.10
-     command: taosd
\ No newline at end of file

tests/pytest/cluster/clusterEnvSetup/node5.yml

-version: '3.7'
-
-services: 
-  td2.0-node5:
-     build:
-       context: .
-       args:
-         - PACKAGE=${PACKAGE}
-         - TARBITRATORPKG=${TARBITRATORPKG}
-         - EXTRACTDIR=${DIR}
-         - EXTRACTDIR2=${DIR2}
-         - DATADIR=${DATADIR}
-     image: 'tdengine:${VERSION}'
-     container_name: 'tdnode5'     
-     cap_add:
-       - ALL
-     stdin_open: true
-     tty: true
-     environment:
-       TZ: "Asia/Shanghai"
-     command: >
-       sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && 
-       echo $TZ > /etc/timezone &&
-       mkdir /coredump &&
-       echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf &&
-       sysctl -p &&
-       exec my-main-application"
-     extra_hosts:
-      - "tdnode2:172.27.0.8"
-      - "tdnode3:172.27.0.9"
-      - "tdnode4:172.27.0.10"
-      - "tdnode5:172.27.0.11"
-      - "tdnode6:172.27.0.12"
-      - "tdnode7:172.27.0.13"
-      - "tdnode8:172.27.0.14"
-      - "tdnode9:172.27.0.15"
-      - "tdnode10:172.27.0.16"
-     volumes: 
-       # bind data directory
-       - type: bind
-         source: ${DATADIR}/node5/data
-         target: /var/lib/taos
-       # bind log directory
-       - type: bind
-         source: ${DATADIR}/node5/log
-         target: /var/log/taos
-       # bind configuration
-       - type: bind
-         source: ${DATADIR}/node5/cfg
-         target: /etc/taos
-       # bind core dump path
-       - type: bind
-         source: ${DATADIR}/node5/core
-         target: /coredump
-       - type: bind
-         source: ${DATADIR}
-         target: /root
-     hostname: tdnode5     
-     networks:
-         taos_update_net:
-             ipv4_address: 172.27.0.11
-     command: taosd
\ No newline at end of file

tests/pytest/dockerCluster/OneMnodeMultipleVnodesTest.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+from basic import *
+from util.sql import tdSql
+
+
+
+class TDTestCase:
+
+    def init(self):
+        # tdLog.debug("start to execute %s" % __file__)
+
+        self.numOfNodes = 5
+        self.dockerDir = "/data"
+        cluster.init(self.numOfNodes, self.dockerDir)        
+        cluster.prepardBuild()
+        for i in range(self.numOfNodes):
+            if i == 0:
+                cluster.cfg("role", "1", i + 1)
+            else:
+                cluster.cfg("role", "2", i + 1)
+        cluster.run()
+
+td = TDTestCase()
+td.init()
+
+
+

tests/pytest/dockerCluster/basic.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import os
+import taos
+
+class BuildDockerCluser:
+
+    def init(self, numOfNodes, dockerDir):
+        self.numOfNodes = numOfNodes
+        self.dockerDir = dockerDir
+
+        self.hostName = "tdnode1"
+        self.user = "root"
+        self.password = "taosdata"
+        self.configDir = "/etc/taos"
+        self.dirs = ["data", "cfg", "log", "core"]
+        self.cfgDict = {
+            "numOfLogLines":"100000000",
+            "mnodeEqualVnodeNum":"0",
+            "walLevel":"1",                                 
+            "numOfThreadsPerCore":"2.0",
+            "monitor":"0",
+            "vnodeBak":"1",
+            "dDebugFlag":"135",
+            "mDebugFlag":"135",
+            "sdbDebugFlag":"135",
+            "rpcDebugFlag":"135",
+            "tmrDebugFlag":"131",
+            "cDebugFlag":"135",
+            "httpDebugFlag":"135",
+            "monitorDebugFlag":"135",
+            "udebugFlag":"135",
+            "jnidebugFlag":"135",
+            "qdebugFlag":"135",
+            "maxSQLLength":"1048576"
+        }     
+
+    # execute command, and return the output 
+    # ref: https://blog.csdn.net/wowocpp/article/details/80775650  
+    def execCmdAndGetOutput(self, cmd):  
+        r = os.popen(cmd)  
+        text = r.read()  
+        r.close()  
+        return text
+    
+    def execCmd(self, cmd):
+        if os.system(cmd) != 0:
+            quit()
+
+    def getTaosdVersion(self):
+        cmd = "taosd -V |grep version|awk '{print $3}'"
+        taosdVersion = self.execCmdAndGetOutput(cmd)
+        cmd = "find %s -name '*server*.tar.gz' | awk -F- '{print $(NF-2)}'|sort|awk 'END {print}'" % self.dockerDir
+        packageVersion = self.execCmdAndGetOutput(cmd)
+
+        if (taosdVersion is None or taosdVersion.isspace()) and (packageVersion is None or packageVersion.isspace()):
+            print("Please install taosd or have a install package ready")
+            quit()
+        else:
+            self.version = taosdVersion  if taosdVersion >= packageVersion else packageVersion
+            return self.version.strip()
+
+    def getConnection(self):
+        self.conn = taos.connect(
+            host = self.hostName,
+            user = self.user,
+            password = self.password,
+            config = self.configDir)
+    
+    def removeFile(self, rootDir, index, dir):
+        cmd = "rm -rf %s/node%d/%s/*" % (rootDir, index, dir)        
+        self.execCmd(cmd)
+
+    def clearEnv(self):
+        cmd = "cd %s && docker-compose down --remove-orphans" % self.dockerDir
+        self.execCmd(cmd)
+        for i in range(1, self.numOfNodes + 1):            
+            self.removeFile(self.dockerDir, i, self.dirs[0])
+            self.removeFile(self.dockerDir, i, self.dirs[1])
+            self.removeFile(self.dockerDir, i, self.dirs[2])
+
+    def createDir(self, rootDir, index, dir):
+        cmd = "mkdir -p %s/node%d/%s" % (rootDir, index, dir)
+        self.execCmd(cmd)
+
+    def createDirs(self):
+        for i in range(1, self.numOfNodes + 1):
+            for j in range(len(self.dirs)):
+                self.createDir(self.dockerDir, i, self.dirs[j])
+
+    def addExtraCfg(self, option, value):
+        self.cfgDict.update({option: value})
+
+    def cfg(self, option, value, nodeIndex):
+        cfgPath = "%s/node%d/cfg/taos.cfg" % (self.dockerDir, nodeIndex)
+        cmd = "echo '%s %s' >> %s" % (option, value, cfgPath)
+        self.execCmd(cmd)
+    
+    def updateLocalhosts(self):
+        cmd = "grep '172.27.0.7 *tdnode1' /etc/hosts"
+        result = self.execCmdAndGetOutput(cmd)
+        if result and not result.isspace():
+            cmd = "echo '172.27.0.7 tdnode1' >> /etc/hosts"
+            self.execCmd(cmd)
+    
+    def deploy(self):
+        self.clearEnv()
+        self.createDirs()
+        for i in range(1, self.numOfNodes + 1):
+            self.cfg("firstEp", "tdnode1:6030", i)
+
+            for key, value in self.cfgDict.items():
+                self.cfg(key, value, i)
+
+    def createDondes(self):
+        self.cursor = self.conn.cursor()        
+        for i in range(2, self.numOfNodes + 1):            
+            self.cursor.execute("create dnode tdnode%d" % i)
+    
+    def startArbitrator(self):
+        for i in range(1, self.numOfNodes + 1):
+            self.cfg("arbitrator", "tdnode1:6042", i)
+        cmd = "docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator"
+        self.execCmd(cmd)
+
+    def prepardBuild(self):
+        if self.numOfNodes < 2 or self.numOfNodes > 10:
+            print("the number of nodes must be between 2 and 10")
+            exit(0)
+        self.clearEnv()
+        self.createDirs()
+        self.updateLocalhosts()
+        self.deploy()
+
+    def run(self):           
+        cmd = "./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.getTaosdVersion(), self.dockerDir)        
+        self.execCmd(cmd)
+        self.getConnection()
+        self.createDondes()
+
+cluster = BuildDockerCluser()
\ No newline at end of file

tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh → tests/pytest/dockerCluster/buildClusterEnv.sh

@@ -32,43 +32,14 @@ do
   esac
 done
 
-function addTaoscfg {
-  for((i=1;i<=$NUM_OF_NODES;i++))
-  do 
-    touch $DOCKER_DIR/node$i/cfg/taos.cfg
-    echo 'firstEp          tdnode1:6030' > $DOCKER_DIR/node$i/cfg/taos.cfg
-    echo 'fqdn             tdnode'$i >> $DOCKER_DIR/node$i/cfg/taos.cfg
-    echo 'arbitrator       tdnode1:6042' >> $DOCKER_DIR/node$i/cfg/taos.cfg
-  done
-}
-
-function createDIR {
-  for((i=1;i<=$NUM_OF_NODES;i++))
-  do    
-    mkdir -p $DOCKER_DIR/node$i/data
-    mkdir -p $DOCKER_DIR/node$i/log
-    mkdir -p $DOCKER_DIR/node$i/cfg
-    mkdir -p $DOCKER_DIR/node$i/core
-  done
-}
-
-function cleanEnv {
-  echo "Clean up docker environment"    
-  for((i=1;i<=$NUM_OF_NODES;i++))
-  do    
-    rm -rf $DOCKER_DIR/node$i/data/*    
-    rm -rf $DOCKER_DIR/node$i/log/*
-  done
-}
-
 function prepareBuild {
 
-  if [ -d $CURR_DIR/../../../../release ]; then
+  if [ -d $CURR_DIR/../../../release ]; then
     echo release exists
-    rm -rf $CURR_DIR/../../../../release/*
+    rm -rf $CURR_DIR/../../../release/*
   fi
 
-  cd $CURR_DIR/../../../../packaging 
+  cd $CURR_DIR/../../../packaging 
 
   if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
     if [ ! -e $DOCKER_DIR/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
@@ -76,17 +47,17 @@ function prepareBuild {
       echo "generating TDeninge enterprise packages"
       ./release.sh -v cluster -n $VERSION >> /dev/null 2>&1
       
-      if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ]; then
+      if [ ! -e $CURR_DIR/../../../release/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ]; then
         echo "no TDengine install package found"
         exit 1
       fi
 
-      if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
+      if [ ! -e $CURR_DIR/../../../release/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
         echo "no arbitrator install package found"
         exit 1
       fi
 
-      cd $CURR_DIR/../../../../release
+      cd $CURR_DIR/../../../release
       mv TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR
       mv TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR
     fi
@@ -96,17 +67,17 @@ function prepareBuild {
       echo "generating TDeninge community packages"
       ./release.sh -v edge -n $VERSION >> /dev/null 2>&1
       
-      if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then
+      if [ ! -e $CURR_DIR/../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then
         echo "no TDengine install package found"
         exit 1
       fi
 
-      if [ ! -e $CURR_DIR/../../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
+      if [ ! -e $CURR_DIR/../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then
         echo "no arbitrator install package found"
         exit 1
       fi
 
-      cd $CURR_DIR/../../../../release
+      cd $CURR_DIR/../../../release
       mv TDengine-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR
       mv TDengine-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR
     fi   
@@ -147,13 +118,10 @@ function clusterUp {
     done
     docker_run=$docker_run" up -d"
   fi
-  echo $docker_run |sh 
+  echo $docker_run |sh
   
   echo "docker compose finish"
 }
 
-createDIR
-cleanEnv
-addTaoscfg
 prepareBuild
 clusterUp
\ No newline at end of file

tests/pytest/cluster/clusterEnvSetup/docker-compose.yml → tests/pytest/dockerCluster/docker-compose.yml

@@ -53,7 +53,7 @@ services:
          source: ${DATADIR}/node1/core
          target: /coredump
        - type: bind
-         source: /data
+         source: ${DATADIR}
          target: /root
      hostname: tdnode1
      networks:
@@ -90,6 +90,11 @@ services:
       - "tdnode3:172.27.0.9"
       - "tdnode4:172.27.0.10"
       - "tdnode5:172.27.0.11"
+      - "tdnode6:172.27.0.12"
+      - "tdnode7:172.27.0.13"
+      - "tdnode8:172.27.0.14"
+      - "tdnode9:172.27.0.15"
+      - "tdnode10:172.27.0.16"
      volumes: 
        # bind data directory
        - type: bind

tests/pytest/cluster/clusterEnvSetup/node3.yml → tests/pytest/dockerCluster/node3.yml

@@ -26,6 +26,7 @@ services:
        sysctl -p &&
        exec my-main-application"
      extra_hosts:
+      - "tdnode1:172.27.0.7"
       - "tdnode2:172.27.0.8"
       - "tdnode3:172.27.0.9"
       - "tdnode4:172.27.0.10"