Merge pull request #5350 from taosdata/xiaoping/add_test_case

[TD-3155]<test>: add test case

Jenkinsfile

@@ -227,6 +227,8 @@ pipeline {
               ./test-all.sh p4
               cd ${WKC}/tests
               ./test-all.sh full jdbc
+              cd ${WKC}/tests
+              ./test-all.sh full unit
               date'''
             }
           }

tests/Jenkinsfile

@@ -55,9 +55,15 @@ pipeline {
             sh '''
             cd ${WKC}/tests
             ./test-all.sh b1
+            date'''
+            sh '''
             cd ${WKC}/tests
             ./test-all.sh full jdbc
             date'''
+            sh '''
+            cd ${WKC}/tests
+            ./test-all.sh full unit
+            date'''
           }
         }
 

tests/pytest/cluster/clusterSetup.py

@@ -11,15 +11,9 @@
 
 # -*- coding: utf-8 -*-
 
-import os
-import sys
-sys.path.insert(0, os.getcwd())
 from fabric import Connection
-from util.sql import *
-from util.log import *
-import taos
 import random
-import threading
+import time
 import logging
 
 class Node:
@@ -76,6 +70,19 @@ class Node:
             print("remove taosd error for node %d " % self.index)
             logging.exception(e)
     
+    def forceStopOneTaosd(self):
+        try:
+            self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')")
+        except Exception as e:
+            print("kill taosd error on node%d " % self.index)            
+    
+    def startOneTaosd(self):
+        try:
+            self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &")
+        except Exception as e:
+            print("start taosd error on node%d " % self.index)
+            logging.exception(e)    
+    
     def installTaosd(self, packagePath):
         self.conn.put(packagePath, self.homeDir)
         self.conn.cd(self.homeDir)
@@ -122,100 +129,51 @@ class Node:
 
 class Nodes:
     def __init__(self):
-        self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/')
-        self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/')
-        self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/')
-        self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/')
-        self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/')        
+        self.tdnodes = []
+        self.tdnodes.append(Node(0, 'root', '52.143.103.7', 'node1', 'a', '/root/'))
+        self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/'))
+        self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/'))
+        self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/'))
+        self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/'))
+
+    def stopOneNode(self, index):
+        self.tdnodes[index].forceStopOneTaosd()
+    
+    def startOneNode(self, index):
+        self.tdnodes[index].startOneTaosd()
 
     def stopAllTaosd(self):
-        self.node1.stopTaosd()
-        self.node2.stopTaosd()
-        self.node3.stopTaosd()
-    
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].stopTaosd()
+
     def startAllTaosd(self):
-        self.node1.startTaosd()
-        self.node2.startTaosd()
-        self.node3.startTaosd()
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].startTaosd()                    
     
     def restartAllTaosd(self):
-        self.node1.restartTaosd()
-        self.node2.restartTaosd()
-        self.node3.restartTaosd()
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].restartTaosd()       
     
     def addConfigs(self, configKey, configValue):          
-        self.node1.configTaosd(configKey, configValue)
-        self.node2.configTaosd(configKey, configValue)
-        self.node3.configTaosd(configKey, configValue)
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].configTaosd(configKey, configValue)        
     
-    def removeConfigs(self, configKey, configValue):          
-        self.node1.removeTaosConfig(configKey, configValue)
-        self.node2.removeTaosConfig(configKey, configValue)
-        self.node3.removeTaosConfig(configKey, configValue)        
+    def removeConfigs(self, configKey, configValue): 
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].removeTaosConfig(configKey, configValue)  
     
     def removeAllDataFiles(self):
-        self.node1.removeData()
-        self.node2.removeData()
-        self.node3.removeData()
-
-class ClusterTest:
-    def __init__(self, hostName):
-        self.host = hostName
-        self.user = "root"
-        self.password = "taosdata"
-        self.config = "/etc/taos"        
-        self.dbName = "mytest"
-        self.stbName = "meters"
-        self.numberOfThreads = 20
-        self.numberOfTables = 10000
-        self.numberOfRecords = 1000
-        self.tbPrefix = "t"
-        self.ts = 1538548685000
-        self.repeat = 1        
-
-    def connectDB(self):
-        self.conn = taos.connect(
-            host=self.host,
-            user=self.user,
-            password=self.password,
-            config=self.config)
-
-    def createSTable(self, replica):
-        cursor = self.conn.cursor()
-        tdLog.info("drop database if exists %s" % self.dbName)
-        cursor.execute("drop database if exists %s" % self.dbName)
-        tdLog.info("create database %s replica %d" % (self.dbName, replica))
-        cursor.execute("create database %s replica %d" % (self.dbName, replica))
-        tdLog.info("use %s" % self.dbName)
-        cursor.execute("use %s" % self.dbName)
-        tdLog.info("drop table if exists %s" % self.stbName)
-        cursor.execute("drop table if exists %s" % self.stbName)
-        tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName)
-        cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName)
-        cursor.close()
-
-    def insertData(self, threadID):
-        print("Thread %d: starting" % threadID)
-        cursor = self.conn.cursor()
-        tablesPerThread = int(self.numberOfTables / self.numberOfThreads)
-        baseTableID = tablesPerThread * threadID
-        for i in range (tablesPerThread):
-            cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i))            
-            query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i)
-            base = self.numberOfRecords * i            
-            for j in range(self.numberOfRecords):
-                query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10)) 
-            cursor.execute(query)      
-        cursor.close()
-        print("Thread %d: finishing" % threadID)
-    
-    def run(self):        
-        threads = []
-        tdLog.info("Inserting data")
-        for i in range(self.numberOfThreads):
-            thread = threading.Thread(target=self.insertData, args=(i,))
-            threads.append(thread)
-            thread.start()
-
-        for i in range(self.numberOfThreads):
-            threads[i].join()
\ No newline at end of file
+        for i in range(len(self.tdnodes)):
+            self.tdnodes[i].removeData()
+
+# kill taosd randomly every 10 mins
+nodes = Nodes()
+loop = 0
+while True:
+    loop = loop + 1    
+    index = random.randint(0, 4)
+    print("loop: %d, kill taosd on node%d" %(loop, index))
+    nodes.stopOneNode(index)
+    time.sleep(60)
+    nodes.startOneNode(index)
+    time.sleep(600)
\ No newline at end of file

tests/pytest/insert/metadataUpdate.py

@@ -54,10 +54,11 @@ class TDTestCase:
         p.terminate()
         
         tdSql.execute("insert into tb values(%d, 1, 2)" % (self.ts + 1))
+        tdSql.execute("insert into tb(ts, col1, col2) values(%d, 1, 2)" % (self.ts + 2))
 
         print("==============step2")
         tdSql.query("select * from tb")
-        tdSql.checkRows(2)
+        tdSql.checkRows(3)
 
     def stop(self):
         tdSql.close()

tests/test-all.sh

@@ -160,9 +160,10 @@ function runPyCaseOneByOnefq {
 totalFailed=0
 totalPyFailed=0
 totalJDBCFailed=0
+totalUnitFailed=0
 
 corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern`
-if [ "$2" != "jdbc" ] && [ "$2" != "python" ]; then
+if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ]; then
   echo "### run TSIM test case ###"
   cd $tests_dir/script
 
@@ -231,7 +232,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ]; then
   fi
 fi
 
-if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] ; then
+if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ]; then
   echo "### run Python test case ###"
 
   cd $tests_dir
@@ -300,8 +301,8 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] ; then
 fi
 
 
-if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then
-  echo "### run JDBC test case ###"  
+if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then
+  echo "### run JDBC test cases ###"
 
   cd $tests_dir
 
@@ -318,7 +319,7 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then
   nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 &
   sleep 30
   
-  cd $tests_dir/../src/connector/jdbc
+  cd $tests_dir/../src/connector/jdbc  
   
   mvn test > jdbc-out.log 2>&1
   tail -n 20 jdbc-out.log
@@ -343,4 +344,40 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$1" == "full" ]; then
   dohavecore 1
 fi
 
-exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed))
+if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$1" == "full" ]; then
+  echo "### run Unit tests ###"  
+
+  stopTaosd
+  cd $tests_dir
+
+  if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
+    cd ../../
+  else
+    cd ../
+  fi
+
+  pwd
+  cd debug/build/bin
+  nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 &
+  sleep 30
+  
+  pwd
+  ./queryTest > unittest-out.log 2>&1
+  tail -n 20 unittest-out.log
+
+  totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'`  
+  totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'`
+  totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess`
+
+  if [ "$totalUnitSuccess" -gt "0" ]; then
+    echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}"
+  fi
+  
+  if [ "$totalUnitFailed" -ne "0" ]; then
+    echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}"
+  fi
+  dohavecore 1
+fi
+
+
+exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed))