Merge pull request #5164 from taosdata/test/jenkins

reblance server load

Jenkinsfile

@@ -80,25 +80,28 @@ pipeline {
           }
           steps {
           sh'''
-          cd ${WORKSPACE}
+          cp -r ${WORKSPACE} ${WORKSPACE}.tes
+          cd ${WORKSPACE}.tes
           git checkout develop
           git pull
           git fetch origin +refs/pull/${CHANGE_ID}/merge
           git checkout -qf FETCH_HEAD
           '''
           script{
-            skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 1 ",returnStdout:true) 
+            skipstage=sh(script:"git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) 
           }
+          sh'''
+          rm -rf ${WORKSPACE}.tes
+          '''
           }
       }
     
       stage('Parallel test stage') {
-        
         //only build pr
         when {
               changeRequest()
                expression {
-                    skipstage != 1
+                    skipstage != 0
               }
           }
       parallel {
@@ -124,12 +127,12 @@ pipeline {
             
             pre_test()
             timeout(time: 45, unit: 'MINUTES'){
-            sh '''
-            date
-            cd ${WKC}/tests
-            find pytest -name '*'sql|xargs rm -rf
-            ./test-all.sh p2
-            date'''
+                sh '''
+                date
+                cd ${WKC}/tests
+                find pytest -name '*'sql|xargs rm -rf
+                ./test-all.sh p2
+                date'''
             }
           }
         }
@@ -161,6 +164,7 @@ pipeline {
 
         stage('test_crash_gen_s3') {
           agent{label "b2"}
+          
           steps {
             pre_test()
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
@@ -172,17 +176,20 @@ pipeline {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
                 cd ${WKC}/tests/pytest
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
                 ./handle_crash_gen_val_log.sh
                 '''
-            }
+            }  
             timeout(time: 45, unit: 'MINUTES'){
-              sh '''
-              date
-              cd ${WKC}/tests
-              ./test-all.sh b2fq
-              date
-              '''
-            }
+                sh '''
+                date
+                cd ${WKC}/tests
+                ./test-all.sh b2fq
+                date
+                '''
+            }         
+            
           }
         }
 
@@ -216,6 +223,8 @@ pipeline {
               date
               cd ${WKC}/tests
               ./test-all.sh b4fq
+              cd ${WKC}/tests
+              ./test-all.sh p4
               date'''
             }
           }

tests/pytest/concurrent_inquiry.sh

+#!/bin/bash
+
+# This is the script for us to try to cause the TDengine server or client to crash
+#    
+# PREPARATION
+#
+# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree
+# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory
+# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg
+# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg
+# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above
+# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils
+#
+# RUNNING THIS SCRIPT
+# 
+# This script assumes the source code directory is intact, and that the binaries has been built in the
+# build/ directory, as such, will will load the Python libraries in the directory tree, and also load
+# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env
+# variables below.
+#
+# Running the script is simple, no parameter is needed (for now, but will change in the future).
+#
+# Happy Crashing...
+
+
+# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory
+EXEC_DIR=`dirname "$0"`
+if [[ $EXEC_DIR != "." ]]
+then
+    echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)"
+    exit -1
+fi
+
+CURR_DIR=`pwd`
+IN_TDINTERNAL="community"
+if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  TAOS_DIR=$CURR_DIR/../../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib
+else
+  TAOS_DIR=$CURR_DIR/../..
+  TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+  LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
+fi
+
+# Now getting ready to execute Python
+# The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk
+PYTHON_EXEC=python3.8
+
+# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
+export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd)
+
+# Then let us set up the library path so that our compiled SO file can be loaded by Python
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
+
+# Now we are all let, and let's see if we can find a crash. Note we pass all params
+CONCURRENT_INQUIRY=concurrent_inquiry.py
+if [[ $1 == '--valgrind' ]]; then
+  shift
+  export PYTHONMALLOC=malloc
+  VALGRIND_OUT=valgrind.out 
+  VALGRIND_ERR=valgrind.err
+  # How to generate valgrind suppression file: https://stackoverflow.com/questions/17159578/generating-suppressions-for-memory-leaks
+  # valgrind --leak-check=full --gen-suppressions=all --log-fd=9 python3.8 ./concurrent_inquiry.py $@ 9>>memcheck.log
+  echo Executing under VALGRIND, with STDOUT/ERR going to $VALGRIND_OUT and $VALGRIND_ERR, please watch them from a different terminal.
+  valgrind  \
+    --leak-check=yes \
+    --suppressions=crash_gen/valgrind_taos.supp \
+    $PYTHON_EXEC \
+    $CONCURRENT_INQUIRY $@ > $VALGRIND_OUT 2> $VALGRIND_ERR 
+elif [[ $1 == '--helgrind' ]]; then
+  shift
+  HELGRIND_OUT=helgrind.out 
+  HELGRIND_ERR=helgrind.err
+  valgrind  \
+    --tool=helgrind \
+    $PYTHON_EXEC \
+    $CONCURRENT_INQUIRY $@ > $HELGRIND_OUT 2> $HELGRIND_ERR
+else
+  $PYTHON_EXEC $CONCURRENT_INQUIRY $@
+fi
+

tests/pytest/pytest_2.sh

@@ -7,15 +7,10 @@ python3 ./test.py -f update/append_commit_data.py
 python3 ./test.py -f update/append_commit_last-0.py
 python3 ./test.py -f update/append_commit_last.py
 python3 ./test.py -f update/merge_commit_data.py
-python3 ./test.py -f update/merge_commit_data-0.py
+
 python3 ./test.py -f update/merge_commit_data2.py
 python3 ./test.py -f update/merge_commit_data2_update0.py
 python3 ./test.py -f update/merge_commit_last-0.py
 python3 ./test.py -f update/merge_commit_last.py
 python3 ./test.py -f update/bug_td2279.py
 
-# wal
-python3 ./test.py -f wal/addOldWalTest.py
-
-# function
-python3 ./test.py -f functions/all_null_value.py
\ No newline at end of file

tests/pytest/pytest_3.sh

@@ -70,26 +70,7 @@ python3 testCompress.py
 python3 testNoCompress.py
 python3 testMinTablesPerVnode.py
 
-# functions
-python3 ./test.py -f functions/function_avg.py -r 1
-python3 ./test.py -f functions/function_bottom.py -r 1
-python3 ./test.py -f functions/function_count.py -r 1
-python3 ./test.py -f functions/function_diff.py -r 1
-python3 ./test.py -f functions/function_first.py -r 1
-python3 ./test.py -f functions/function_last.py -r 1
-python3 ./test.py -f functions/function_last_row.py -r 1
-python3 ./test.py -f functions/function_leastsquares.py -r 1
-python3 ./test.py -f functions/function_max.py -r 1
-python3 ./test.py -f functions/function_min.py -r 1
-python3 ./test.py -f functions/function_operations.py -r 1 
-python3 ./test.py -f functions/function_percentile.py -r 1
-python3 ./test.py -f functions/function_spread.py -r 1
-python3 ./test.py -f functions/function_stddev.py -r 1
-python3 ./test.py -f functions/function_sum.py -r 1
-python3 ./test.py -f functions/function_top.py -r 1
-python3 ./test.py -f functions/function_twa.py -r 1
-python3 ./test.py -f functions/function_twa_test2.py
-python3 ./test.py -f functions/function_stddev_td2555.py
+
 python3 queryCount.py
 python3 ./test.py -f query/queryGroupbyWithInterval.py
 python3 client/twoClients.py

tests/pytest/pytest_4.sh

+python3 ./test.py -f update/merge_commit_data-0.py
+# wal
+python3 ./test.py -f wal/addOldWalTest.py
+
+# function
+python3 ./test.py -f functions/all_null_value.py
+# functions
+python3 ./test.py -f functions/function_avg.py -r 1
+python3 ./test.py -f functions/function_bottom.py -r 1
+python3 ./test.py -f functions/function_count.py -r 1
+python3 ./test.py -f functions/function_diff.py -r 1
+python3 ./test.py -f functions/function_first.py -r 1
+python3 ./test.py -f functions/function_last.py -r 1
+python3 ./test.py -f functions/function_last_row.py -r 1
+python3 ./test.py -f functions/function_leastsquares.py -r 1
+python3 ./test.py -f functions/function_max.py -r 1
+python3 ./test.py -f functions/function_min.py -r 1
+python3 ./test.py -f functions/function_operations.py -r 1 
+python3 ./test.py -f functions/function_percentile.py -r 1
+python3 ./test.py -f functions/function_spread.py -r 1
+python3 ./test.py -f functions/function_stddev.py -r 1
+python3 ./test.py -f functions/function_sum.py -r 1
+python3 ./test.py -f functions/function_top.py -r 1
+python3 ./test.py -f functions/function_twa.py -r 1
+python3 ./test.py -f functions/function_twa_test2.py
+python3 ./test.py -f functions/function_stddev_td2555.pyhao
\ No newline at end of file

tests/script/jenkins/basic_4.txt

-./test.sh -f unique/dnode/alternativeRole.sim
-./test.sh -f unique/dnode/balance1.sim
-./test.sh -f unique/dnode/balance2.sim
-./test.sh -f unique/dnode/balance3.sim
-./test.sh -f unique/dnode/balancex.sim
-./test.sh -f unique/dnode/offline1.sim
-./test.sh -f unique/dnode/offline2.sim
-./test.sh -f unique/dnode/reason.sim
-./test.sh -f unique/dnode/remove1.sim
-./test.sh -f unique/dnode/remove2.sim
-./test.sh -f unique/dnode/vnode_clean.sim
 
 ./test.sh -f unique/http/admin.sim
 ./test.sh -f unique/http/opentsdb.sim

tests/script/jenkins/basic_5.txt

+./test.sh -f unique/dnode/alternativeRole.sim
+./test.sh -f unique/dnode/balance1.sim
+./test.sh -f unique/dnode/balance2.sim
+./test.sh -f unique/dnode/balance3.sim
+./test.sh -f unique/dnode/balancex.sim
+./test.sh -f unique/dnode/offline1.sim
+./test.sh -f unique/dnode/offline2.sim
+
 ./test.sh -f general/stream/metrics_del.sim
 ./test.sh -f general/stream/metrics_replica1_vnoden.sim
 ./test.sh -f general/stream/restart_stream.sim

tests/script/jenkins/basic_6.txt

+./test.sh -f unique/dnode/reason.sim
+./test.sh -f unique/dnode/remove1.sim
+./test.sh -f unique/dnode/remove2.sim
+./test.sh -f unique/dnode/vnode_clean.sim
+
 ./test.sh -f unique/db/commit.sim
 ./test.sh -f unique/db/delete.sim
 ./test.sh -f unique/db/delete_part.sim

tests/test-all.sh

@@ -261,6 +261,9 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "p3" ]; then
     echo "### run Python_3 test ###"
     runPyCaseOneByOnefq pytest_3.sh
+  elif [ "$1" == "p4" ]; then
+    echo "### run Python_4 test ###"
+    runPyCaseOneByOnefq pytest_4.sh
   elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
     exit $(($totalFailed + $totalPyFailed))
   elif [ "$1" == "smoke" ] || [ -z "$1" ]; then