support full test

tests/compatibility/init.sh

+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -o old version package"
+    echo -e "\t -n new version package"
+    echo -e "\t -c client package"
+    echo -e "\t -h help"
+}
+
+while getopts "w:o:n:c:h" opt; do
+    case $opt in
+        w)
+            WORK_DIR=$OPTARG
+            ;;
+        o)
+            TAOS_PKG1=$OPTARG
+            ;;
+        n)
+            TAOS_PKG2=$OPTARG
+            ;;
+        c)
+            CLIENT_PKG=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORK_DIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$TAOS_PKG1" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$TAOS_PKG2" ]; then
+    usage
+    exit 1
+fi
+if [ ! -z "$CLIENT_PKG" ]; then
+    if [ ! -f "$CLIENT_PKG" ]; then
+        echo "$CLIENT_PKG not found"
+        exit 1
+    fi
+fi
+
+script_dir=`dirname $0`
+cd $script_dir
+
+source settings.sh
+
+code_dir=$WORK_DIR/TDinternal
+container_workdir1=$WORK_DIR/compatibility/$container_name1
+container_workdir2=$WORK_DIR/compatibility/$container_name2
+container_workdir3=$WORK_DIR/compatibility/$container_name3
+container_workdir4=$WORK_DIR/compatibility/$container_name4
+
+
+mkdir -p $container_workdir1
+mkdir -p $container_workdir2
+mkdir -p $container_workdir3
+mkdir -p $container_workdir4
+
+docker rm -f $container_name1 >/dev/null 2>&1
+docker rm -f $container_name2 >/dev/null 2>&1
+docker rm -f $container_name3 >/dev/null 2>&1
+docker rm -f $container_name4 >/dev/null 2>&1
+
+net_name=mynet
+docker network create --driver bridge --subnet 172.31.30.0/24 --gateway 172.31.30.1 $net_name
+
+./init_container.sh -d $code_dir -w $container_workdir1 -p $TAOS_PKG1 -q $TAOS_PKG2 -n $net_name -c $container_name1 &
+./init_container.sh -d $code_dir -w $container_workdir2 -p $TAOS_PKG1 -q $TAOS_PKG2 -n $net_name -c $container_name2 &
+./init_container.sh -d $code_dir -w $container_workdir3 -p $TAOS_PKG1 -q $TAOS_PKG2 -n $net_name -c $container_name3 &
+./init_container.sh -d $code_dir -w $container_workdir4 -p $TAOS_PKG1 -q $TAOS_PKG2 -n $net_name -c $container_name4 &
+
+RET=0
+pids=`jobs -p`
+for pid in $pids; do
+    wait $pid
+    status=$?
+    if [ $status -ne 0 ]; then
+        echo "init container $pid status is $status!"
+        RET=$status
+    fi
+done
+
+if [ $RET -eq 0 ]; then
+    if [ -z "$CLIENT_PKG" ]; then
+        docker exec $container_name4 pip3 install /home/TDinternal/community/src/connector/python
+        RET=$?
+    else
+        pkg_name=`basename $CLIENT_PKG`
+        pkg_dir=`echo "$pkg_name"|sed "s/-Linux-x64.tar.gz//"`
+        docker cp $CLIENT_PKG $container_name4:/home/
+        docker exec $container_name4 sh -c "cd /home;tar xzf $pkg_name;if [ -d /home/$pkg_dir/connector/python/linux/python3 ]; then pip3 install /home/$pkg_dir/connector/python/linux/python3; else pip3 install /home/$pkg_dir/connector/python; fi"
+        RET=$?
+    fi
+fi
+
+if [ $RET -eq 0 ]; then
+    echo "containers created"
+else
+    echo "containers create failed"
+fi
+
+exit $RET
+

tests/compatibility/init_container.sh

+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -d TDinternal code dir"
+    echo -e "\t -p old package"
+    echo -e "\t -q new package"
+    echo -e "\t -c container name"
+    echo -e "\t -h help"
+}
+
+while getopts "w:p:q:n:c:d:h" opt; do
+    case $opt in
+        w)
+            WORK_DIR=$OPTARG
+            ;;
+        d)
+            CODE_DIR=$OPTARG
+            ;;
+        p)
+            TAOS_PKG1=$OPTARG
+            ;;
+        q)
+            TAOS_PKG2=$OPTARG
+            ;;
+        c)
+            CONTAINER_NAME=$OPTARG
+            ;;
+        n)
+            NET_NAME=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORK_DIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$TAOS_PKG1" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$TAOS_PKG2" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$CONTAINER_NAME" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$NET_NAME" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$CODE_DIR" ]; then
+    usage
+    exit 1
+fi
+if [ ! -f "$TAOS_PKG1" ]; then
+    echo "$TAOS_PKG1 not found"
+    exit 1
+fi
+if [ ! -f "$TAOS_PKG2" ]; then
+    echo "$TAOS_PKG2 not found"
+    exit 1
+fi
+
+pkg_name1=`basename $TAOS_PKG1`
+pkg_dir1=`echo "$pkg_name1"|sed "s/-Linux-x64.tar.gz//"`
+pkg_name2=`basename $TAOS_PKG2`
+pkg_dir2=`echo "$pkg_name2"|sed "s/-Linux-x64.tar.gz//"`
+
+RET=0
+docker run -d --name $CONTAINER_NAME \
+    --hostname $CONTAINER_NAME \
+    --net $NET_NAME --ulimit core=-1 -it \
+    -v $TAOS_PKG1:/home/tdengine1.tar.gz:ro \
+    -v $TAOS_PKG2:/home/tdengine2.tar.gz:ro \
+    -v $WORK_DIR/coredump:/home/coredump \
+    -v $CODE_DIR:/home/TDinternal \
+    taos_test:v1.0 bash
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "docker run failed with $RET"
+    exit $RET
+fi
+
+docker exec $CONTAINER_NAME sh -c "cd /home;tar xzf tdengine1.tar.gz;tar xzf tdengine2.tar.gz;cd $pkg_dir1;./install.sh -v server -e no"
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "docker exec install.sh failed with $RET"
+    exit $RET
+fi
+exit 0
+

tests/compatibility/run_test.sh

+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -o old package"
+    echo -e "\t -n new package"
+    echo -e "\t -c client package"
+    echo -e "\t -h help"
+}
+
+while getopts "w:o:n:c:h" opt; do
+    case $opt in
+        w)
+            WORK_DIR=$OPTARG
+            ;;
+        o)
+            OLD_PACKAGE=$OPTARG
+            ;;
+        n)
+            NEW_PACKAGE=$OPTARG
+            ;;
+        c)
+            CLIENT_PACKAGE_PARAM="-c $OPTARG"
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORK_DIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$OLD_PACKAGE" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$NEW_PACKAGE" ]; then
+    usage
+    exit 1
+fi
+
+script_dir=`dirname $0`
+cd $script_dir
+
+pkg_name=`basename $NEW_PACKAGE`
+new_version=`echo "$pkg_name"|sed "s/TDengine-enterprise-server-//"|sed "s/-Linux-x64.tar.gz//"`
+pkg_name=`basename $OLD_PACKAGE`
+old_version=`echo "$pkg_name"|sed "s/TDengine-enterprise-server-//"|sed "s/-Linux-x64.tar.gz//"`
+
+source settings.sh
+
+containers="$container_name1 $container_name2 $container_name3"
+
+# initialize all containers and install with old version package
+./init.sh -w $WORK_DIR -o $OLD_PACKAGE -n $NEW_PACKAGE $CLIENT_PACKAGE_PARAM
+
+# upgrade with new version package
+function upgrade() {
+    local container_name=$1
+    local new_pkg_name=`basename $NEW_PACKAGE`
+    local new_pkg_dir=`echo "$new_pkg_name"|sed "s/-Linux-x64.tar.gz//"`
+    local ret=0
+    echo "upgrade ${container_name}"
+    docker exec $container_name service taosd stop
+    ret=$?
+    if [ $ret -ne 0 ]; then
+        echo "docker exec $container_name service taosd stop, exit: $ret"
+        return $ret
+    fi
+    docker exec $container_name sh -c "cd /home/$new_pkg_dir;./install.sh -v server -e no"
+    ret=$?
+    if [ $ret -ne 0 ]; then
+        echo "docker exec $container_name install.sh, exit: $ret"
+        return $ret
+    fi
+    docker exec $container_name service taosd start
+    ret=$?
+    if [ $ret -ne 0 ]; then
+        echo "docker exec $container_name service taosd start, exit: $ret"
+        return $ret
+    fi
+    return 0
+}
+
+function checkStatus() {
+    local container_name=$1
+    local check_version=$2
+    echo "python3 manualTest/TD-5114/checkClusterStatus.py $container_name $check_version"
+    docker exec $container_name4 sh -c "cd /home/TDinternal/community/tests/pytest;python3 manualTest/TD-5114/checkClusterStatus.py $container_name $check_version"
+    return $?
+}
+
+# config container /etc/taos/taos.cfg
+taos_cfg=/etc/taos/taos.cfg
+for container in $containers; do
+    docker exec $container sed -i "s/^.*firstEp.*$/firstEp $container_name1:6030/" $taos_cfg
+    docker exec $container sed -i "s/^.*fqdn.*$/fqdn $container/" $taos_cfg
+    docker exec $container sed -i "s/^.*numOfMnodes.*$/numOfMnodes 3/" $taos_cfg
+done
+
+# start taosd
+docker exec $container_name1 service taosd start
+docker exec $container_name4 taos -h $container_name1 -s "CREATE DNODE \"$container_name2:6030\";"
+docker exec $container_name4 taos -h $container_name1 -s "CREATE DNODE \"$container_name3:6030\";"
+
+# start taosd
+docker exec $container_name2 service taosd start
+docker exec $container_name3 service taosd start
+
+sleep 10
+
+# show nodes
+docker exec $container_name4 taos -h $container_name1 -s "SHOW DNODES;"
+docker exec $container_name4 taos -h $container_name1 -s "SHOW MNODES;"
+
+# check cluster status
+for container in $containers; do
+    checkStatus $container $old_version
+    RET=$?
+    if [ $RET -ne 0 ]; then
+        echo "check cluster status $container error: $RET"
+        exit $RET
+    fi
+    echo "check cluster status $container ret: $RET"
+done
+
+sleep 1
+
+# upgrade
+upgrade ${container_name3}
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "upgrade ${container_name3} error: $RET"
+    exit $RET
+fi
+sleep 10
+# check cluster status
+checkStatus ${container_name3} $old_version
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "check cluster status ${container_name3} error: $RET"
+    exit $RET
+fi
+echo "check cluster status ${container_name3} ret: $RET"
+
+# upgrade
+upgrade ${container_name2}
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "upgrade ${container_name2} error: $RET"
+    exit $RET
+fi
+sleep 10
+# check cluster status
+checkStatus ${container_name2} $old_version
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "check cluster status ${container_name2} error: $RET"
+    exit $RET
+fi
+echo "check cluster status ${container_name2} ret: $RET"
+
+# upgrade
+upgrade ${container_name1}
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "upgrade ${container_name1} error: $RET"
+    exit $RET
+fi
+sleep 10
+# check cluster status
+checkStatus ${container_name3} $new_version
+RET=$?
+if [ $RET -ne 0 ]; then
+    echo "check cluster status ${container_name3} error: $RET"
+    exit $RET
+fi
+echo "check cluster status ${container_name3} ret: $RET"
+
+exit $RET
+

tests/compatibility/settings.sh

+
+container_name1=compat_container1
+container_name2=compat_container2
+container_name3=compat_container3
+container_name4=compat_container4

tests/parallel_test/arm.cases.task

+4,,pytest,python3 test.py -f insert/double.py 
+4,,pytest,python3 test.py -f insert/date.py 
+4,,pytest,python3 test.py -f insert/bug3654.py 
+4,,pytest,python3 test.py -f insert/bool.py 
+4,,pytest,python3 test.py -f insert/bigint.py 
+4,,pytest,python3 test.py -f insert/basic.py 
+4,,pytest,python3 test.py -f insert/alterTableAndInsert.py 
+4,,pytest,python3 test.py -f import_merge/importHeadPartOverlap.py 
+4,,pytest,python3 test.py -f functions/function_stddev_td2555.py 
+4,,pytest,python3 test.py -f dbmgmt/nanoSecondCheck.py 
+4,,pytest,python3 bug2265.py 
+4,,pytest,python3 test.py -f alter/alterTabAddTagWithNULL.py 
+4,,pytest,python3 test.py -f alter/alter_debugFlag.py 
+4,,pytest,python3 test.py -f alter/alter_create_exception.py 
+3,,pytest,python3 test.py -f tag_lite/binary.py 
+3,,pytest,python3 test.py -f query/filterAllIntTypes.py 

tests/parallel_test/arm.json

+[{
+    "host":"192.168.1.207",
+    "username":"root",
+    "workdir":"/var/data/jenkins/workspace",
+    "thread":2
+},
+{
+    "host":"192.168.1.204",
+    "username":"root",
+    "workdir":"/var/data/jenkins/workspace",
+    "thread":2
+}]

tests/parallel_test/container_build.sh

+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -c community version"
+    echo -e "\t -t make thread count"
+    echo -e "\t -h help"
+}
+
+while getopts "w:t:ch" opt; do
+    case $opt in
+        w)
+            WORKDIR=$OPTARG
+            ;;
+        c)
+            COMMUNITY=community
+            ;;
+        t)
+            THREAD_COUNT=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$THREAD_COUNT" ]; then
+    THREAD_COUNT=1
+fi
+
+ulimit -c unlimited
+
+INTERNAL_REPDIR=$WORKDIR/TDinternal
+
+docker run \
+    -v $INTERNAL_REPDIR:/home \
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd /home/$COMMUNITY;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
+
+ret=$?
+exit $ret

tests/parallel_test/run.sh

@@ -6,10 +6,11 @@ function usage() {
     echo -e "\t -t task file"
     echo -e "\t -b branch"
     echo -e "\t -l log dir"
+    echo -e "\t -o default timeout value"
     echo -e "\t -h help"
 }
 
-while getopts "m:t:b:l:h" opt; do
+while getopts "m:t:b:l:o:h" opt; do
     case $opt in
         m)
             config_file=$OPTARG
@@ -23,6 +24,9 @@ while getopts "m:t:b:l:h" opt; do
         l)
             log_dir=$OPTARG
             ;;
+        o)
+            timeout_param="-o $OPTARG"
+            ;;
         h)
             usage
             exit 0
@@ -201,18 +205,21 @@ function run_thread() {
         if [ -z "$case_file" ]; then
             continue
         fi
-        case_file="$exec_dir/${case_file}.${index}.${thread_no}"
+        case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}"
         count=$(( count + 1 ))
         local case_path=`dirname "$case_file"`
         if [ ! -z "$case_path" ]; then
             mkdir -p $log_dir/$case_path
         fi
-        cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir}"
+        cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param}"
         # echo "$thread_no $count $cmd"
         local ret=0
         local redo_count=1
         start_time=`date +%s`
         while [ ${redo_count} -lt 6 ]; do
+            if [ -f $log_dir/$case_file.log ]; then
+                cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog
+            fi
             echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log
             echo -e "\e[33m >>>>> \e[0m ${case_cmd}"
             date >>$log_dir/$case_file.log
@@ -220,6 +227,7 @@ function run_thread() {
             # ret=${PIPESTATUS[0]}
             $cmd >>$log_dir/$case_file.log 2>&1
             ret=$?
+            echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log
             if [ $ret -eq 0 ]; then
                 break
             fi

tests/parallel_test/run_case.sh

 #!/bin/bash
 
 CONTAINER_TESTDIR=/home/community
-# CONTAINER_TESTDIR=/root/tang/repository/TDengine
-
-# export PATH=$PATH:$CONTAINER_TESTDIR/debug/build/bin
 
 function usage() {
     echo "$0"
     echo -e "\t -d execution dir"
     echo -e "\t -c command"
+    echo -e "\t -o default timeout value"
     echo -e "\t -h help"
 }
 
-while getopts "d:c:h" opt; do
+while getopts "d:c:o:h" opt; do
     case $opt in
         d)
             exec_dir=$OPTARG
@@ -20,6 +18,9 @@ while getopts "d:c:h" opt; do
         c)
             cmd=$OPTARG
             ;;
+        o)
+            TIMEOUT_CMD="timeout $OPTARG"
+            ;;
         h)
             usage
             exit 0
@@ -41,29 +42,15 @@ if [ -z "$cmd" ]; then
     exit 0
 fi
 
-go env -w GOPROXY=https://goproxy.cn
-echo "StrictHostKeyChecking no" >>/etc/ssh/ssh_config
+export PATH=$PATH:/home/debug/build/bin
 ln -s  /home/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
-npm config -g set unsafe-perm
-npm config -g set registry https://registry.npm.taobao.org
 mkdir -p /home/sim/tsim
 mkdir -p /var/lib/taos/subscribe
-rm -rf ${CONTAINER_TESTDIR}/src/connector/nodejs/node_modules
-rm -rf ${CONTAINER_TESTDIR}/tests/examples/nodejs/node_modules
-rm -rf ${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/node_modules
-# ln -s /home/node_modules ${CONTAINER_TESTDIR}/src/connector/nodejs/
-# ln -s /home/node_modules ${CONTAINER_TESTDIR}/tests/examples/nodejs/
-# ln -s /home/node_modules ${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/
-# echo "$cmd"|grep -q "nodejs"
-# if [ $? -eq 0 ]; then
-#     cd $CONTAINER_TESTDIR/src/connector/nodejs
-#     npm install node-gyp-build@4.3.0 --ignore-scripts
-# fi
 
 cd $CONTAINER_TESTDIR/tests/$exec_dir
 ulimit -c unlimited
 
-$cmd
+$TIMEOUT_CMD $cmd
 RET=$?
 
 if [ $RET -ne 0 ]; then

tests/parallel_test/run_container.sh

@@ -6,10 +6,11 @@ function usage() {
     echo -e "\t -d execution dir"
     echo -e "\t -c command"
     echo -e "\t -t thread number"
+    echo -e "\t -o default timeout value"
     echo -e "\t -h help"
 }
 
-while getopts "w:d:c:t:h" opt; do
+while getopts "w:d:c:t:o:h" opt; do
     case $opt in
         w)
             WORKDIR=$OPTARG
@@ -23,6 +24,9 @@ while getopts "w:d:c:t:h" opt; do
         t)
             thread_no=$OPTARG
             ;;
+        o)
+            timeout_param="-o $OPTARG"
+            ;;
         h)
             usage
             exit 0
@@ -71,12 +75,6 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then
 fi
 MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir"
 echo "$thread_no -> ${exec_dir}:$cmd"
-echo "$cmd"|grep -q "nodejs"
-if [ $? -eq 0 ]; then
-    MOUNT_NODE_MOD="-v $TMP_DIR/thread_volume/$thread_no/node_modules:${CONTAINER_TESTDIR}/src/connector/nodejs/node_modules \
--v $TMP_DIR/thread_volume/$thread_no/node_modules:${CONTAINER_TESTDIR}/tests/examples/nodejs/node_modules \
--v $TMP_DIR/thread_volume/$thread_no/node_modules:${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/node_modules"
-fi
 if [ -f "$REPDIR/src/plugins/taosadapter/example/config/taosadapter.toml" ]; then
     TAOSADAPTER_TOML="-v $REPDIR/src/plugins/taosadapter/example/config/taosadapter.toml:/etc/taos/taosadapter.toml:ro"
 fi
@@ -99,9 +97,7 @@ docker run \
     -v $REPDIR/README.md:$CONTAINER_TESTDIR/README.md:ro \
     -v $REPDIR/src/connector/python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \
     -e LD_LIBRARY_PATH=/home/debug/build/lib:/home/debug/build/lib64 \
-    -e PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/debug/build/bin:/usr/local/go/bin:/usr/local/node-v12.20.0-linux-x64/bin:/usr/local/apache-maven-3.8.4/bin:/usr/local/jdk1.8.0_144/bin \
-    -e JAVA_HOME=/usr/local/jdk1.8.0_144 \
-    --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd"
+    --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $timeout_param
 ret=$?
 exit $ret