Merge pull request #10115 from taosdata/CI_OPT_2.0

Ci opt 2.0

tests/parallel_test/cases.task

-# 806,,pytest,test.py -f update/append_commit_data.py too long time
+113,,pytest,python3 test.py -f update/append_commit_data.py
 # 20,,pytest,insert/retentionpolicy.py change date time
-# ,,pytest,python3 test.py -f tools/taosdemoTestTblAlt.py never quit
 440,,pytest,python3 test.py -f tools/taosdemoTestQuery.py
-# 432,,pytest,python3 test.py -f tools/taosdemoTestLimitOffset.py
-# 431,,pytest,python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py
+432,,pytest,python3 test.py -f tools/taosdemoTestLimitOffset.py
+431,,pytest,python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py
 296,,pytest,python3 ./test.py -f update/merge_commit_data.py
 294,,pytest,python3 ./test.py -f update/merge_commit_data-0.py
 258,,script,./test.sh -f general/stream/stream_3.sim
 256,,script,./test.sh -f general/stream/restart_stream.sim
 243,,pytest,python3 ./test.py -f update/merge_commit_data2.py
 232,,pytest,python3 ./test.py -f query/queryNullValueTest.py
-# 218,,pytest,python3 test.py -f tools/taosdemoTestWithoutMetric.py
-# 218,,pytest,python3 test.py -f tools/taosdemoTestSampleData.py
-# 218,,pytest,python3 test.py -f tools/taosdemoTest.py
-# 218,,pytest,python3 test.py -f tools/taosdemoTestInterlace.py
-# 217,,pytest,python3 test.py -f tools/taosdemoTestWithJson.py
-# 217,,pytest,python3 ./test.py -f perfbenchmark/bug3433.py
-# 216,,pytest,python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py 
-# 216,,pytest,python3 ./test.py -f query/query1970YearsAf.py
+218,,pytest,python3 test.py -f tools/taosdemoTestWithoutMetric.py
+218,,pytest,python3 test.py -f tools/taosdemoTestSampleData.py
+218,,pytest,python3 test.py -f tools/taosdemoTest.py
+218,,pytest,python3 test.py -f tools/taosdemoTestInterlace.py
+217,,pytest,python3 test.py -f tools/taosdemoTestWithJson.py
+217,,pytest,python3 ./test.py -f perfbenchmark/bug3433.py
+216,,pytest,python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py 
+216,,pytest,python3 ./test.py -f query/query1970YearsAf.py
 215,2,script,./test.sh -f unique/cluster/balance3.sim
-# 215,,pytest,python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py
+215,,pytest,python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py
 204,,pytest,python3 ./test.py -f update/merge_commit_data2_update0.py
 198,,pytest,python3 ./test.py -f stream/stream1.py
 196,,pytest,python3 ./test.py -f update/merge_commit_last.py
@@ -191,6 +190,7 @@
 29,,pytest,python3 ./test.py -f import_merge/importLastT.py
 29,,pytest,python3 ./test.py -f import_merge/importCacheFileTPO.py
 29,,pytest,python3 ./test.py -f import_merge/importCacheFileTO.py
+29,,pytest,python3 test.py -f tools/taosdemoTestTblAlt.py
 28,,script,./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim
 28,,script,./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
 28,,script,./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
@@ -524,7 +524,7 @@
 13,,pytest,python3 ./test.py -f table/column_num.py
 13,,pytest,python3 ./test.py -f table/column_name.py
 13,,pytest,python3 ./test.py -f table/boundary.py
-# 13,,pytest,python3 test.py -f subscribe/supertable.py
+13,,pytest,python3 test.py -f subscribe/supertable.py
 13,,pytest,python3 test.py -f query/queryFillTest.py
 13,,pytest,python3 ./test.py -f query/isNullTest.py
 13,,pytest,python3 ./test.py -f query/filter.py
@@ -580,7 +580,7 @@
 12,,pytest,python3 ./test.py -f insert/bug3654.py
 12,,pytest,python3 ./test.py -f insert/bigint.py
 12,,pytest,python3 ./test.py -f import_merge/importHRestart.py
-# 12,,pytest,python3 ./test.py -f functions/queryTestCases.py
+12,,pytest,python3 ./test.py -f functions/queryTestCases.py
 12,,pytest,python3 ./test.py -f functions/function_stddev.py -r 1
 12,,pytest,python3 ./test.py -f functions/function_spread.py -r 1
 12,,pytest,python3 ./test.py -f functions/function_percentile.py -r 1
@@ -630,5 +630,5 @@
 7,,pytest,python3 testNoCompress.py
 6,,pytest,python3 client/twoClients.py
 5,,pytest,python3 bug2265.py
-# 4,,pytest,python3 test.py -f subscribe/singlemeter.py
+4,,pytest,python3 test.py -f subscribe/singlemeter.py
 3,,pytest,python3 testMinTablesPerVnode.py

tests/parallel_test/run.sh

@@ -140,7 +140,7 @@ function build_src() {
         flock -x $lock_file -c "echo \"${hosts[index]} taos-tools build failed\" >>$log_dir/failed.log"
         return
     fi
-    script="cp -rf ${workdirs[index]}/taos-tools/build/build/bin/* ${workdirs[index]}/TDinternal/debug/build/bin/;cp -rf ${workdirs[index]}/taos-tools/build/build/lib/* ${workdirs[index]}/TDinternal/debug/build/lib/;cp -rf ${workdirs[index]}/taos-tools/build/build/lib64/* ${workdirs[index]}/TDinternal/debug/build/lib/;cp -rf ${workdirs[index]}/TDinternal/debug/build/bin/demo ${workdirs[index]}/TDinternal/debug/build/bin/taosdemo"
+    script="cp -rf ${workdirs[index]}/taos-tools/build/build/bin/* ${workdirs[index]}/TDinternal/debug/build/bin/;cp -rf ${workdirs[index]}/taos-tools/build/build/lib/* ${workdirs[index]}/TDinternal/debug/build/lib/;cp -rf ${workdirs[index]}/taos-tools/build/build/lib64/* ${workdirs[index]}/TDinternal/debug/build/lib/;cp -rf ${workdirs[index]}/TDinternal/debug/build/bin/taosBenchmark ${workdirs[index]}/TDinternal/debug/build/bin/taosdemo"
     cmd="${ssh_script} sh -c \"$script\""
     ${cmd}
 }
@@ -150,7 +150,7 @@ function rename_taosdemo() {
     if [ -z ${passwords[index]} ]; then
         ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}"
     fi
-    local script="cp -rf ${workdirs[index]}/TDinternal/debug/build/bin/demo ${workdirs[index]}/TDinternal/debug/build/bin/taosdemo"
+    local script="cp -rf ${workdirs[index]}/TDinternal/debug/build/bin/taosBenchmark ${workdirs[index]}/TDinternal/debug/build/bin/taosdemo 2>/dev/null"
     cmd="${ssh_script} sh -c \"$script\""
     ${cmd}
 }
@@ -182,7 +182,7 @@ function run_thread() {
         fi
         local case_redo_time=`echo "$line"|cut -d, -f2`
         if [ -z "$case_redo_time" ]; then
-            case_redo_time=1
+            case_redo_time=${DEFAULT_RETRY_TIME:-2}
         fi
         local exec_dir=`echo "$line"|cut -d, -f3`
         local case_cmd=`echo "$line"|cut -d, -f4`
@@ -232,6 +232,10 @@ function run_thread() {
             if [ $? -eq 0 ]; then
                 redo=1
             fi
+            grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
             grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log
             if [ $? -eq 0 ]; then
                 redo=1
@@ -273,7 +277,7 @@ function run_thread() {
             cat $log_dir/$case_file.log
             echo "====================================================="
             echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m"
-            if [ ! -z $corefile ]; then
+            if [ ! -z "$corefile" ]; then
                 echo -e "\e[34m corefiles: $corefile \e[0m"
             fi
         fi

tests/parallel_test/run_case.sh

@@ -43,11 +43,17 @@ fi
 
 go env -w GOPROXY=https://goproxy.cn
 echo "StrictHostKeyChecking no" >>/etc/ssh/ssh_config
-ln -s  $CONTAINER_TESTDIR/debug/build/lib/libtaos.so /usr/lib/libtaos.so
+ln -s  /home/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
 npm config -g set unsafe-perm
 npm config -g set registry https://registry.npm.taobao.org
 mkdir -p /home/sim/tsim
-
+mkdir -p /var/lib/taos/subscribe
+rm -rf ${CONTAINER_TESTDIR}/src/connector/nodejs/node_modules
+rm -rf ${CONTAINER_TESTDIR}/tests/examples/nodejs/node_modules
+rm -rf ${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/node_modules
+# ln -s /home/node_modules ${CONTAINER_TESTDIR}/src/connector/nodejs/
+# ln -s /home/node_modules ${CONTAINER_TESTDIR}/tests/examples/nodejs/
+# ln -s /home/node_modules ${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/
 # echo "$cmd"|grep -q "nodejs"
 # if [ $? -eq 0 ]; then
 #     cd $CONTAINER_TESTDIR/src/connector/nodejs

tests/parallel_test/run_container.sh

@@ -77,24 +77,29 @@ if [ $? -eq 0 ]; then
 -v $TMP_DIR/thread_volume/$thread_no/node_modules:${CONTAINER_TESTDIR}/tests/examples/nodejs/node_modules \
 -v $TMP_DIR/thread_volume/$thread_no/node_modules:${CONTAINER_TESTDIR}/tests/connectorTest/nodejsTest/nanosupport/node_modules"
 fi
+if [ -f "$REPDIR/src/plugins/taosadapter/example/config/taosadapter.toml" ]; then
+    TAOSADAPTER_TOML="-v $REPDIR/src/plugins/taosadapter/example/config/taosadapter.toml:/etc/taos/taosadapter.toml:ro"
+fi
 
 docker run \
     -v $REPDIR/tests:$CONTAINER_TESTDIR/tests \
     -v $MOUNT_DIR \
     -v "$TMP_DIR/thread_volume/$thread_no/sim:${CONTAINER_TESTDIR}/sim" \
     -v ${TMP_DIR}/thread_volume/$thread_no/coredump:/home/coredump \
-    -v $INTERNAL_REPDIR/debug:$CONTAINER_TESTDIR/debug:ro \
+    -v $INTERNAL_REPDIR/debug:/home/debug:ro \
     -v $REPDIR/deps:$CONTAINER_TESTDIR/deps:ro \
     -v $REPDIR/src:$CONTAINER_TESTDIR/src \
     -v $REPDIR/src/inc/taos.h:/usr/include/taos.h:ro \
+    $TAOSADAPTER_TOML \
     -v $REPDIR/tests/examples:$CONTAINER_TESTDIR/tests/examples \
     -v $REPDIR/snap:$CONTAINER_TESTDIR/snap:ro \
     -v $REPDIR/alert:$CONTAINER_TESTDIR/alert:ro \
     -v $REPDIR/packaging/cfg/taos.cfg:/etc/taos/taos.cfg:ro \
+    -v $REPDIR/packaging:$CONTAINER_TESTDIR/packaging:ro \
     -v $REPDIR/README.md:$CONTAINER_TESTDIR/README.md:ro \
     -v $REPDIR/src/connector/python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \
-    -e LD_LIBRARY_PATH=$CONTAINER_TESTDIR/debug/build/lib:$CONTAINER_TESTDIR/debug/build/lib64 \
-    -e PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$CONTAINER_TESTDIR/debug/build/bin:/usr/local/go/bin:/usr/local/node-v12.20.0-linux-x64/bin:/usr/local/apache-maven-3.8.4/bin:/usr/local/jdk1.8.0_144/bin \
+    -e LD_LIBRARY_PATH=/home/debug/build/lib:/home/debug/build/lib64 \
+    -e PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/debug/build/bin:/usr/local/go/bin:/usr/local/node-v12.20.0-linux-x64/bin:/usr/local/apache-maven-3.8.4/bin:/usr/local/jdk1.8.0_144/bin \
     -e JAVA_HOME=/usr/local/jdk1.8.0_144 \
     --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd"
 ret=$?