[test/plugin] Enhance PluginTest (#4677)

1. Re-balance the workload of plugintests
2. Restore testcases of MySQL, remove on #4641
3. Support user to runs plugin-test on debug mode. It will not remove the log files and actualData.yaml
4. remove paramater 'parallel_run_size' and disable parallel test.

.github/workflows/plugins-test.0.yaml

@@ -48,9 +48,9 @@ jobs:
           - { name: 'elasticsearch-6.x-scenario', title: 'ElasticSearch 6.7.1-6.8.4 (7)' }
           - { name: 'elasticsearch-7.x-scenario', title: 'ElasticSearch 7.0.0-7.5.2 (14)' }
           - { name: 'feign-scenario', title: 'Feign 9.0.0-9.5.1 (8)' }
-          - { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0' }
-          - { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0' }
-          - { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0' }
+          - { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0(19)' }
+          - { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0 (18)' }
+          - { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0 (4)' }
           - { name: 'gateway-scenario', title: 'Spring-Cloud-Gateway 2.1.x (3)' }
           - { name: 'grpc-scenario', title: 'gRPC 1.6.0-1.25.0 (22)' }
           - { name: 'gson-scenario', title: 'Gson (7)' }

.github/workflows/plugins-test.1.yaml

@@ -47,6 +47,7 @@ jobs:
           - { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' }
           - { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.11.1 (22)' }
           - { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' }
+          - { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' }
     steps:
       - uses: actions/checkout@v2
       - name: checkout submodules

.github/workflows/plugins-test.2.yaml

@@ -34,8 +34,7 @@ jobs:
       matrix:
         case:
           - { name: 'okhttp-scenario', title: 'OKHttp 3.0.x-3.14.x (34)' }
-          - { name: 'play-scenario', title: 'Play! Framework' }
-          - { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' }
+          - { name: 'play-scenario', title: 'Play! Framework (12)' }
           - { name: 'postgresql-scenario', title: 'PostgreSQL 9.2.x-9.4.x (36)' }
           - { name: 'pulsar-scenario', title: 'Pulsar 2.2.0-2.4.1 (7)' }
           - { name: 'rabbitmq-scenario', title: 'RabbitMQ (12)' }
@@ -49,10 +48,6 @@ jobs:
           - { name: 'sofarpc-scenario', title: 'SofaRPC 5.4.0-5.6.2 (23)' }
           - { name: 'solrj-7.x-scenario', title: 'SolrJ 7.x (12)' }
           - { name: 'spring-3.0.x-scenario', title: 'Spring 3.0.x (8)' }
-          - { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' }
-          - { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' }
-          - { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' }
-          - { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' }
           - { name: 'spring-cloud-feign-1.1.x-scenario', title: 'SpringCloud Feign 1.1.x (8)' }
           - { name: 'spring-cloud-feign-1.2.x-scenario', title: 'SpringCloud Feign 1.2.x-1.4.x (23)' }
           - { name: 'spring-cloud-feign-2.x-scenario', title: 'SpringCloud Feign 2.x (14)' }

.github/workflows/plugins-test.3.yaml

@@ -37,6 +37,10 @@ jobs:
           - { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' }
           - { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' }
           - { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' }
+          - { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' }
+          - { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' }
+          - { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' }
+          - { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' }
     steps:
       - uses: actions/checkout@v2
       - name: checkout submodules

test/plugin/containers/jvm-container/docker/run.sh

@@ -16,6 +16,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $DEBUG_MODE ]] && set -ex
 
 function exitOnError() {
     echo -e "\033[31m[ERROR] $1\033[0m">&2
@@ -23,8 +24,10 @@ function exitOnError() {
 }
 
 function exitAndClean() {
+    [[ -n $DEBUG_MODE ]] && exit $1;
+
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
-    [[ -d ${SCENARIO_HOME}/logs ]] && rm -rf ${SCENARIO_HOME}/logs
+    [[ -d ${LOGS_HOME} ]] && rm -rf ${LOGS_HOME}
     exit $1
 }
 
@@ -52,6 +55,9 @@ fi
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
+LOGS_HOME=${SCENARIO_HOME}/logs
+
+[[ ! -d $LOGS_HOME ]] && mkdir -p ${LOGS_HOME}
 
 unzip -q ${SCENARIO_HOME}/*.zip -d /var/run/
 if [[ ! -f /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} ]]; then
@@ -59,7 +65,7 @@ if [[ ! -f /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} ]]; then
 fi
 
 echo "To start mock collector"
-${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>/dev/null &
+${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${LOGS_HOME}/collector.out &
 healthCheck http://localhost:12800/receiveData
 
 # start applications
@@ -71,10 +77,10 @@ export agent_opts="
     -Dskywalking.collector.discovery_check_interval=2
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
-    -Dskywalking.logging.dir=/usr/local/skywalking/scenario/logs
+    -Dskywalking.logging.dir=${LOGS_HOME}
     -Dskywalking.agent.authentication=test-token
     -Xms256m -Xmx256m ${agent_opts}"
-exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>/dev/null &
+exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>${LOGS_HOME}/scenario.out &
 
 healthCheck ${SCENARIO_HEALTH_CHECK_URL}
 
@@ -91,7 +97,7 @@ java -jar \
     -Xmx256m -Xms256m \
     -DcaseName="${SCENARIO_NAME}-${SCENARIO_VERSION}" \
     -DtestCasePath=${SCENARIO_HOME}/data/ \
-    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>/dev/null
+    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>${LOGS_HOME}/validatolr.out
 status=$?
 
 if [[ $status -eq 0 ]]; then

test/plugin/containers/tomcat-container/docker/catalina.sh

@@ -107,7 +107,6 @@
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
 
 export AGENT_FILE_PATH=/usr/local/skywalking/scenario/agent
-echo "${AGENT_FILE_PATH}/skywalking-agent.jar"
 if [ -f "${AGENT_FILE_PATH}/skywalking-agent.jar" ]; then
     CATALINA_OPTS="$CATALINA_OPTS
     -javaagent:${JACOCO_HOME}/jacocoagent.jar=classdumpdir=${JACOCO_HOME}/classes/${SCENARIO_NAME}${SCENARIO_VERSION},destfile=${JACOCO_HOME}/${SCENARIO_NAME}${SCENARIO_VERSION}.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*:org.apache.skywalking.apm.testcase.*
@@ -118,7 +117,7 @@ if [ -f "${AGENT_FILE_PATH}/skywalking-agent.jar" ]; then
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
     -Dskywalking.agent.authentication=test-token
-    -Dskywalking.logging.dir=/usr/local/skywalking/scenario/logs
+    -Dskywalking.logging.dir=${LOGS_HOME}
     -Xms256m -Xmx256m -XX:PermSize=64M -XX:MaxPermSize=64"
 fi
 

test/plugin/containers/tomcat-container/docker/run.sh

@@ -16,14 +16,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $DEBUG_MODE ]] && set -ex
+
 function exitOnError() {
     echo -e "\033[31m[ERROR] $1\033[0m">&2
     exitAndClean 1
 }
 
 function exitAndClean() {
+    [[ -n $DEBUG_MODE ]] && exit $1
+
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
-    [[ -d ${SCENARIO_HOME}/logs ]] && rm -rf ${SCENARIO_HOME}/logs
+    [[ -d ${LOGS_HOME} ]] && rm -rf ${LOGS_HOME}
     exit $1
 }
 
@@ -46,6 +50,9 @@ function healthCheck() {
 
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
+export LOGS_HOME=${SCENARIO_HOME}/logs # share to catalina.sh
+
+[[ ! -d ${LOGS_HOME} ]] && mkdir $LOGS_HOME
 
 # Speed up launch tomcat
 rm /usr/local/tomcat/webapps/* -rf # remove needn't app
@@ -56,11 +63,11 @@ cp ${SCENARIO_HOME}/*.war /usr/local/tomcat/webapps/
 
 # start mock collector
 echo "To start mock collector"
-${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>/dev/null &
+${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${LOGS_HOME}/collector.out &
 healthCheck http://localhost:12800/receiveData
 
 echo "To start tomcat"
-/usr/local/tomcat/bin/catalina.sh start 1>/dev/null &
+/usr/local/tomcat/bin/catalina.sh start 1>${LOGS_HOME}/catalina.out &
 
 healthCheck ${SCENARIO_HEALTH_CHECK_URL}
 
@@ -77,7 +84,7 @@ java -jar \
     -Xmx256m -Xms256m \
     -DcaseName="${SCENARIO_NAME}-${SCENARIO_VERSION}" \
     -DtestCasePath=${SCENARIO_HOME}/data/ \
-    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>/dev/null
+    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>${LOGS_HOME}/validator.out
 status=$?
 
 if [[ $status -eq 0 ]]; then

test/plugin/run.sh

@@ -20,20 +20,21 @@ set -ex
 
 home="$(cd "$(dirname $0)"; pwd)"
 scenario_name=""
-parallel_run_size=1
 force_build="off"
 cleanup="off"
+debug_mode=
 
 mvnw=${home}/../../mvnw
 agent_home="${home}"/../../skywalking-agent
 jacoco_home="${home}"/../jacoco
 scenarios_home="${home}/scenarios"
+num_of_testcases=0
 
 print_help() {
     echo  "Usage: run.sh [OPTION] SCENARIO_NAME"
     echo -e "\t-f, --force_build \t\t do force to build Plugin-Test tools and images"
-    echo -e "\t--parallel_run_size, \t\t parallel size of test cases. Default: 1"
     echo -e "\t--cleanup, \t\t\t remove the related images and directories"
+    echo -e "\t--debug, \t\t\t to save the log files and actualData.yaml"
 }
 
 parse_commandline() {
@@ -47,22 +48,13 @@ parse_commandline() {
             --cleanup)
                 cleanup="on"
                 ;;
-            --parallel_run_size)
-                test $# -lt 2 && exitWithMessage "Missing value for the optional argument '$_key'."
-                parallel_run_size="$2"
-                shift
-                ;;
-            --parallel_run_size=*)
-                parallel_run_size="${_key##--parallel_run_size=}"
+            --debug)
+                debug_mode="on";
                 ;;
             -h|--help)
                 print_help
                 exit 0
                 ;;
-            -h*)
-                print_help
-                exit 0
-                ;;
             *)
                 scenario_name=$1
                 ;;
@@ -78,10 +70,8 @@ exitWithMessage() {
 
 exitAndClean() {
     elapsed=$(( `date +%s` - $start_stamp ))
-    num_of_testcases="`ls -l ${task_state_house} |grep -c FINISH`"
     [[ $1 -eq 1 ]] && printSystemInfo
-    printf "Scenarios: %s, Testcases: %d, parallel_run_size: %d, Elapsed: %02d:%02d:%02d \n" \
-        ${scenario_name} "${num_of_testcases}" "${parallel_run_size}" \
+    printf "Scenarios: ${scenario_name}, Testcases: ${num_of_testcases}, Elapsed: %02d:%02d:%02d \n" \
         $(( ${elapsed}/3600 )) $(( ${elapsed}%3600/60 )) $(( ${elapsed}%60 ))
     exit $1
 }
@@ -90,17 +80,6 @@ printSystemInfo(){
   bash ${home}/script/systeminfo.sh
 }
 
-waitForAvailable() {
-    while [[ `ls -l ${task_state_house} |grep -c RUNNING` -ge ${parallel_run_size} ]]
-    do
-        sleep 2
-    done
-
-    if [[ `ls -l ${task_state_house} |grep -c FAILURE` -gt 0 ]]; then
-        exitAndClean 1
-    fi
-}
-
 do_cleanup() {
     images=$(docker images -q "skywalking/agent-test-*:${BUILD_NO:=local}")
     [[ -n "${images}" ]] && docker rmi -f ${images}
@@ -155,9 +134,7 @@ fi
 [[ "$force_build" == "on" ]] && ${mvnw} --batch-mode -f ${home}/pom.xml clean package -DskipTests -DBUILD_NO=${BUILD_NO:=local} docker:build
 
 workspace="${home}/workspace/${scenario_name}"
-task_state_house="${workspace}/.states"
 [[ -d ${workspace} ]] && rm -rf $workspace
-mkdir -p ${task_state_house}
 
 plugin_runner_helper="${home}/dist/plugin-runner-helper.jar"
 if [[ ! -f ${plugin_runner_helper} ]]; then
@@ -211,27 +188,23 @@ do
         -Doutput.dir=${case_work_base} \
         -Dagent.dir=${_agent_home} \
         -Djacoco.home=${jacoco_home} \
+        -Ddebug.mode=${debug_mode} \
         -Ddocker.image.version=${BUILD_NO:=local} \
         ${plugin_runner_helper} 1>${case_work_logs_dir}/helper.log
 
     [[ $? -ne 0 ]] && exitWithMessage "${testcase_name}, generate script failure!"
 
     echo "start container of testcase.name=${testcase_name}"
-    bash ${case_work_base}/scenario.sh ${task_state_house} 1>${case_work_logs_dir}/${testcase_name}.log
-    sleep 3
-    waitForAvailable
-    rm -rf ${case_work_base}
+    bash ${case_work_base}/scenario.sh $debug_mode 1>${case_work_logs_dir}/${testcase_name}.log
+    status=$?
+    if [[ $status == 0 ]]; then
+        [[ -z $debug_mode ]] && rm -rf ${case_work_base}
+    else
+        exitWithMessage "Testcase ${testcase_name} failed!"
+    fi
+    num_of_testcases=$(($num_of_testcases+1))
 done
 
 echo -e "\033[33m${scenario_name} has already sumbitted\033[0m"
 
-# wait to finish
-while [[ `ls -l ${task_state_house} |grep -c RUNNING` -gt 0 ]]; do
-    sleep 1
-done
-
-if [[ `ls -l ${task_state_house} |grep -c FAILURE` -gt 0 ]]; then
-    exitAndClean 1
-fi
-
 exitAndClean 0

test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/ConfigurationImpl.java

@@ -160,6 +160,11 @@ public class ConfigurationImpl implements IConfiguration {
         return System.getProperty("jacoco.home");
     }
 
+    @Override
+    public String debugMode() {
+        return System.getProperty("debug.mode");
+    }
+
     @Override
     public Map<String, Object> toMap() {
         final Map<String, Object> root = new HashMap<>();
@@ -176,6 +181,7 @@ public class ConfigurationImpl implements IConfiguration {
         root.put("docker_image_version", dockerImageVersion());
         root.put("docker_container_name", dockerContainerName());
         root.put("jacoco_home", jacocoHome());
+        root.put("debug_mode", debugMode());
 
         root.put("expose", caseConfiguration().getExpose());
         root.put("hostname", caseConfiguration().getHostname());

test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java

@@ -56,6 +56,8 @@ public interface IConfiguration {
 
     String jacocoHome();
 
+    String debugMode();
+
     Map<String, Object> toMap();
 
     String extendEntryHeader();

test/plugin/runner-helper/src/main/resources/container-start-script.template

@@ -30,6 +30,9 @@ docker run \
         <#if catalina_opts??>
         --env CATALINA_OPTS=${catalina_opts} \
         </#if>
+        <#if debug_mode??>
+        --env DEBUG_MODE=${debug_mode} \
+        </#if>
         -v ${agent_home}:/usr/local/skywalking/scenario/agent \
         -v ${scenario_home}:/usr/local/skywalking/scenario \
         -v ${jacoco_home}:/jacoco \

test/plugin/runner-helper/src/main/resources/scenario.sh

@@ -16,22 +16,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $1 ]] && set -ex
+
 PRG="$0"
 PRGDIR=`dirname "$PRG"`
 [ -z "$SCENARIO_HOME" ] && SCENARIO_HOME=`cd "$PRGDIR" >/dev/null; pwd`
 
-state_house=$1
 testcase_name=${scenario_name}-${scenario_version}
 
 status=1
-<#noparse>touch ${state_house}/${testcase_name}.RUNNING</#noparse>
 
 ${running_script}
 
-<#noparse>
-if [[ ${status} -eq 0 ]]; then
-  mv ${state_house}/${testcase_name}.RUNNING ${state_house}/${testcase_name}.FINISH
-else
-  mv ${state_house}/${testcase_name}.RUNNING ${state_house}/${testcase_name}.FAILURE
-fi
-</#noparse>
+exit $status
\ No newline at end of file

test/plugin/scenarios/mysql-scenario/support-version.list

@@ -36,4 +36,13 @@
 5.1.26
 5.1.24
 5.1.22
-5.1.20
\ No newline at end of file
+5.1.20
+5.1.18
+5.1.16
+5.1.14
+5.1.12
+5.1.10
+5.1.8
+5.1.6
+5.1.4
+5.1.2
\ No newline at end of file