Add CI build job barriers to prevent unnecessary builds (#3662)

* Add CI build job barriers to prevent unnecessary builds

* Complete change file sets to trigger build jobs

* Polish

Jenkinsfile-Agent-Test

@@ -46,12 +46,22 @@ pipeline {
         }
 
         stage('Compile agent Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -Pagent clean package -DskipTests'
             }
         }
 
         stage('Compile plugin-test tools Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl1_${BUILD_ID} docker:build'
             }
@@ -61,7 +71,13 @@ pipeline {
                 echo "reserve."
             }
         }
+
         stage('Run Agent Plugin Tests') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             parallel {
                 stage('Group1') {
                     stages {

Jenkinsfile-Agent-Test-2

@@ -46,12 +46,22 @@ pipeline {
         }
 
         stage('Compile agent Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -Pagent clean package -DskipTests'
             }
         }
 
         stage('Compile plugin-test tools Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl2_${BUILD_ID} docker:build'
             }
@@ -61,7 +71,13 @@ pipeline {
                 echo "reserve."
             }
         }
+
         stage('Run Agent Plugin Tests') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             parallel {
                 stage('Group1') {
                     stages {

Jenkinsfile-Agent-Test-3

@@ -46,12 +46,22 @@ pipeline {
         }
 
         stage('Compile agent Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -Pagent clean package -DskipTests'
             }
         }
 
         stage('Compile plugin-test tools Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -f test/plugin/pom.xml clean package -DskipTests -Dbuild_id=wl3_${BUILD_ID} docker:build'
             }
@@ -61,7 +71,13 @@ pipeline {
                 echo "reserve."
             }
         }
+
         stage('Run Agent Plugin Tests') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/agent-build-condition.sh')
+                }
+            }
             parallel {
                 stage('Group1') {
                     stages {

Jenkinsfile-E2E

@@ -44,6 +44,11 @@ pipeline {
         }
 
         stage('Prepare Distribution Package') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh')
+                }
+            }
             steps {
                 // although these checks are done in ci-it, since they are lightweight/cheap
                 // we're using them as a barrier here to filter out some invalid PRs (fast-fail)
@@ -54,12 +59,22 @@ pipeline {
         }
 
         stage('Compile Test Codes') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh')
+                }
+            }
             steps {
                 sh './mvnw -f test/e2e/pom.xml -pl e2e-base clean install'
             }
         }
 
         stage('Run End-to-End Tests') {
+            when {
+                expression {
+                    return sh(returnStatus: true, script: 'bash tools/ci/e2e-build-condition.sh')
+                }
+            }
             parallel {
                 stage('Group 1') {
                     stages {

tools/ci/agent-build-condition.sh

+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bash -x $(dirname "$0")/changed.sh --any-of '^pom.xml$' \
+  '^Jenkinsfile-Agent-.*$' \
+  '^apm-application-toolkit/.*$' \
+  '^apm-commons/.*$' \
+  '^apm-protocol/.*$' \
+  '^apm-sniffer/.*$' \
+  '^test/plugin/.*$' \
+  '^tools/ci/agent-build-condition.sh$' \
+  '^tools/ci/changed.sh$'

tools/ci/changed.sh

+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+patterns=()
+any_of=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --any-of)
+      any_of=1
+      ;;
+    *)
+      patterns+=($1)
+      ;;
+  esac
+  shift
+done
+
+[[ ${#patterns[@]} -eq 0 ]] && echo 'No file pattern is specified, exiting' && exit 1
+
+changed_files=$(git diff --name-only origin/${ghprbTargetBranch:-master}..${ghprbActualCommit:-HEAD})
+
+test_results=()
+
+for file in ${changed_files}; do
+  for pattern in ${patterns[@]}; do
+    if [[ ${file} =~ ${pattern} ]]; then
+      test_results+=("Hit: ${file} matches pattern ${pattern}")
+    else
+      test_results+=("Miss: ${file} does not match pattern ${pattern}")
+    fi
+  done
+done
+
+IFS=$'\n' ; echo "${test_results[*]}"
+
+if [[ ${any_of} -eq 1 ]]; then
+  for test_result in ${test_results[@]}; do
+    [[ ${test_result} =~ ^Hit:.+$ ]] && echo ${test_result} && exit 1
+  done
+fi
+
+exit 0
\ No newline at end of file

tools/ci/e2e-build-condition.sh

+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bash -x $(dirname "$0")/changed.sh --any-of '^pom.xml$' \
+  '^Jenkinsfile-E2E$' \
+  '^apm-commons/.*$' \
+  '^apm-protocol/.*$' \
+  '^apm-sniffer/.*$' \
+  '^apm-webapp/.*$' \
+  '^oap-server/.*$' \
+  '^test/e2e/.*$' \
+  '^tools/ci/e2e-build-condition.sh$' \
+  '^tools/ci/changed.sh$'