diff --git a/tools/azure-pipelines/build_properties.sh b/tools/azure-pipelines/build_properties.sh new file mode 100755 index 0000000000000000000000000000000000000000..34bdf4254f59cc671f270e2b37b27e5e262bd6ff --- /dev/null +++ b/tools/azure-pipelines/build_properties.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Returns 0 if the change is a documentation-only pull request +# +function is_docs_only_pullrequest() { + # check if it is a pull request branch, as generated by ci-bot: + if [[ ! $BUILD_SOURCEBRANCHNAME == ci_* ]] ; then + echo "INFO: Not a pull request."; + return 1 + fi + PR_ID=`echo "$BUILD_SOURCEBRANCHNAME" | cut -f2 -d_` + if ! [[ "$PR_ID" =~ ^[0-9]+$ ]] ; then + echo "ERROR: Extracted PR_ID is not a number, but this: '$PR_ID'" + return 1 + fi + # check if it is docs only pull request + # 1. Get PR details + GITHUB_PULL_DETAIL=`curl --silent "https://api.github.com/repos/apache/flink/pulls/$PR_ID"` + + # 2. Check if this build is in sync with the PR + GITHUB_PULL_HEAD_SHA=`echo $GITHUB_PULL_DETAIL | jq -r ".head.sha"` + THIS_BRANCH_SHA=`git rev-parse HEAD` + + if [[ "$GITHUB_PULL_HEAD_SHA" != "$THIS_BRANCH_SHA" ]] ; then + echo "INFO: SHA mismatch: GITHUB_PULL_HEAD_SHA=$GITHUB_PULL_HEAD_SHA != THIS_BRANCH_SHA=$THIS_BRANCH_SHA"; + # sha mismatch. There's some timing issue, and we can't trust the result + return 1 + fi + + # 3. Get number of commits in PR + GITHUB_NUM_COMMITS=`echo $GITHUB_PULL_DETAIL | jq -r ".commits"` + + if [[ $(git diff --name-only HEAD..HEAD~$GITHUB_NUM_COMMITS | grep -v "docs/") == "" ]] ; then + echo "INFO: This is a docs only change. Changed files:" + git diff --name-only HEAD..HEAD~$GITHUB_NUM_COMMITS + return 0 + fi + return 1 +} diff --git a/tools/azure-pipelines/free_disk_space.sh b/tools/azure-pipelines/free_disk_space.sh index 44339cd92bc81794ccbaa1ba37bc322e2910608c..4053a04b819cf1a782755304e5c072bfc19cf534 100755 --- a/tools/azure-pipelines/free_disk_space.sh +++ b/tools/azure-pipelines/free_disk_space.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index 7fd3e9f6e45bed4405055fa11c8c58d262f929a3..7a7ed90dae872a4f2e831709653d7738bda84ed3 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -175,6 +175,18 @@ jobs: workspace: clean: all steps: + # Skip e2e test execution if this is a documentation only pull request (master / release builds will still be checked regularly) + - bash: | + source ./tools/azure-pipelines/build_properties.sh + is_docs_only_pullrequest + if [[ "$?" == 0 ]] ; then + echo "##[debug]This is a documentation-only change. Skipping e2e execution." + echo "##vso[task.setvariable variable=skip;]1" + else + echo "##[debug]This is a regular CI build. Continuing ..." + echo "##vso[task.setvariable variable=skip;]0" + fi + displayName: Check if Docs only PR - task: Cache@2 inputs: key: $(CACHE_KEY) @@ -182,37 +194,47 @@ jobs: path: $(MAVEN_CACHE_FOLDER) displayName: Cache Maven local repo continueOnError: true + condition: not(eq(variables['SKIP'], '1')) - task: Cache@2 inputs: key: e2e-cache | flink-end-to-end-tests/**/*.java, !**/avro/** path: $(E2E_CACHE_FOLDER) displayName: Cache E2E files continueOnError: true + condition: not(eq(variables['SKIP'], '1')) - script: | echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_11_X64" echo "##vso[task.setvariable variable=PATH]$JAVA_HOME_11_X64/bin:$PATH" displayName: "Set to jdk11" condition: eq('${{parameters.jdk}}', 'jdk11') - script: | + echo "Setting up Maven" source ./tools/ci/maven-utils.sh setup_maven - displayName: Setup Maven 3.2.5 - - script: ./tools/azure-pipelines/setup_docker.sh - displayName: Setup Docker - - script: ./tools/azure-pipelines/free_disk_space.sh - displayName: Free up disk space - - script: sudo apt-get install -y bc + + echo "Setting up Docker" + ./tools/azure-pipelines/setup_docker.sh + + echo "Free up disk space" + ./tools/azure-pipelines/free_disk_space.sh + + echo "Installing required software" + sudo apt-get install -y bc + displayName: Prepare E2E run + condition: not(eq(variables['SKIP'], '1')) - script: ${{parameters.environment}} ./tools/ci/compile.sh displayName: Build Flink + condition: not(eq(variables['SKIP'], '1')) - script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh displayName: Run e2e tests env: IT_CASE_S3_BUCKET: $(SECRET_S3_BUCKET) IT_CASE_S3_ACCESS_KEY: $(SECRET_S3_ACCESS_KEY) IT_CASE_S3_SECRET_KEY: $(SECRET_S3_SECRET_KEY) + condition: not(eq(variables['SKIP'], '1')) # upload debug artifacts - task: PublishPipelineArtifact@1 - condition: and(succeededOrFailed(), not(eq(variables['ARTIFACT_DIR'], ''))) + condition: and(succeededOrFailed(), not(eq(variables['SKIP'], '1')), not(eq(variables['ARTIFACT_DIR'], ''))) displayName: Upload Logs inputs: targetPath: $(ARTIFACT_DIR)