diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 88f6eb870a753fc7c347dd82e8c2a42f5bfbb900..d486643a5dbf7160362a2410d07f44658f664fe3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,7 +53,7 @@ jobs: vmImage: 'ubuntu-latest' e2e_pool_definition: vmImage: 'ubuntu-16.04' - environment: PROFILE="-Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" + environment: PROFILE="-Dinclude-hadoop -Dhadoop.version=2.8.3 -Dinclude_hadoop_aws -Dscala-2.11" run_end_to_end: false container: flink-build-container jdk: jdk8 diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index aa51819d4c38cbb0125aafad20168a9514f6ab9f..ad6b86c694415fda8de6c3483008d1cc8851646c 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -39,6 +39,15 @@ FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd -P)`" # absolutized and normalized echo "flink-end-to-end-test directory: $END_TO_END_DIR" echo "Flink distribution directory: $FLINK_DIR" +echo "Java and Maven version" +java -version +mvn -version + +echo "Free disk space" +df -h + +echo "Running with profile '$PROFILE'" + # Template for adding a test: # run_test "" "$END_TO_END_DIR/test-scripts/" ["skip_check_exceptions"] @@ -49,6 +58,10 @@ echo "Flink distribution directory: $FLINK_DIR" # those checks are disabled, one should take care that a proper checks are performed in the tests itself that ensure that the test finished # in an expected state. +printf "\n\n==============================================================================\n" +printf "Running bash end-to-end tests\n" +printf "==============================================================================\n" + ################################################################################ # Checkpointing tests ################################################################################ @@ -84,12 +97,15 @@ run_test "Resuming Externalized Checkpoint after terminal failure (file, sync) e run_test "Resuming Externalized Checkpoint after terminal failure (rocks, non-incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true false true" "skip_check_exceptions" run_test "Resuming Externalized Checkpoint after terminal failure (rocks, incremental) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh 2 2 rocks true true true" "skip_check_exceptions" +run_test "RocksDB Memory Management end-to-end test" "$END_TO_END_DIR/test-scripts/test_rocksdb_state_memory_control.sh" + ################################################################################ # Docker ################################################################################ run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh" run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs" +run_test "Wordcount on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh dummy-fs" ################################################################################ # High Availability @@ -118,8 +134,10 @@ run_test "Run kubernetes session test" "$END_TO_END_DIR/test-scripts/test_kubern # Mesos ################################################################################ -run_test "Run Mesos WordCount test" "$END_TO_END_DIR/test-scripts/test_mesos_wordcount.sh" -run_test "Run Mesos multiple submission test" "$END_TO_END_DIR/test-scripts/test_mesos_multiple_submissions.sh" +if [[ $PROFILE == *"include-hadoop"* ]]; then + run_test "Run Mesos WordCount test" "$END_TO_END_DIR/test-scripts/test_mesos_wordcount.sh" + run_test "Run Mesos multiple submission test" "$END_TO_END_DIR/test-scripts/test_mesos_multiple_submissions.sh" +fi ################################################################################ # Miscellaneous @@ -134,7 +152,9 @@ run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_b run_test "Batch SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_sql.sh" run_test "Streaming SQL end-to-end test (Old planner)" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh old" "skip_check_exceptions" run_test "Streaming SQL end-to-end test (Blink planner)" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh blink" "skip_check_exceptions" -run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh" "skip_check_exceptions" +if [[ $PROFILE == *"include-hadoop"* ]]; then + run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh" "skip_check_exceptions" +fi run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh" "skip_check_exceptions" run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh s3" "skip_check_exceptions" run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4" @@ -152,7 +172,9 @@ run_test "Walkthrough DataStream Scala nightly end-to-end test" "$END_TO_END_DIR run_test "Test PubSub connector with Docker based Google PubSub Emulator" "$END_TO_END_DIR/test-scripts/test_streaming_gcp_pubsub.sh" -run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" +if [[ ${PROFILE} != *"jdk11"* ]]; then + run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" +fi run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" "skip_check_exceptions" run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" "skip_check_exceptions" @@ -163,13 +185,14 @@ run_test "SQL Client end-to-end test (Blink planner)" "$END_TO_END_DIR/test-scri run_test "TPC-H end-to-end test (Blink planner)" "$END_TO_END_DIR/test-scripts/test_tpch.sh" run_test "TPC-DS end-to-end test (Blink planner)" "$END_TO_END_DIR/test-scripts/test_tpcds.sh" - run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_heavy_deployment.sh" "skip_check_exceptions" run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25" run_test "Dependency shading of table modules test" "$END_TO_END_DIR/test-scripts/test_table_shaded_dependencies.sh" +run_test "Shaded Hadoop S3A with credentials provider end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh hadoop_with_provider" + ################################################################################ # Sticky Scheduling ################################################################################ @@ -182,5 +205,31 @@ run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR run_test "Local recovery and sticky scheduling end-to-end test" "$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh 4 10 rocks true true" "skip_check_exceptions" printf "\n[PASS] All bash e2e-tests passed\n" -printf "\n Please remember to run java e2e-tests by running mvn verify -Dcategories=\"\" -DdistDir= in flink-end-to-end-tests\n" -exit 0 + +printf "\n\n==============================================================================\n" +printf "Running Java end-to-end tests\n" +printf "==============================================================================\n" + +HERE="`dirname \"$0\"`" +HERE="`( cd \"${HERE}\" && pwd -P)`" +if [ -z "${HERE}" ] ; then + # error; for some reason, the path is not accessible + # to the script (e.g. permissions re-evaled after suid) + exit 1 # fail +fi +ARTIFACTS_DIR="${HERE}/artifacts" +mkdir -p $ARTIFACTS_DIR || { echo "FAILURE: cannot create log directory '${ARTIFACTS_DIR}'." ; exit 1; } + +LOG4J_PROPERTIES=${HERE}/../tools/log4j-travis.properties + +MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configurationFile=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" +MVN_COMMON_OPTIONS="-nsu -B -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.wagon.http.pool=false -Dfast -Pskip-webui-build" +e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',') +e2e_modules="${e2e_modules},$(find flink-walkthroughs -mindepth 2 -maxdepth 2 -name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')" + +PROFILE="$PROFILE -Pe2e-travis1 -Pe2e-travis2 -Pe2e-travis3 -Pe2e-travis4 -Pe2e-travis5 -Pe2e-travis6" +mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl ${e2e_modules} -DdistDir=$(readlink -e build-target) + +EXIT_CODE=$? + +exit $EXIT_CODE diff --git a/tools/azure-pipelines/free_disk_space.sh b/tools/azure-pipelines/free_disk_space.sh new file mode 100755 index 0000000000000000000000000000000000000000..44339cd92bc81794ccbaa1ba37bc322e2910608c --- /dev/null +++ b/tools/azure-pipelines/free_disk_space.sh @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# The Azure provided machines typically have the following disk allocation: +# Total space: 85GB +# Allocated: 67 GB +# Free: 17 GB +# This script frees up 28 GB of disk space by deleting unneeded packages and +# large directories. +# The Flink end to end tests download and generate more than 17 GB of files, +# causing unpredictable behavior and build failures. +# +echo "==============================================================================" +echo "Freeing up disk space on CI system" +echo "==============================================================================" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h +echo "Removing large packages" +sudo apt-get remove -y '^ghc-8.*' +sudo apt-get remove -y '^dotnet-.*' +sudo apt-get remove -y '^llvm-.*' +sudo apt-get remove -y 'php.*' +sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel +sudo apt-get autoremove -y +sudo apt-get clean +df -h +echo "Removing large directories" +# deleting 15GB +rm -rf /usr/share/dotnet/ +df -h diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml index 770c669eb0006804b12d70215a9a939f2d6c664d..2056f51965ea17188ebf51559d5a4801bbc91f17 100644 --- a/tools/azure-pipelines/jobs-template.yml +++ b/tools/azure-pipelines/jobs-template.yml @@ -133,7 +133,7 @@ jobs: #condition: or(eq(variables['MODE'], 'e2e'), eq(${{parameters.run_end_to_end}}, 'true')) # We are running this in a separate pool pool: ${{parameters.e2e_pool_definition}} - timeoutInMinutes: 240 + timeoutInMinutes: 200 cancelTimeoutInMinutes: 1 workspace: clean: all @@ -151,12 +151,16 @@ jobs: displayName: "Set to jdk11" condition: eq('${{parameters.jdk}}', 'jdk11') - script: ./tools/travis/setup_maven.sh - - script: ./tools/azure-pipelines/setup_kubernetes.sh + displayName: Setup Maven 3.2.5 + - script: ./tools/azure-pipelines/setup_docker.sh + displayName: Setup Docker + - script: ./tools/azure-pipelines/free_disk_space.sh + displayName: Free up disk space - script: M2_HOME=/home/vsts/maven_cache/apache-maven-3.2.5/ PATH=/home/vsts/maven_cache/apache-maven-3.2.5/bin:$PATH ${{parameters.environment}} STAGE=compile ./tools/azure_controller.sh compile - displayName: Build + displayName: Build Flink # TODO remove pre-commit tests script by adding the tests to the nightly script - - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh - displayName: Test - precommit +# - script: FLINK_DIR=build-target ./flink-end-to-end-tests/run-pre-commit-tests.sh +# displayName: Test - precommit - script: FLINK_DIR=`pwd`/build-target flink-end-to-end-tests/run-nightly-tests.sh - displayName: Run nightly e2e tests + displayName: Run e2e tests diff --git a/tools/azure-pipelines/setup_kubernetes.sh b/tools/azure-pipelines/setup_docker.sh similarity index 100% rename from tools/azure-pipelines/setup_kubernetes.sh rename to tools/azure-pipelines/setup_docker.sh