Run integration tests in a second stage (#10791)

* Run integration tests in a second stage

* maybe run the integration tests

* better

* drop sudo

.travis.yml

@@ -15,7 +15,6 @@
 
 language: java
 
-sudo: true
 dist: xenial
 
 jdk:
@@ -46,6 +45,7 @@ install: MAVEN_OPTS='-Xmx3000m' travis_wait 15 ${MVN} clean install -q -ff ${MAV
 stages:
   - name: test  # jobs that do not specify a stage get this default value
     if: type != cron
+  - name: it
   - name: cron
     if: type = cron
 
@@ -312,6 +312,7 @@ jobs:
     # START - Integration tests for Compile with Java 8 and Run with Java 8
     - &integration_batch_index
       name: "(Compile=openjdk8, Run=openjdk8) batch index integration test"
+      stage: it
       jdk: openjdk8
       services: &integration_test_services
         - docker
@@ -333,6 +334,7 @@ jobs:
 
     - &integration_input_format
       name: "(Compile=openjdk8, Run=openjdk8) input format integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -345,6 +347,7 @@ jobs:
 
     - &integration_input_source
       name: "(Compile=openjdk8, Run=openjdk8) input source integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -358,6 +361,7 @@ jobs:
     - &integration_perfect_rollup_parallel_batch_index
       name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch index integration test"
       jdk: openjdk8
+      stage: it
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=perfect-rollup-parallel-batch-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
       script: *run_integration_test
@@ -369,6 +373,7 @@ jobs:
 
     - &integration_kafka_index
       name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=kafka-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -381,6 +386,7 @@ jobs:
 
     - &integration_kafka_index_slow
       name: "(Compile=openjdk8, Run=openjdk8) kafka index integration test slow"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=kafka-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -393,6 +399,7 @@ jobs:
 
     - &integration_kafka_transactional_index
       name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -401,6 +408,7 @@ jobs:
 
     - &integration_kafka_transactional_index_slow
       name: "(Compile=openjdk8, Run=openjdk8) transactional kafka index integration test slow"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=kafka-transactional-index-slow' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -413,6 +421,7 @@ jobs:
 
     - &integration_kafka_format_tests
       name: "(Compile=openjdk8, Run=openjdk8) Kafka index integration test with various formats"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -425,6 +434,7 @@ jobs:
 
     - &integration_query
       name: "(Compile=openjdk8, Run=openjdk8) query integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=query' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -433,6 +443,7 @@ jobs:
 
     - &integration_query_retry
       name: "(Compile=openjdk8, Run=openjdk8) query retry integration test for missing segments"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=query-retry' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -441,6 +452,7 @@ jobs:
 
     - &integration_security
       name: "(Compile=openjdk8, Run=openjdk8) security integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=security' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -449,6 +461,7 @@ jobs:
 
     - &integration_realtime_index
       name: "(Compile=openjdk8, Run=openjdk8) realtime index integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=realtime-index' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -457,6 +470,7 @@ jobs:
 
     - &integration_append_ingestion
       name: "(Compile=openjdk8, Run=openjdk8) append ingestion integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=append-ingestion' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -469,6 +483,7 @@ jobs:
 
     - &integration_compaction_tests
       name: "(Compile=openjdk8, Run=openjdk8) compaction integration test"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-Dgroups=compaction' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -481,6 +496,7 @@ jobs:
 
     - &integration_tests
       name: "(Compile=openjdk8, Run=openjdk8) other integration tests"
+      stage: it
       jdk: openjdk8
       services: *integration_test_services
       env: TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,query-retry,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage,append-ingestion,compaction,high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
@@ -562,6 +578,7 @@ jobs:
 
     - &integration_batch_index_k8s
       name: "(Compile=openjdk8, Run=openjdk8, Cluster Build On K8s) ITNestedQueryPushDownTest integration test"
+      stage: it
       jdk: openjdk8
       services: &integration_test_services_k8s
         - docker