[FLINK-6466] [build] Build Hadoop 2.8.0 convenience binaries

Update Hadoop versions and replace Hadoop 2.3 with 2.8 in build and
continuous integration scripts. flink-yarn-tests can now be enabled for
all supported Hadoop versions and is made non-optional.

This closes #3832

.travis.yml

@@ -16,32 +16,32 @@ matrix:
   include:
   # Always run test groups A and B together
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.7.2 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.8.0 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis,jdk8 -Dmaven.javadoc.skip=true"
 
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk8"
-      env: PROFILE="-Dhadoop.version=2.6.3 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.7.3 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
 
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "openjdk7"
-      env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.11 -Pinclude-yarn-tests,flink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.6.5 -Dscala-2.11 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
 
     - jdk: "oraclejdk7"
-      env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-a,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk7"
-      env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-b,include-kinesis -Dmaven.javadoc.skip=true"
     - jdk: "oraclejdk7"
-      env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
+      env: PROFILE="-Dhadoop.version=2.4.1 -Pflink-fast-tests-c,include-kinesis -Dmaven.javadoc.skip=true"
 
 
 git:

flink-shaded-hadoop/flink-shaded-hadoop2/pom.xml

@@ -681,13 +681,13 @@ under the License.
 			<dependency>
 				<groupId>org.apache.httpcomponents</groupId>
 				<artifactId>httpcore</artifactId>
-				<version>4.2.5</version>
+				<version>4.4.4</version>
 			</dependency>
-			
+
 			<dependency>
 				<groupId>org.apache.httpcomponents</groupId>
 				<artifactId>httpclient</artifactId>
-				<version>4.2.6</version>
+				<version>4.5.2</version>
 			</dependency>
 		</dependencies>
 	</dependencyManagement>

flink-shaded-hadoop/flink-shaded-include-yarn-tests/pom.xml → flink-shaded-hadoop/flink-shaded-yarn-tests/pom.xml

@@ -29,8 +29,8 @@ under the License.
 		<relativePath>..</relativePath>
 	</parent>
 
-	<artifactId>flink-shaded-include-yarn-tests</artifactId>
-	<name>flink-shaded-include-yarn-tests</name>
+	<artifactId>flink-shaded-yarn-tests</artifactId>
+	<name>flink-shaded-yarn-tests</name>
 
 	<packaging>jar</packaging>
 

flink-shaded-hadoop/pom.xml

@@ -37,6 +37,7 @@ under the License.
 	<modules>
 		<module>flink-shaded-hadoop2</module>
 		<module>flink-shaded-hadoop2-uber</module>
+		<module>flink-shaded-yarn-tests</module>
 	</modules>
 
 	<dependencies>
@@ -63,15 +64,6 @@ under the License.
 		</dependency>
 	</dependencies>
 
-	<profiles>
-		<profile>
-			<id>include-yarn-tests</id>
-			<modules>
-				<module>flink-shaded-include-yarn-tests</module>
-			</modules>
-		</profile>
-	</profiles>
-
 	<build>
 		<plugins>
 			<!-- Relocate the Hadoop's Guava dependency into a different namespace and

flink-yarn-tests/pom.xml

@@ -88,7 +88,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-shaded-include-yarn-tests</artifactId>
+			<artifactId>flink-shaded-yarn-tests</artifactId>
 			<version>${project.version}</version>
 			<exclusions>
 				<exclusion>

pom.xml

@@ -76,6 +76,7 @@ under the License.
 		<module>flink-mesos</module>
 		<module>flink-metrics</module>
 		<module>flink-yarn</module>
+		<module>flink-yarn-tests</module>
 		<module>flink-fs-tests</module>
 	</modules>
 
@@ -84,7 +85,7 @@ under the License.
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<!-- Internal property to reduce build times on TravisCi -->
 		<flink-fast-tests-pattern>never-match-me</flink-fast-tests-pattern>
-		<hadoop.version>2.3.0</hadoop.version>
+		<hadoop.version>2.4.1</hadoop.version>
 		<!-- Need to use a user property here because the surefire
 			 forkCount is not exposed as a property. With this we can set
 			 it on the "mvn" commandline in travis. -->
@@ -493,14 +494,6 @@ under the License.
 			</properties>
 		</profile>
 
-		<!-- Profile to deactivate the YARN tests -->
-		<profile>
-			<id>include-yarn-tests</id>
-			<modules>
-				<module>flink-yarn-tests</module>
-			</modules>
-		</profile>
-		
 		<profile>
 			<id>vendor-repos</id>
 			<!-- Add vendor maven repositories -->

tools/create_release_files.sh

@@ -91,7 +91,7 @@ fi
 
 usage() {
   set +x
-  echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+  echo "./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
   echo ""
   echo "usage:"
   echo "[--scala-version <version>] [--hadoop-version <version>]"
@@ -102,12 +102,12 @@ usage() {
   echo "  USER_NAME=APACHEID GPG_PASSPHRASE=XXX GPG_KEY=KEYID \ "
   echo "  GIT_AUTHOR=\"`git config --get user.name` <`git config --get user.email`>\" \ "
   echo "  GIT_REPO=github.com/apache/flink.git \ "
-  echo "  ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+  echo "  ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
   echo ""
   echo "example 2: build local release"
   echo "  NEW_VERSION=1.2.0 RELEASE_BRANCH=master OLD_VERSION=1.2-SNAPSHOT \ "
   echo "  GPG_PASSPHRASE=XXX GPG_KEY=XXX IS_LOCAL_DIST=true \ "
-  echo "  ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.2"
+  echo "  ./create_release_files.sh --scala-version 2.11 --hadoop-version 2.7.3"
 
   exit 1
 }
@@ -272,25 +272,25 @@ make_source_release
 
 # build dist by input parameter of "--scala-vervion xxx --hadoop-version xxx"
 if [ "$SCALA_VERSION" == "none" ] && [ "$HADOOP_VERSION" == "none" ]; then
-  make_binary_release "hadoop2" "" 2.10
-  make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.10"
-  make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.10"
-  make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.10"
-
-  make_binary_release "hadoop2" "" 2.11
-  make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "2.11"
-  make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "2.11"
-  make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "2.11"
+  make_binary_release "hadoop2" "" "2.10"
+  make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.10"
+  make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.10"
+  make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.10"
+
+  make_binary_release "hadoop2" "" "2.11"
+  make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "2.11"
+  make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "2.11"
+  make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "2.11"
 elif [ "$SCALA_VERSION" == none ] && [ "$HADOOP_VERSION" != "none" ]
 then
   make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.10"
   make_binary_release "hadoop2" "-Dhadoop.version=$HADOOP_VERSION" "2.11"
 elif [ "$SCALA_VERSION" != none ] && [ "$HADOOP_VERSION" == "none" ]
 then
-  make_binary_release "hadoop2" "" $SCALA_VERSION
-  make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" "$SCALA_VERSION"
-  make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" "$SCALA_VERSION"
-  make_binary_release "hadoop27" "-Dhadoop.version=2.7.2" "$SCALA_VERSION"
+  make_binary_release "hadoop2" "" "$SCALA_VERSION"
+  make_binary_release "hadoop26" "-Dhadoop.version=2.6.5" "$SCALA_VERSION"
+  make_binary_release "hadoop27" "-Dhadoop.version=2.7.3" "$SCALA_VERSION"
+  make_binary_release "hadoop28" "-Dhadoop.version=2.8.0" "$SCALA_VERSION"
 else
   make_binary_release "hadoop2x" "-Dhadoop.version=$HADOOP_VERSION" "$SCALA_VERSION"
 fi