From ac7d87158b3f2ebc72633fc8c925ded5fd408273 Mon Sep 17 00:00:00 2001 From: Robert Metzger Date: Tue, 22 Nov 2016 11:24:14 +0100 Subject: [PATCH] [FLINK-4895] Drop Hadoop1 support and remove related build infrastructure This closes #2850 --- .travis.yml | 5 - docs/_config.yml | 1 - docs/setup/building.md | 33 +--- .../flink-hadoop-compatibility/pom.xml | 2 +- .../mapreduce/example/WordCount.java | 1 - flink-batch-connectors/flink-hbase/pom.xml | 128 +++++--------- flink-dist/pom.xml | 24 +-- flink-fs-tests/pom.xml | 2 +- flink-java/pom.xml | 2 +- flink-mesos/pom.xml | 2 +- .../resources/archetype-resources/pom.xml | 1 - .../resources/archetype-resources/pom.xml | 1 - flink-runtime/pom.xml | 2 +- .../fs/hdfs/HadoopDataOutputStream.java | 109 +----------- .../flink-shaded-hadoop1/pom.xml | 159 ------------------ flink-shaded-hadoop/pom.xml | 8 +- .../flink-connector-filesystem/pom.xml | 2 +- flink-streaming-connectors/pom.xml | 16 +- flink-yarn-tests/pom.xml | 2 +- flink-yarn/pom.xml | 2 +- pom.xml | 44 +---- tools/create_release_files.sh | 11 -- tools/deploy_to_maven.sh | 14 +- tools/generate_specific_pom.sh | 129 -------------- 24 files changed, 63 insertions(+), 637 deletions(-) delete mode 100644 flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml delete mode 100755 tools/generate_specific_pom.sh diff --git a/.travis.yml b/.travis.yml index e15673e692a..1445f8d31e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,11 +35,6 @@ matrix: - jdk: "oraclejdk7" env: PROFILE="-Dhadoop.version=2.3.0 -Pflink-fast-tests-b,include-kinesis" - - jdk: "openjdk7" - env: PROFILE="-Dhadoop.profile=1 -Pflink-fast-tests-a,include-kinesis" - - jdk: "openjdk7" - env: PROFILE="-Dhadoop.profile=1 -Pflink-fast-tests-b,include-kinesis" - git: depth: 100 diff --git a/docs/_config.yml b/docs/_config.yml index 1f9f157bcc4..b3d8d6fc75e 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -27,7 +27,6 @@ # we change the version for the complete docs when forking of a release branch # etc. version: "1.2-SNAPSHOT" -version_hadoop1: "1.2-hadoop1-SNAPSHOT" version_short: "1.2" # Used for the top navbar w/o snapshot suffix is_snapshot_version: true diff --git a/docs/setup/building.md b/docs/setup/building.md index c6ef5df688d..8c609976daa 100644 --- a/docs/setup/building.md +++ b/docs/setup/building.md @@ -76,31 +76,11 @@ mvn clean install ## Hadoop Versions -{% info %} Most users do not need to do this manually. The [download page]({{ site.download_url }}) contains binary packages for common Hadoop versions. +{% info %} Most users do not need to do this manually. The [download page]({{ site.download_url }}) contains binary packages for common Hadoop versions. Flink has dependencies to HDFS and YARN which are both dependencies from [Apache Hadoop](http://hadoop.apache.org). There exist many different versions of Hadoop (from both the upstream project and the different Hadoop distributions). If you are using a wrong combination of versions, exceptions can occur. -There are two main versions of Hadoop that we need to differentiate: -- **Hadoop 1**, with all versions starting with zero or one, like *0.20*, *0.23* or *1.2.1*. -- **Hadoop 2**, with all versions starting with 2, like *2.6.0*. - -The main differentiation between Hadoop 1 and Hadoop 2 is the availability of [Hadoop YARN](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html), Hadoop's cluster resource manager. - -**By default, Flink is using the Hadoop 2 dependencies**. - -### Hadoop 1 - -To build Flink for Hadoop 1, issue the following command: - -~~~bash -mvn clean install -DskipTests -Dhadoop.profile=1 -~~~ - -The `-Dhadoop.profile=1` flag instructs Maven to build Flink for Hadoop 1. Note that the features included in Flink change when using a different Hadoop profile. In particular, there is no support for YARN and HBase in Hadoop 1 builds. - -### Hadoop 2.x - -Hadoop 2.X versions are only supported from version 2.3.0 upwards. +Hadoop is only supported from version 2.3.0 upwards. You can also specify a specific Hadoop version to build against: ~~~bash @@ -176,12 +156,3 @@ in the compiler configuration of the `pom.xml` file of the module causing the er {% top %} -## Internals - -The builds with Maven are controlled by [properties](http://maven.apache.org/pom.html#Properties) and [build profiles](http://maven.apache.org/guides/introduction/introduction-to-profiles.html). There are two profiles, one for `hadoop1` and one for `hadoop2`. When the `hadoop2` profile is enabled (default), the system will also build the YARN client. - -To enable the `hadoop1` profile, set `-Dhadoop.profile=1` when building. Depending on the profile, there are two Hadoop versions, set via properties. For `hadoop1`, we use 1.2.1 by default, for `hadoop2` it is 2.3.0. - -You can change these versions with the `hadoop-two.version` (or `hadoop-one.version`) property. For example `-Dhadoop-two.version=2.4.0`. - -{% top %} diff --git a/flink-batch-connectors/flink-hadoop-compatibility/pom.xml b/flink-batch-connectors/flink-hadoop-compatibility/pom.xml index 8143a03cefc..8f423d936c8 100644 --- a/flink-batch-connectors/flink-hadoop-compatibility/pom.xml +++ b/flink-batch-connectors/flink-hadoop-compatibility/pom.xml @@ -62,7 +62,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} diff --git a/flink-batch-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java b/flink-batch-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java index 3de3f7285f9..ed83d78f2cd 100644 --- a/flink-batch-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java +++ b/flink-batch-connectors/flink-hadoop-compatibility/src/test/java/org/apache/flink/test/hadoopcompatibility/mapreduce/example/WordCount.java @@ -77,7 +77,6 @@ public class WordCount { HadoopOutputFormat hadoopOutputFormat = new HadoopOutputFormat(new TextOutputFormat(), job); hadoopOutputFormat.getConfiguration().set("mapreduce.output.textoutputformat.separator", " "); hadoopOutputFormat.getConfiguration().set("mapred.textoutputformat.separator", " "); // set the value for both, since this test - // is being executed with both types (hadoop1 and hadoop2 profile) TextOutputFormat.setOutputPath(job, new Path(outputPath)); // Output & Execute diff --git a/flink-batch-connectors/flink-hbase/pom.xml b/flink-batch-connectors/flink-hbase/pom.xml index a1e0ad6486e..70a56925034 100644 --- a/flink-batch-connectors/flink-hbase/pom.xml +++ b/flink-batch-connectors/flink-hbase/pom.xml @@ -34,8 +34,7 @@ under the License. jar - 0.98.22-hadoop1 - 1.2.3 + 1.2.3 @@ -85,7 +84,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} provided @@ -198,100 +197,51 @@ under the License. - - - - - hadoop-1 - - - - hadoop.profile1 - - - - ${hbase.hadoop1.version} - - true - - - - - - hadoop-2 - - - hadoop-2-repo2 - https://repo.maven.apache.org/maven2 - - true - - - false - - - - - - - !hadoop.profile - - - - ${hbase.hadoop2.version} - - - - - - org.apache.hbase - hbase-server - ${hbase.version} - tests - test - - - - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - test - + + + org.apache.hbase + hbase-server + ${hbase.version} + tests + test + - - org.apache.hbase - hbase-hadoop-compat - ${hbase.version} - test - test-jar - + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + test + - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test-jar - test - + + org.apache.hbase + hbase-hadoop-compat + ${hbase.version} + test + test-jar + - - org.apache.hbase - hbase-hadoop2-compat - ${hbase.version} - test - test-jar - - + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + test-jar + test + - + + org.apache.hbase + hbase-hadoop2-compat + ${hbase.version} + test + test-jar + + + cdh5.1.3 - 2 0.98.1-cdh5.1.3 2.3.0-cdh5.1.3 diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index 319f6afc776..93feec623c9 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -131,28 +131,16 @@ under the License. flink-statebackend-rocksdb_2.10 ${project.version} + + + org.apache.flink + flink-yarn_2.10 + ${project.version} + - - - include-yarn - - - - !hadoop.profile - - - - - org.apache.flink - flink-yarn_2.10 - ${project.version} - - - - symlink-build-target diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml index 4ccaaec5511..f4806088e00 100644 --- a/flink-fs-tests/pom.xml +++ b/flink-fs-tests/pom.xml @@ -38,7 +38,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} test diff --git a/flink-java/pom.xml b/flink-java/pom.xml index 5bc81c64bef..728b6eeda43 100644 --- a/flink-java/pom.xml +++ b/flink-java/pom.xml @@ -43,7 +43,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} diff --git a/flink-mesos/pom.xml b/flink-mesos/pom.xml index 8814762b9c6..bf60f071346 100644 --- a/flink-mesos/pom.xml +++ b/flink-mesos/pom.xml @@ -50,7 +50,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index e62d9d4cca2..cf66bcf90af 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -166,7 +166,6 @@ under the License. Everything else will be packaged into the fat-jar --> org.apache.flink:flink-annotations - org.apache.flink:flink-shaded-hadoop1 org.apache.flink:flink-shaded-hadoop2 org.apache.flink:flink-shaded-curator-recipes org.apache.flink:flink-core diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index a62cf79a90b..afb9b509c87 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -170,7 +170,6 @@ under the License. Everything else will be packaged into the fat-jar --> org.apache.flink:flink-annotations - org.apache.flink:flink-shaded-hadoop1 org.apache.flink:flink-shaded-hadoop2 org.apache.flink:flink-shaded-curator-recipes org.apache.flink:flink-core diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index 69fdd21b11c..eec75c949b3 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -52,7 +52,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataOutputStream.java b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataOutputStream.java index d6fbc195f6e..8787181fbd6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataOutputStream.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/fs/hdfs/HadoopDataOutputStream.java @@ -19,8 +19,6 @@ package org.apache.flink.runtime.fs.hdfs; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import org.apache.flink.core.fs.FSDataOutputStream; @@ -57,78 +55,12 @@ public class HadoopDataOutputStream extends FSDataOutputStream { @Override public void flush() throws IOException { - if (HFLUSH_METHOD != null) { - try { - HFLUSH_METHOD.invoke(fdos); - } - catch (InvocationTargetException e) { - Throwable cause = e.getTargetException(); - if (cause instanceof IOException) { - throw (IOException) cause; - } - else if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - else if (cause instanceof Error) { - throw (Error) cause; - } - else { - throw new IOException("Exception while invoking hflush()", cause); - } - } - catch (IllegalAccessException e) { - throw new IOException("Cannot invoke hflush()", e); - } - } - else if (HFLUSH_ERROR != null) { - if (HFLUSH_ERROR instanceof NoSuchMethodException) { - throw new UnsupportedOperationException("hflush() method is not available in this version of Hadoop."); - } - else { - throw new IOException("Cannot access hflush() method", HFLUSH_ERROR); - } - } - else { - throw new UnsupportedOperationException("hflush() is not available in this version of Hadoop."); - } + fdos.hflush(); } @Override public void sync() throws IOException { - if (HSYNC_METHOD != null) { - try { - HSYNC_METHOD.invoke(fdos); - } - catch (InvocationTargetException e) { - Throwable cause = e.getTargetException(); - if (cause instanceof IOException) { - throw (IOException) cause; - } - else if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - else if (cause instanceof Error) { - throw (Error) cause; - } - else { - throw new IOException("Exception while invoking hsync()", cause); - } - } - catch (IllegalAccessException e) { - throw new IOException("Cannot invoke hsync()", e); - } - } - else if (HSYNC_ERROR != null) { - if (HSYNC_ERROR instanceof NoSuchMethodException) { - throw new UnsupportedOperationException("hsync() method is not available in this version of Hadoop."); - } - else { - throw new IOException("Cannot access hsync() method", HSYNC_ERROR); - } - } - else { - throw new UnsupportedOperationException("hsync() is not available in this version of Hadoop."); - } + fdos.hsync(); } /** @@ -138,42 +70,5 @@ public class HadoopDataOutputStream extends FSDataOutputStream { public org.apache.hadoop.fs.FSDataOutputStream getHadoopOutputStream() { return fdos; } - - // ------------------------------------------------------------------------ - // utilities to bridge hsync and hflush to hadoop, even through it is not supported in Hadoop 1 - // ------------------------------------------------------------------------ - - private static final Method HFLUSH_METHOD; - private static final Method HSYNC_METHOD; - - private static final Throwable HFLUSH_ERROR; - private static final Throwable HSYNC_ERROR; - - static { - Method hflush = null; - Method hsync = null; - Throwable flushError = null; - Throwable syncError = null; - - try { - hflush = org.apache.hadoop.fs.FSDataOutputStream.class.getMethod("hflush"); - } - catch (Throwable t) { - flushError = t; - } - - try { - hsync = org.apache.hadoop.fs.FSDataOutputStream.class.getMethod("hsync"); - } - catch (Throwable t) { - syncError = t; - } - - HFLUSH_METHOD = hflush; - HSYNC_METHOD = hsync; - - HFLUSH_ERROR = flushError; - HSYNC_ERROR = syncError; - } } diff --git a/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml deleted file mode 100644 index 891ab7d99c0..00000000000 --- a/flink-shaded-hadoop/flink-shaded-hadoop1/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - - - 4.0.0 - - - org.apache.flink - flink-shaded-hadoop - 1.2-SNAPSHOT - .. - - - flink-shaded-hadoop1 - flink-shaded-hadoop1 - - jar - - - - org.apache.hadoop - hadoop-core - ${hadoop.version} - - - asm - asm - - - org.ow2.asm - asm - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jsp-api-2.1 - - - org.mortbay.jetty - jsp-2.1 - - - org.eclipse.jdt - core - - - javax.servlet - servlet-api - - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-json - - - org.codehaus.jettison - jettison - - - com.sun.jersey - jersey-server - - - tomcat - jasper-compiler - - - tomcat - jasper-runtime - - - javax.servlet.jsp - jsp-api - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-grizzly2 - - - com.sun.jersey.jersey-test-framework - jersey-test-framework-core - - - javax.servlet - javax.servlet-api - - - com.sun.jersey - jersey-client - - - com.sun.jersey - jersey-grizzly2 - - - org.glassfish.grizzly - grizzly-http - - - org.glassfish.grizzly - grizzly-framework - - - org.glassfish.grizzly - grizzly-http-server - - - org.glassfish.grizzly - grizzly-rcm - - - org.glassfish.grizzly - grizzly-http-servlet - - - org.glassfish - javax.servlet - - - com.sun.jersey.contribs - jersey-guice - - - - - - diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml index 47c1a7f2576..d1c8d494547 100644 --- a/flink-shaded-hadoop/pom.xml +++ b/flink-shaded-hadoop/pom.xml @@ -35,7 +35,7 @@ under the License. pom - ${shading-artifact.name} + flink-shaded-hadoop2 @@ -50,12 +50,6 @@ under the License. include-yarn-tests - - - - !hadoop.profile - - flink-shaded-include-yarn-tests diff --git a/flink-streaming-connectors/flink-connector-filesystem/pom.xml b/flink-streaming-connectors/flink-connector-filesystem/pom.xml index ef7e72ba701..20c48c606c3 100644 --- a/flink-streaming-connectors/flink-connector-filesystem/pom.xml +++ b/flink-streaming-connectors/flink-connector-filesystem/pom.xml @@ -52,7 +52,7 @@ under the License. org.apache.flink - ${shading-artifact.name} + flink-shaded-hadoop2 ${project.version} provided diff --git a/flink-streaming-connectors/pom.xml b/flink-streaming-connectors/pom.xml index 78e39cadb51..a4c27c23296 100644 --- a/flink-streaming-connectors/pom.xml +++ b/flink-streaming-connectors/pom.xml @@ -48,25 +48,11 @@ under the License. flink-connector-nifi flink-connector-cassandra flink-connector-redis + flink-connector-filesystem - - hadoop-2 - - - - !hadoop.profile - - - - - flink-connector-filesystem - - - error never-match-me - 1.2.1 - 2.3.0 + 2.3.0 @@ -441,43 +440,6 @@ under the License. - - - hadoop-1 - - - - hadoop.profile1 - - - - ${hadoop-one.version} - flink-shaded-hadoop1 - - - - - - hadoop-2 - - - - !hadoop.profile - - - - ${hadoop-two.version} - flink-shaded-hadoop2 - - - flink-yarn - - flink-fs-tests - - - include-yarn-tests diff --git a/tools/create_release_files.sh b/tools/create_release_files.sh index 8a4b7967785..fdf50a53a96 100755 --- a/tools/create_release_files.sh +++ b/tools/create_release_files.sh @@ -70,7 +70,6 @@ OLD_VERSION=${OLD_VERSION:-1.1-SNAPSHOT} RELEASE_VERSION=${NEW_VERSION} RELEASE_CANDIDATE=${RELEASE_CANDIDATE:-rc1} RELEASE_BRANCH=${RELEASE_BRANCH:-master} -NEW_VERSION_HADOOP1=${NEW_VERSION_HADOOP1:-"$RELEASE_VERSION-hadoop1"} USER_NAME=${USER_NAME:-yourapacheidhere} MVN=${MVN:-mvn} GPG=${GPG:-gpg} @@ -115,7 +114,6 @@ make_source_release() { #change version of documentation cd docs perl -pi -e "s#^version: .*#version: ${NEW_VERSION}#" _config.yml - perl -pi -e "s#^version_hadoop1: .*#version_hadoop1: ${NEW_VERSION}-hadoop1#" _config.yml perl -pi -e "s#^version_short: .*#version_short: ${NEW_VERSION}#" _config.yml cd .. @@ -185,14 +183,6 @@ deploy_to_maven() { echo "Deploying Scala 2.10 version" cd tools && ./change-scala-version.sh 2.10 && cd .. $MVN clean deploy -Dgpg.executable=$GPG -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10 - - - echo "Deploying Scala 2.10 / hadoop 1 version" - ../generate_specific_pom.sh $NEW_VERSION $NEW_VERSION_HADOOP1 pom.xml - - - sleep 4 - $MVN clean deploy -Dgpg.executable=$GPG -Prelease,docs-and-source --settings deploysettings.xml -DskipTests -Dgpg.keyname=$GPG_KEY -Dgpg.passphrase=$GPG_PASSPHRASE -DretryFailedDeploymentCount=10 } copy_data() { @@ -211,7 +201,6 @@ prepare make_source_release -make_binary_release "hadoop1" "-Dhadoop.profile=1" 2.10 make_binary_release "hadoop2" "" 2.10 make_binary_release "hadoop24" "-Dhadoop.version=2.4.1" 2.10 make_binary_release "hadoop26" "-Dhadoop.version=2.6.3" 2.10 diff --git a/tools/deploy_to_maven.sh b/tools/deploy_to_maven.sh index 74ae8a79ae0..676d8d0dbd5 100755 --- a/tools/deploy_to_maven.sh +++ b/tools/deploy_to_maven.sh @@ -75,13 +75,8 @@ rm -rf dummy-lifecycle-mapping-plugin CURRENT_FLINK_VERSION=`getVersion` -if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then - CURRENT_FLINK_VERSION_HADOOP1=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop1-SNAPSHOT} -else - CURRENT_FLINK_VERSION_HADOOP1="$CURRENT_FLINK_VERSION-hadoop1" -fi -echo "detected current version as: '$CURRENT_FLINK_VERSION' ; hadoop1: $CURRENT_FLINK_VERSION_HADOOP1 " +echo "detected current version as: '$CURRENT_FLINK_VERSION'" # # This script deploys our project to sonatype SNAPSHOTS. @@ -92,13 +87,6 @@ if [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then MVN_SNAPSHOT_OPTS="-B -Pdocs-and-source -DskipTests -Drat.skip=true -Drat.ignoreErrors=true \ -DretryFailedDeploymentCount=10 --settings deploysettings.xml clean deploy" - # Deploy hadoop v1 to maven - echo "Generating poms for hadoop1" - ./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_HADOOP1 pom.hadoop1.xml - mvn -f pom.hadoop1.xml ${MVN_SNAPSHOT_OPTS} - # deploy to s3 - deploy_to_s3 $CURRENT_FLINK_VERSION "hadoop1" - # hadoop2 scala 2.10 echo "deploy standard version (hadoop2) for scala 2.10" mvn ${MVN_SNAPSHOT_OPTS} diff --git a/tools/generate_specific_pom.sh b/tools/generate_specific_pom.sh deleted file mode 100755 index 11feefd671f..00000000000 --- a/tools/generate_specific_pom.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - - -# -# Inspired by and modified from: -# https://github.com/apache/hbase/blob/trunk/dev-support/generate-hadoopX-poms.sh -# - -function usage { - echo "Usage: $0 CURRENT_VERSION NEW_VERSION [POM_NAME]" - echo "For example, $0 0.8-incubating-SNAPSHOT 0.8-hadoop1-incubating-SNAPSHOT" - echo "Presumes VERSION has hadoop1 or hadoop2 in it. POM_NAME is optional and" - echo "allows to specify a different name for the generated pom." - exit 1 -} - -if [[ "$#" -lt 2 ]]; then usage; fi - -old_version="$1" -new_version="$2" -new_pom_name="$3" - -# Get hadoop version from the new Flink version -hadoop_version=`echo "$new_version" | sed -n 's/.*\(hadoop[12]\).*/\1/p'` -if [[ -z $hadoop_version ]]; then usage ; fi - -echo "hadoop version $hadoop_version" - - -here="`dirname \"$0\"`" # relative -here="`( cd \"$here\" && pwd )`" # absolutized and normalized -if [ -z "$here" ] ; then - # error; for some reason, the path is not accessible - # to the script (e.g. permissions re-evaled after suid) - exit 1 # fail -fi -flink_home="`dirname \"$here\"`" - - -hadoop1= -hadoop2= -default='!hadoop.profile<\/name>' -notdefault='hadoop.profile<\/name>' -case "${hadoop_version}" in - hadoop1) - hadoop1="${default}" - hadoop2="${notdefault}2<\/value>" - ;; - hadoop2) - hadoop1="${notdefault}1<\/value>" - hadoop2="${default}" - ;; - *) echo "Unknown ${hadoop_version}" - usage - ;; -esac - -nupom=$new_pom_name -if [[ -z "$new_pom_name" ]]; then - nupom="pom.${hadoop_version}.xml" -fi -echo "Using $nupom as name for the generated pom file." - -# export relevant variables for find command subshells -export old_version -export new_version -export hadoop1 -export hadoop2 -export nupom - -# paths may contain spaces -find "$flink_home" -name pom.xml -exec bash -c ' - - p="$0" - - # write into tmp file because in-place replacement is not possible (if nupom="pom.xml") - tmp_nuname1="`dirname "$p"`/__generate_specific_pom_tmp1" - tmp_nuname2="`dirname "$p"`/__generate_specific_pom_tmp2" - nuname="`dirname "$p"`/${nupom}" - # Now we do search and replace of explicit strings. The best way of - # seeing what the below does is by doing a diff between the original - # pom and the generated pom (pom.hadoop1.xml or pom.hadoop2.xml). We - # replace the version string in all poms, we change modules to - # include reference to the non- standard pom name, we adjust - # relative paths so child modules can find the parent pom, and we - # enable/disable hadoop 1 and hadoop 2 profiles as appropriate - # removing a comment string too. We output the new pom beside the - # original. - - # To avoid accidentally replace version numbers in our dependencies - # sharing the version number with the current release use the following. - - echo "p=$p, old_version=${old_version}, new_version=$new_version" - - perl -0777 -pe "s:org.apache.flink\n([\t ]*([a-z]+-)+[a-z0-9\.\_]+\n[\t ]*)${old_version}:org.apache.flink\n\1${new_version}:g" "$p" > "$tmp_nuname1" - - # replace the version also in the quickstart poms (so that the hadoop1 quickstart creates an hadoop1 project) - perl -0777 -pe "s:${old_version}:${new_version}:g" "$tmp_nuname1" > "$tmp_nuname2" - - # Alternatively when no version collisions are present this is enough: - # sed -e "s/${old_version}/${new_version}/" $p > "$tmp_nuname1" - - sed -e "s/\([^<]*\)/\1\/${nupom}/" \ - -e "s/\(relativePath>\.\.\)/\1\/${nupom}/" \ - -e "s/.*name>.*/${hadoop1}/" \ - -e "s/.*name>.*/${hadoop2}/" \ - "$tmp_nuname2" > "$tmp_nuname1" - rm "$tmp_nuname2" - mv "$tmp_nuname1" "$nuname" - -' "{}" \; # pass file name as argument - -- GitLab