[FLINK-1024] Enable automated snapshot deployment through Apache's Maven infrastructure

dc55cc35 · Robert Metzger · 3827f062 · dc55cc35 · dc55cc35 · dc55cc35
8 changed file
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,9 +15,9 @@ notifications:

 env:
    global: 
-        # username and password for sonatype (maven deploy)
-        - secure: "iQrrSH9vL/ANeqj0u2qC1Ft4eAor2KF6mrqA4T6/5Woxqoq22siw9SGcSW7UmLr3fpbzqsapK8XX8Rkpgsi6V7YTpACZyj0tIX6knoy08KK4qJaGEDx+8FlM61l9qQCQ5gFVVu+cFvI+xd3tDCoZuGSZUjZOjNpYZZqSunsOZ7E="
-        - secure: "aip84Xhcy9jYUtLrdfL4uWNATmLgqwIU/Gn9dR85uPJHeVPq7gSq3yrmiLJkEUgc6WyLdsKmiGKPPqwAG2Hc57i/cdz3YSq0Cn/Ksuxviec66HvNIhIFLwuXQkWF/W7MLmMgL6kVD8PxMn+A3d/4aEUKX4ELhoAmBhULOYiXi+E="
+        # username and password for Apache Nexus (maven deploy)
+        - secure: "Nu2oNTrIAmxIkNEZzALw+GT2QBogEh/mqecSqoKDCk0oFjUZhrnrsIZYD/8zTG9fAVa5Gx4uWH4W824Va5RlBZvCs9UTh5TF25K2ORR9dB9FiXZ+Vjjig78sKJF7N73WVIOsHCSKpoBKnVkvNwxuAkPTMYjn3sswRh1pMu2VQ90="
+        - secure: "jMllQXAHpE+ijYXjvQvh0xml6DCL5pmESuWRtd0Wi4v56HHxKHc/Tty/CJvX8whVDLaHNFtwlbaIN9asSyAu1OyGhpWCqsmsxWF4atvKFua1oX45XMB26Ymf7Yr7aq7lcx66j0cYpfBXY4tFTFPiT05QnZ8XsHzEnv4Tpgif2dg="
        # New s3 deployment
        - ARTIFACTS_S3_BUCKET="stratosphere-bin"
        - secure: "AECzVxihEhYfnNcrY/wLirTkKkmSATycvTfKsBmxD07bg6BmaVgsOl4degUu4YL50e6agpoWul6irGxTg0bjLMAwg1ZGyRx57NFvNQ7JYDHK6EWmJ7BsK2WO7HiYzfau+ZAaL36WpOMi0UUPpuNXMvULqaE9b4jZqo1Wo/WDcyU="

--- a/deploysettings.xml
+++ b/deploysettings.xml
@@ -24,12 +24,12 @@ under the License.
                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
  <servers>
    <server>
-      <id>sonatype-nexus-snapshots</id>
+      <id>apache.snapshots.https</id>
      <username>${sonatype_user}</username>
      <password>${sonatype_pw}</password>
    </server>
    <server>
-      <id>sonatype-nexus-staging</id>
+      <id>apache.releases.https</id>
      <username>${sonatype_user}</username>
      <password>${sonatype_pw}</password>
    </server>

--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -32,6 +32,20 @@ under the License.
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 	</properties>

+	<repositories>
+		<repository>
+			<id>apache.snapshots</id>
+			<name>Apache Development Snapshot Repository</name>
+			<url>https://repository.apache.org/content/repositories/snapshots/</url>
+			<releases>
+				<enabled>false</enabled>
+			</releases>
+			<snapshots>
+				<enabled>true</enabled>
+			</snapshots>
+		</repository>
+	</repositories>
+	
 	<!-- These two requirements are the minimum to use and develop Flink. 
 		You can add others like <artifactId>pact-scala-core</artifactId> for Scala! -->
 	<dependencies>

--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -27,7 +27,20 @@ under the License.

  <name>Your Job's Name</name>
  <url>http://www.myorganization.org</url>
-
+  
+  <repositories>
+    <repository>
+      <id>apache.snapshots</id>
+      <name>Apache Development Snapshot Repository</name>
+      <url>https://repository.apache.org/content/repositories/snapshots/</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

--- a/flink-quickstart/pom.xml
+++ b/flink-quickstart/pom.xml
@@ -33,91 +33,75 @@ under the License.
 	<name>flink-quickstart</name>
 	<description>Parent project for different quickstart archetypes for Apache Flink</description>
 	
+	<modules>
+		<module>flink-quickstart-java</module>
+		<module>flink-quickstart-scala</module>
+	</modules>

-	<!-- See http://www.imixs.org/jee/archetype/build.html -->
-	<!-- Distributen Management oss.sonatype.org -->
-	<distributionManagement>
-		<snapshotRepository>
-			<id>sonatype-nexus-snapshots</id>
-			<name>Sonatype Nexus Snapshots</name>
-			<url>http://oss.sonatype.org/content/repositories/snapshots</url>
-		</snapshotRepository>
-		<repository>
-			<id>sonatype-nexus-staging</id>
-			<name>Nexus Release Repository</name>
-			<url>http://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
-		</repository>
-	</distributionManagement>
-
-		<modules>
-			<module>flink-quickstart-java</module>
-			<module>flink-quickstart-scala</module>
-		</modules>
-
-		<profiles>
-			<profile>
-				<id>release</id>
-					<build>
-						<plugins>
-							<!-- source attachment -->
-							<plugin>
-								<groupId>org.apache.maven.plugins</groupId>
-								<artifactId>maven-source-plugin</artifactId>
-								<version>2.2.1</version>
-								<executions>
-									<execution>
-										<id>attach-sources</id>
-										<goals>
-											<goal>jar</goal>
-										</goals>
-									</execution>
-								</executions>
-							</plugin>
-							<!-- Javadocs -->
-							<plugin>
-								<groupId>org.apache.maven.plugins</groupId>
-								<artifactId>maven-javadoc-plugin</artifactId>
-								<version>2.9.1</version>
-									<executions>
-									<execution>
-										<id>attach-javadocs</id>
-										<goals>
-											<goal>jar</goal>
-										</goals>
-									</execution>
-								</executions>
-							</plugin>
-							<!-- signing -->
-							<plugin>
+	<profiles>
+		<profile>
+			<id>release</id>
+				<build>
+					<plugins>
+						<!-- source attachment -->
+						<plugin>
 							<groupId>org.apache.maven.plugins</groupId>
-							<artifactId>maven-gpg-plugin</artifactId>
-							<version>1.4</version>
+							<artifactId>maven-source-plugin</artifactId>
+							<version>2.2.1</version>
 							<executions>
 								<execution>
-								<id>sign-artifacts</id>
-								<phase>verify</phase>
-								<goals>
-									<goal>sign</goal>
-								</goals>
+									<id>attach-sources</id>
+									<goals>
+										<goal>jar</goal>
+									</goals>
 								</execution>
 							</executions>
+						</plugin>
+						<!-- Javadocs -->
+						<plugin>
+							<groupId>org.apache.maven.plugins</groupId>
+							<artifactId>maven-javadoc-plugin</artifactId>
+							<version>2.9.1</version>
+								<executions>
+								<execution>
+									<id>attach-javadocs</id>
+									<goals>
+										<goal>jar</goal>
+									</goals>
+								</execution>
+							</executions>
+						</plugin>
+						<!-- signing -->
+						<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-gpg-plugin</artifactId>
+						<version>1.4</version>
+						<executions>
+							<execution>
+							<id>sign-artifacts</id>
+							<phase>verify</phase>
+							<goals>
+								<goal>sign</goal>
+							</goals>
+							</execution>
+						</executions>
+						</plugin>
+					</plugins>
+					<pluginManagement>
+						<plugins>
+							<plugin>
+								<groupId>org.apache.maven.plugins</groupId>
+								<artifactId>maven-release-plugin</artifactId>
+								<version>2.1</version>
+								<configuration>
+									<mavenExecutorId>forked-path</mavenExecutorId>
+									<useReleaseProfile>false</useReleaseProfile>
+									<arguments>${arguments} -Psonatype-oss-release</arguments>
+								</configuration>
 							</plugin>
 						</plugins>
-						<pluginManagement>
-							<plugins>
-								<plugin>
-									<groupId>org.apache.maven.plugins</groupId>
-									<artifactId>maven-release-plugin</artifactId>
-									<version>2.1</version>
-									<configuration>
-										<mavenExecutorId>forked-path</mavenExecutorId>
-										<useReleaseProfile>false</useReleaseProfile>
-										<arguments>${arguments} -Psonatype-oss-release</arguments>
-									</configuration>
-								</plugin>
-							</plugins>
-						</pluginManagement>
-					</build>
-			</profile>
-		</profiles>
+					</pluginManagement>
+				</build>
+		</profile>
+	</profiles>
 </project>
\ No newline at end of file
--- a/flink-quickstart/quickstart-SNAPSHOT.sh
+++ b/flink-quickstart/quickstart-SNAPSHOT.sh
@@ -30,7 +30,7 @@ mvn archetype:generate								\
  -Dversion=0.1										\
  -Dpackage=org.apache.flink 						\
  -DinteractiveMode=false							\
-  -DarchetypeCatalog=https://oss.sonatype.org/content/repositories/snapshots/
+  -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/

 #
 # Give some guidance

--- a/flink-quickstart/quickstart-scala-SNAPSHOT.sh
+++ b/flink-quickstart/quickstart-scala-SNAPSHOT.sh
@@ -30,7 +30,7 @@ mvn archetype:generate								\
  -Dversion=0.1										\
  -Dpackage=org.apache.flink.quickstart 			\
  -DinteractiveMode=false							\
-  -DarchetypeCatalog=https://oss.sonatype.org/content/repositories/snapshots/
+  -DarchetypeCatalog=https://repository.apache.org/content/repositories/snapshots/

 #
 # Give some guidance

--- a/tools/deploy_to_maven.sh
+++ b/tools/deploy_to_maven.sh
@@ -46,20 +46,20 @@ function getVersion() {
 	  # to the script (e.g. permissions re-evaled after suid)
 	  exit 1  # fail
 	fi
-	stratosphere_home="`dirname \"$here\"`"
-	cd $stratosphere_home
+	flink_home="`dirname \"$here\"`"
+	cd $flink_home
 	echo `mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)'`
 }

 # this will take a while
-CURRENT_STRATOSPHERE_VERSION=`getVersion`
-if [[ "$CURRENT_STRATOSPHERE_VERSION" == *-SNAPSHOT ]]; then
-	CURRENT_STRATOSPHERE_VERSION_YARN=${CURRENT_STRATOSPHERE_VERSION/-SNAPSHOT/-hadoop2-SNAPSHOT}
+CURRENT_FLINK_VERSION=`getVersion`
+if [[ "$CURRENT_FLINK_VERSION" == *-SNAPSHOT ]]; then
+	CURRENT_FLINK_VERSION_YARN=${CURRENT_FLINK_VERSION/-SNAPSHOT/-hadoop2-SNAPSHOT}
 else
-	CURRENT_STRATOSPHERE_VERSION_YARN="$CURRENT_STRATOSPHERE_VERSION-hadoop2"
+	CURRENT_FLINK_VERSION_YARN="$CURRENT_FLINK_VERSION-hadoop2"
 fi

-echo "detected current version as: '$CURRENT_STRATOSPHERE_VERSION' ; yarn: $CURRENT_STRATOSPHERE_VERSION_YARN "
+echo "detected current version as: '$CURRENT_FLINK_VERSION' ; yarn: $CURRENT_FLINK_VERSION_YARN "

 # Check if push/commit is eligible for pushing
 echo "Job: $TRAVIS_JOB_NUMBER ; isPR: $TRAVIS_PULL_REQUEST"
@@ -70,24 +70,24 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
 	# It will deploy both a hadoop v1 and a hadoop v2 (yarn) artifact
 	# 

-	if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then 
+	if [[ $TRAVIS_JOB_NUMBER == *1 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then 
 		# Deploy regular hadoop v1 to maven
-		mvn -DskipTests deploy --settings deploysettings.xml; 
+		mvn -DskipTests -Drat.ignoreErrors=true deploy --settings deploysettings.xml; 
 	fi

-	if [[ $TRAVIS_JOB_NUMBER == *4 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then 
+	if [[ $TRAVIS_JOB_NUMBER == *4 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then 
 		# deploy hadoop v2 (yarn)
 		echo "Generating poms for hadoop-yarn."
-		./tools/generate_specific_pom.sh $CURRENT_STRATOSPHERE_VERSION $CURRENT_STRATOSPHERE_VERSION_YARN
+		./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_YARN
 		# all these tweaks assume a yarn build.
 		# performance tweaks here: no "clean deploy" so that actually nothing is being rebuild (could cause wrong poms inside the jars?)
 		# skip tests (they were running already)
 		# skip javadocs generation (already generated)
-		mvn -B -f pom.hadoop2.xml -DskipTests -Dmaven.javadoc.skip=true deploy --settings deploysettings.xml; 
+		mvn -B -f pom.hadoop2.xml -DskipTests -Drat.ignoreErrors=true -Dmaven.javadoc.skip=true deploy --settings deploysettings.xml; 
 	fi

-	if [[ $TRAVIS_JOB_NUMBER == *5 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_STRATOSPHERE_VERSION == *SNAPSHOT* ]] ; then 
-		cd stratosphere-java
+	if [[ $TRAVIS_JOB_NUMBER == *5 ]] && [[ $TRAVIS_PULL_REQUEST == "false" ]] && [[ $CURRENT_FLINK_VERSION == *SNAPSHOT* ]] ; then 
+		cd flink-java
 		mvn javadoc:javadoc
 		cd target
 		cd apidocs
@@ -118,19 +118,19 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
 	if [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then 
 		#generate yarn poms & build for yarn.
 		# it is not required to generate poms for this build.
-		#./tools/generate_specific_pom.sh $CURRENT_STRATOSPHERE_VERSION $CURRENT_STRATOSPHERE_VERSION_YARN pom.xml
+		#./tools/generate_specific_pom.sh $CURRENT_FLINK_VERSION $CURRENT_FLINK_VERSION_YARN pom.xml
 		#mvn -B -DskipTests clean install
-		CURRENT_STRATOSPHERE_VERSION=$CURRENT_STRATOSPHERE_VERSION_YARN
-		YARN_ARCHIVE="stratosphere-dist/target/*yarn.tar.gz"
+		CURRENT_FLINK_VERSION=$CURRENT_FLINK_VERSION_YARN
+		YARN_ARCHIVE="flink-dist/target/*yarn.tar.gz"
 	fi
 	if [[ $TRAVIS_JOB_NUMBER == *3 ]] || [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then 
-	#	cd stratosphere-dist
+	#	cd flink-dist
 	#	mvn -B -DskipTests -Pdebian-package package
 	#	cd ..
 		echo "Uploading build to amazon s3. Job Number: $TRAVIS_JOB_NUMBER"
-		mkdir stratosphere
-		cp -r stratosphere-dist/target/stratosphere-dist-*-bin/stratosphere*/* stratosphere/
-		tar -czf stratosphere-$CURRENT_STRATOSPHERE_VERSION.tgz stratosphere
+		mkdir flink
+		cp -r flink-dist/target/flink-dist-*-bin/flink*/* flink/
+		tar -czf flink-$CURRENT_FLINK_VERSION.tgz flink
 		
 		# upload the two in parallel
 		if [[ $TRAVIS_JOB_NUMBER == *6 ]] ; then
@@ -138,7 +138,7 @@ if [[ $TRAVIS_PULL_REQUEST == "false" ]] ; then
 			mv $YARN_ARCHIVE .
 			travis-artifacts upload --path *yarn.tar.gz --target-path / 
 		fi
-		travis-artifacts upload --path stratosphere-$CURRENT_STRATOSPHERE_VERSION.tgz   --target-path / 
+		travis-artifacts upload --path flink-$CURRENT_FLINK_VERSION.tgz   --target-path / 
 	fi

 fi # pull request check