[FLINK-9091][build] Dependency convergence run against dependency-reduced poms

This closes #6102.

pom.xml

@@ -644,6 +644,29 @@ under the License.
 			</build>
 		</profile>
 
+		<profile>
+			<id>check-convergence</id>
+			<activation>
+				<property>
+					<name>check-convergence</name>
+				</property>
+			</activation>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-enforcer-plugin</artifactId>
+						<executions>
+							<execution>
+								<id>dependency-convergence</id>
+								<phase>validate</phase>
+							</execution>
+						</executions>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+
 		<profile>
 			<id>legacyCode</id>
 			<activation>
@@ -1243,6 +1266,8 @@ under the License.
 					</execution>
 					<execution>
 						<id>dependency-convergence</id>
+						<!-- disabled by default as it interacts badly with shade-plugin -->
+						<phase>none</phase>
 						<goals>
 							<goal>enforce</goal>
 						</goals>

tools/check_dependency_convergence.sh

+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+HERE="`dirname \"$0\"`"				# relative
+HERE="`( cd \"$HERE\" && pwd )`" 	# absolutized and normalized
+if [ -z "$HERE" ] ; then
+	# error; for some reason, the path is not accessible
+	# to the script (e.g. permissions re-evaled after suid)
+	exit 1  # fail
+fi
+
+FLINK_DIR=HERE
+
+if [[ $(basename ${HERE}) == "tools" ]] ; then
+  FLINK_DIR="${HERE}/.."
+fi
+
+FLINK_DIR="`( cd \"${FLINK_DIR}\" && pwd )`" 
+
+echo ${FLINK_DIR}
+
+# get list of all flink modules
+# searches for directories containing a pom.xml file
+# sorts the list alphabetically
+# only accepts directories starting with "flink" to filter force-shading
+modules=$(find . -maxdepth 3 -name 'pom.xml' -printf '%h\n' | sort -u | grep "flink")
+
+for module in ${modules}
+do
+    # we are only interested in child modules
+    for other_module in ${modules}
+    do 
+        if [[ "${other_module}" != "${module}" && "${other_module}" = "${module}"/* ]]; then
+        echo "excluding ${module} since it is not a leaf module"
+            continue 2
+        fi
+    done
+    
+    cd "${module}"
+    echo "checking ${module}"
+    output=$(mvn validate -nsu -Dcheckstyle.skip=true -Dcheck-convergence)
+    exit_code=$?
+    if [[ ${exit_code} != 0 ]]; then
+        echo "dependency convergence failed."
+        echo "${output}"
+        exit ${exit_code}
+    fi
+    cd "${FLINK_DIR}"
+done
+
+exit 0

tools/travis_mvn_watchdog.sh

@@ -509,12 +509,40 @@ EXIT_CODE=$(<$MVN_EXIT)
 
 echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
 
+# Make sure to kill the watchdog in any case after $MVN_COMPILE has completed
+echo "Trying to KILL watchdog (${WD_PID})."
+( kill $WD_PID 2>&1 ) > /dev/null
+
 rm $MVN_PID
 rm $MVN_EXIT
 
+# only run dependency-convergence in misc because it is the only profile building all of Flink
+case $TEST in
+	(misc)
+		if [ $EXIT_CODE == 0 ]; then
+			printf "\n\n==============================================================================\n"
+			printf "Checking dependency convergence\n"
+			printf "==============================================================================\n"
+
+			./tools/check_dependency_convergence.sh
+			EXIT_CODE=$?
+		else
+			printf "\n==============================================================================\n"
+			printf "Previous build failure detected, skipping dependency-convergence check.\n"
+			printf "==============================================================================\n"
+		fi
+	;;
+esac
+
 # Run tests if compilation was successful
 if [ $EXIT_CODE == 0 ]; then
 
+	# Start watching $MVN_OUT
+	watchdog &
+	echo "STARTED watchdog (${WD_PID})."
+
+	WD_PID=$!
+
 	echo "RUNNING '${MVN_TEST}'."
 
 	# Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
@@ -526,6 +554,10 @@ if [ $EXIT_CODE == 0 ]; then
 
 	echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
 
+	# Make sure to kill the watchdog in any case after $MVN_TEST has completed
+	echo "Trying to KILL watchdog (${WD_PID})."
+	( kill $WD_PID 2>&1 ) > /dev/null
+
 	rm $MVN_PID
 	rm $MVN_EXIT
 else
@@ -536,10 +568,6 @@ fi
 
 # Post
 
-# Make sure to kill the watchdog in any case after $MVN_COMPILE and $MVN_TEST have completed
-echo "Trying to KILL watchdog (${WD_PID})."
-( kill $WD_PID 2>&1 ) > /dev/null
-
 # only misc builds flink-dist and flink-yarn-tests
 case $TEST in
 	(misc)