[FLINK-13417][build] Bundle Zookeeper 3.5 in opt

docs/ops/jobmanager_high_availability.md

@@ -229,6 +229,15 @@ zookeeper.sasl.login-context-name: Client  # default is "Client". The value need
 For more information on Flink configuration for Kerberos security, please see [here]({{ site.baseurl}}/ops/config.html).
 You can also find [here]({{ site.baseurl}}/ops/security-kerberos.html) further details on how Flink internally setups Kerberos-based security.
 
+## Zookeeper Versions
+
+Flink ships with separate Zookeeper clients for 3.4 and 3.5, with 3.4 being in the `lib` directory of the distribution
+and thus used by default, whereas 3.5 is placed in the `opt` directory.
+
+The 3.5 client allows you to secure the Zookeeper connection via SSL, but _may_ not work with 3.4- Zookeeper installations.
+
+You can control which version is used by Flink by placing either jar in the `lib` directory.
+
 ## Bootstrap ZooKeeper
 
 If you don't have a running ZooKeeper installation, you can use the helper scripts, which ship with Flink.

docs/ops/jobmanager_high_availability.zh.md

@@ -229,6 +229,15 @@ zookeeper.sasl.login-context-name: Client  # default is "Client". The value need
 For more information on Flink configuration for Kerberos security, please see [here]({{ site.baseurl}}/ops/config.html).
 You can also find [here]({{ site.baseurl}}/ops/security-kerberos.html) further details on how Flink internally setups Kerberos-based security.
 
+## Zookeeper Versions
+
+Flink ships with separate Zookeeper clients for 3.4 and 3.5, with 3.4 being in the `lib` directory of the distribution
+and thus used by default, whereas 3.5 is placed in the `opt` directory.
+
+The 3.5 client allows you to secure the Zookeeper connection via SSL, but _may_ not work with 3.4- Zookeeper installations.
+
+You can control which version is used by Flink by placing either jar in the `lib` directory.
+
 ## Bootstrap ZooKeeper
 
 If you don't have a running ZooKeeper installation, you can use the helper scripts, which ship with Flink.

flink-dist/pom.xml

@@ -33,6 +33,10 @@ under the License.
 	<name>flink-dist</name>
 	<packaging>jar</packaging>
 
+	<properties>
+		<zookeeper.optional.version>3.5.6</zookeeper.optional.version>
+	</properties>
+
 	<dependencies>
 
 		<!-- Flink project binaries -->
@@ -634,6 +638,25 @@ under the License.
 							<outputDirectory>${project.build.directory}/temporary</outputDirectory>
 						</configuration>
 					</execution>
+					<execution>
+						<id>copy-zk-jars</id>
+						<phase>process-resources</phase>
+						<goals>
+							<goal>copy</goal>
+						</goals>
+						<configuration>
+							<artifactItems>
+								<artifactItem>
+									<groupId>org.apache.flink</groupId>
+									<artifactId>flink-shaded-zookeeper-3</artifactId>
+									<version>${zookeeper.optional.version}-${flink.shaded.version}</version>
+									<type>jar</type>
+									<overWrite>true</overWrite>
+								</artifactItem>
+							</artifactItems>
+							<outputDirectory>${project.build.directory}/temporary</outputDirectory>
+						</configuration>
+					</execution>
 				</executions>
 			</plugin>
 
@@ -693,6 +716,8 @@ under the License.
 								<excludes>
 									<!-- log4j 2 is bundled separately from the flink-dist jar -->
 									<exclude>org.apache.logging.log4j:*</exclude>
+									<!-- Bundled separately so that users can easily switch between ZK 3.4/3.5-->
+									<exclude>org.apache.flink:flink-shaded-zookeeper-3</exclude>
 								</excludes>
 							</artifactSet>
 							<transformers>

flink-dist/src/main/assemblies/bin.xml

@@ -44,6 +44,14 @@ under the License.
 				<include>org.apache.logging.log4j:log4j-1.2-api</include>
 			</includes>
 		</dependencySet>
+		<dependencySet>
+			<outputDirectory>lib</outputDirectory>
+			<unpack>false</unpack>
+			<includes>
+				<include>org.apache.flink:flink-shaded-zookeeper-3:jar:${zookeeper.version}-${flink.shaded.version}</include>
+			</includes>
+			<outputFileNameMapping>flink-shaded-zookeeper-${zookeeper.version}.jar</outputFileNameMapping>
+		</dependencySet>
 	</dependencySets>
 
 	<files>

flink-dist/src/main/assemblies/opt.xml

@@ -176,6 +176,13 @@
 			<destName>flink-ml_${scala.binary.version}-${project.version}.jar</destName>
 			<fileMode>0644</fileMode>
 		</file>
+
+		<file>
+			<source>target/temporary/flink-shaded-zookeeper-3-${zookeeper.optional.version}-${flink.shaded.version}.jar</source>
+			<outputDirectory>opt</outputDirectory>
+			<destName>flink-shaded-zookeeper-${zookeeper.optional.version}.jar</destName>
+			<fileMode>0644</fileMode>
+		</file>
 	</files>
 	<dependencySets>
 		<dependencySet>

pom.xml

@@ -120,6 +120,7 @@ under the License.
 		<scala.binary.version>2.11</scala.binary.version>
 		<chill.version>0.7.6</chill.version>
 		<zookeeper.version>3.4.10</zookeeper.version>
+		<!-- Only the curator2 TestingServer works with ZK 3.4 -->
 		<curator.version>2.12.0</curator.version>
 		<jackson.version>2.10.1</jackson.version>
 		<metrics.version>3.1.5</metrics.version>