[FLINK-4118] Update docker image to 1.0.3 and remove unneeded deps

Some of the changes include:

- Remove unneeded dependencies (nano, wget)
- Remove apt lists to reduce image size
- Reduce number of layers on the docker image (best docker practice)
- Remove useless variables and base the code in generic ones e.g.
FLINK_HOME
- Change the default JDK from oracle to openjdk-8-jre-headless, based on
two reasons:

1. You cannot legally repackage the oracle jdk in docker images
2. The open-jdk headless is more appropriate for a server image (no GUI stuff)

- Return port assignation to the standard FLINK one:

Variable: docker-flink -> flink

taskmanager.rpc.port: 6121 -> 6122
taskmanager.data.port: 6122 -> 6121
jobmanager.web.port: 8080 -> 8081

This closes #2176

flink-contrib/docker-flink/flink/Dockerfile → flink-contrib/docker-flink/Dockerfile

@@ -16,28 +16,25 @@
 # limitations under the License.
 ################################################################################
 
-FROM base
+FROM java:8-jre-alpine
 
-#add passless key to ssh
-RUN ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
-RUN cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/*
+# Install requirements
+RUN apk add --no-cache bash curl snappy supervisor
 
-##Flink 1.0.2 Installation
-###Download:
-RUN mkdir ~/downloads && cd ~/downloads && \
-    wget -q -O - http://mirror.switch.ch/mirror/apache/dist/flink/flink-1.0.2/flink-1.0.2-bin-hadoop27-scala_2.11.tgz| tar -zxvf - -C /usr/local/
-RUN cd /usr/local && ln -s ./flink-1.0.2 flink
+# Configure supervisor
+ADD supervisor.conf /etc/supervisor/
 
+# Install Flink
+ARG FLINK_VERSION=1.0.3
+ARG HADOOP_VERSION=27
+ARG SCALA_VERSION=2.11
+
+RUN curl -s $(curl -s https://www.apache.org/dyn/closer.cgi\?as_json\=1 | awk '/preferred/ {gsub(/"/,""); print $2}')flink/flink-${FLINK_VERSION}/flink-${FLINK_VERSION}-bin-hadoop${HADOOP_VERSION}-scala_${SCALA_VERSION}.tgz | tar xvz -C /usr/local/
+RUN ln -s /usr/local/flink-$FLINK_VERSION /usr/local/flink
 ENV FLINK_HOME /usr/local/flink
 ENV PATH $PATH:$FLINK_HOME/bin
 
-#config files (template)
-ADD conf/flink-conf.yaml /usr/local/flink/conf/
-
-ADD config-flink.sh /usr/local/flink/bin/
-RUN chmod +x /usr/local/flink/bin/config-flink.sh
-
-EXPOSE 6123
-EXPOSE 22
-
-CMD ["/usr/local/flink/bin/config-flink.sh", "taskmanager"]
+# Configure container
+ADD docker-entrypoint.sh $FLINK_HOME/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["sh", "-c"]

flink-contrib/docker-flink/README.md

-#Apache Flink cluster deployment on Docker using Docker-Compose
+Apache Flink cluster deployment on docker using docker-compose
 
-##Installation
-###Install Docker
+# Installation
 
+Install the most recent stable version of docker
 https://docs.docker.com/installation/
 
-if you have issues with Docker-Compose versions incompatible with your version of Docker try
+Install the most recent stable version of docker-compose
+https://docs.docker.com/compose/install/
 
-`curl -sSL https://get.docker.com/ubuntu/ | sudo sh`
+# Build
 
-###Install Docker-Compose
+Images are based on the official Java Alpine (OpenJDK 8) image and run
+supervisord to stay alive when running containers. If you want to build the
+flink image run:
 
-```
-curl -L https://github.com/docker/compose/releases/download/1.1.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
+    sh build.sh
 
-chmod +x /usr/local/bin/docker-compose
-```
-
-###Get the repo
-
-###Build the images
+or
 
-Images are based on Ubuntu Trusty 14.04 and run Supervisord to stay alive when running containers.
+    docker build -t flink .
 
-The base image installs Oracle Java JDK 1.7 and SSH client & server. You can change the SSH password there or add your own key and adjust SSH config.
+If you want to build the container for a specific version of flink/hadoop/scala
+you can configure it in the respective args:
 
-- Run `./build.sh`
+    docker build --build-arg FLINK_VERSION=1.0.3 --build-arg HADOOP_VERSION=26 --build-arg SCALA_VERSION=2.10 -t "flink:1.0.3-hadoop2.6-scala_2.10" flink
 
-###Deploy
+# Deploy
 
 - Deploy cluster and see config/setup log output (best run in a screen session)
 
-`docker-compose up`
+        docker-compose up
 
 - Deploy as a daemon (and return)
 
-`docker-compose up -d`
+        docker-compose up -d
 
 - Scale the cluster up or down to *N* TaskManagers
 
-`docker-compose scale taskmanager=<N>`
+        docker-compose scale taskmanager=<N>
 
-- Access the JobManager node with SSH (exposed on Port 220)
+- Access the Job Manager container
 
-`ssh root@localhost -p 220`
+        docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) /bin/sh
 
-or on Mac OS X with boot2docker
+- Kill the cluster
 
-`ssh root@$(boot2docker ip) -p 220`
+        docker-compose kill
 
-The password is 'secret'
+- Upload jar to the cluster
 
-- Kill the cluster
+        docker cp <your_jar> $(docker ps --filter name=flink_jobmanager --format={{.ID}}):/<your_path>
 
-`docker-compose kill`
+- Copy file to all the nodes in the cluster
 
-- Upload a jar to the cluster
-
-`scp -P 220 <your_jar> root@localhost:/<your_path>`
+        for i in $(docker ps --filter name=flink --format={{.ID}}); do
+            docker cp <your_file> $i:/<your_path>
+        done
 
 - Run a topology
 
-`ssh -p 220 root@localhost /usr/local/flink/bin/flink run -c <your_class> <your_jar> <your_params>`
+From the jobmanager:
+
+        docker exec -it $(docker ps --filter name=flink_jobmanager --format={{.ID}}) flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
+
+If you have a local flink installation:
+
+        $FLINK_HOME/bin/flink run -m <jobmanager:port> <your_jar>
 
 or
 
-ssh to the job manager and run the topology from there.
+        $FLINK_HOME/bin/flink run -m <jobmanager:port> -c <your_class> <your_jar> <your_params>
 
-###Ports
+### Ports
 
-- The Web Dashboard is on port `48080`
 - The Web Client is on port `48081`
 - JobManager RPC port `6123` (default, not exposed to host)
-- TaskManagers RPC port `6121` (default, not exposed to host)
-- TaskManagers Data port `6122` (default, not exposed to host)
-- JobManager SSH `220`
-- TaskManagers SSH: randomly assigned port, check wih `docker ps`
+- TaskManagers RPC port `6122` (default, not exposed to host)
+- TaskManagers Data port `6121` (default, not exposed to host)
 
 Edit the `docker-compose.yml` file to edit port settings.

flink-contrib/docker-flink/base/Dockerfile

-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-FROM ubuntu:xenial
-
-#requirements
-RUN apt-get update; apt-get install -y curl wget supervisor openssh-server openssh-client nano
-
-#priviledge separation directory
-RUN mkdir /var/run/sshd
-
-#install Java 8 Oracle JDK
-RUN mkdir -p /usr/java/default && \
-    curl -Ls 'http://download.oracle.com/otn-pub/java/jdk/8u91-b14/jdk-8u91-linux-x64.tar.gz' -H 'Cookie: oraclelicense=accept-securebackup-cookie' | \
-    tar --strip-components=1 -xz -C /usr/java/default/
-ENV JAVA_HOME /usr/java/default/
-
-#Install Java Open JDK
-#RUN apt-get install -y unzip openjdk-7-jre-headless
-#ENV JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
-
-
-RUN echo 'root:secret' | chpasswd
-
-#SSH as root... probably needs to be revised for security!
-RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-
-EXPOSE 22
-
-# supervisor base configuration
-ADD supervisor.conf /etc/supervisor/
-
-# default command / not overridable CMD needed for supervisord
-#CMD ["supervisord", "-c", "/etc/supervisor.conf"]
\ No newline at end of file

flink-contrib/docker-flink/build.sh

-#!/usr/bin/env bash
+#!/bin/sh
 
 ################################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
@@ -18,14 +18,4 @@
 # limitations under the License.
 ################################################################################
 
-
-#Kill cluster and remove all containers
-docker-compose kill
-#docker rm $(docker ps -a -q)
-
-#make sure the config file script is executable
-chmod +x flink/config-flink.sh
-
-#rebuild images
-docker build -t="base" base
-docker build -t="flink" flink
+docker build -t "flink" .

flink-contrib/docker-flink/docker-compose.yml

@@ -16,22 +16,22 @@
 # limitations under the License.
 ################################################################################
 
-jobmanager:
+version: "2"
+services:
+  jobmanager:
     image: flink
     ports:
-        - "48080:8080"
-        - "48081:8081"
-        - "220:22"
-    expose:
-        - "6123"
-        - "22"
-    command: /usr/local/flink/bin/config-flink.sh jobmanager
-taskmanager:
+      - "48081:8081"
+    command: jobmanager
+    volumes:
+      - conf:/usr/local/flink/conf
+
+  taskmanager:
     image: flink
-    ports:
-        - "22"
-    expose:
-        - "6121"
-        - "6122"
-    links:
-        - jobmanager:jobmanager
\ No newline at end of file
+    depends_on:
+      - jobmanager
+    command: taskmanager
+    volumes_from:
+      - jobmanager:ro
+volumes:
+  conf:

flink-contrib/docker-flink/flink/conf/flink-conf.yaml → flink-contrib/docker-flink/docker-entrypoint.sh

+#!/bin/sh
+
 ################################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
 #  or more contributor license agreements.  See the NOTICE file
@@ -16,21 +18,20 @@
 # limitations under the License.
 ################################################################################
 
-jobmanager.rpc.address: %jobmanager%
-jobmanager.rpc.port: 6123
-jobmanager.heap.mb: 128
-
-taskmanager.rpc.port: 6121
-taskmanager.data.port: 6122
-taskmanager.heap.mb: 256
-taskmanager.numberOfTaskSlots: %nb_slots%
-
-parallelization.degree.default: %parallelism%
-
-env.java.home: /usr/java/default
+if [ "$1" = "jobmanager" ]; then
+    echo "Starting Job Manager"
+    sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -i`/g" $FLINK_HOME/conf/flink-conf.yaml
+    sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c ^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+    $FLINK_HOME/bin/jobmanager.sh start cluster
+    echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
+    supervisord -c /etc/supervisor/supervisor.conf
 
-#==============================================================================
-# Web Frontend
-#==============================================================================
+elif [ "$1" = "taskmanager" ]; then
+    echo "Starting Task Manager"
+    $FLINK_HOME/bin/taskmanager.sh start
+    echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
+    supervisord -c /etc/supervisor/supervisor.conf
 
-jobmanager.web.port: 8080
+else
+    $@
+fi

flink-contrib/docker-flink/flink/conf/log4j.properties

-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-log4j.rootLogger=INFO, file
-
-# Log all infos in the given file
-log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file.file=${log.file}
-log4j.appender.file.append=false
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file

flink-contrib/docker-flink/flink/conf/logback-yarn.xml

-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="file" class="ch.qos.logback.core.FileAppender">
-        <file>${log.file}</file>
-        <append>false</append>
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%d{HH:mm:ss} %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <logger name="ch.qos.logback" level="WARN" />
-    <root level="INFO">
-        <appender-ref ref="file"/>
-        <appender-ref ref="console"/>
-    </root>
-</configuration>

flink-contrib/docker-flink/flink/conf/logback.xml

-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~ or more contributor license agreements.  See the NOTICE file
-  ~ distributed with this work for additional information
-  ~ regarding copyright ownership.  The ASF licenses this file
-  ~ to you under the Apache License, Version 2.0 (the
-  ~ "License"); you may not use this file except in compliance
-  ~ with the License.  You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration>
-    <appender name="file" class="ch.qos.logback.core.FileAppender">
-        <file>${log.file}</file>
-        <append>false</append>
-        <encoder>
-            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
-        </encoder>
-    </appender>
-
-    <root level="INFO">
-        <appender-ref ref="file"/>
-    </root>
-</configuration>
\ No newline at end of file

flink-contrib/docker-flink/flink/config-flink.sh

-#!/bin/bash
-
-################################################################################
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-
-
-
-CONF=/usr/local/flink/conf
-EXEC=/usr/local/flink/bin
-
-#set nb_slots = nb CPUs
-#let "nbslots=$2 * `nproc`"
-sed -i -e "s/%nb_slots%/`nproc`/g" $CONF/flink-conf.yaml
-
-#set parallelism
-sed -i -e "s/%parallelism%/1/g" $CONF/flink-conf.yaml
-
-if [ "$1" = "jobmanager" ]; then
-    echo "Configuring Job Manager on this node"
-    sed -i -e "s/%jobmanager%/`hostname -i`/g" $CONF/flink-conf.yaml
-    $EXEC/jobmanager.sh start cluster
-
-elif [ "$1" = "taskmanager" ]; then
-    echo "Configuring Task Manager on this node"
-    sed -i -e "s/%jobmanager%/$JOBMANAGER_PORT_6123_TCP_ADDR/g" $CONF/flink-conf.yaml
-    $EXEC/taskmanager.sh start
-fi
-
-#print out config - debug
-echo "config file: " && cat $CONF/flink-conf.yaml
-
-#add ENV variable to shell for ssh login
-echo "export JAVA_HOME=/usr/java/default;" >> ~/.profile
-echo "export PATH=$PATH:$JAVA_HOME/bin;" >> ~/.profile
-echo "export F=/usr/local/flink/;" >> ~/.profile
-#Uncomment for SSH connection between nodes without prompts
-#echo 'export FLINK_SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"' >> ~/.profile
-
-#run ssh server and supervisor to keep container running.
-/usr/sbin/sshd && supervisord -c /etc/supervisor/supervisor.conf

pom.xml

@@ -908,7 +908,6 @@ under the License.
 						<exclude>**/flink-bin/conf/slaves</exclude>
 						<exclude>**/flink-bin/conf/masters</exclude>
 						<exclude>**/flink-bin/conf/zoo.cfg</exclude>
-						<exclude>flink-contrib/docker-flink/flink/conf/slaves</exclude>
 						<!-- Administrative files in the main trunk. -->
 						<exclude>**/README.md</exclude>
 						<exclude>CHANGELOG</exclude>