Docker: Fixed dockerfile.

docker/build/README.md

@@ -8,4 +8,3 @@ Simply run
 ```
 
 Please notice that deep refactoring is ongoing, which may cause building errors now and then.
-

docker/build/build.sh → docker/build/build_dev.sh

@@ -17,7 +17,7 @@
 ###############################################################################
 
 # Usage:
-#   ./build.sh ./dev.x86_64.dockerfile
+#   ./build_dev.sh ./dev.x86_64.dockerfile
 DOCKERFILE=$1
 
 

docker/build/dev.x86_64.dockerfile

@@ -6,6 +6,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 COPY installers /tmp/installers
 RUN bash /tmp/installers/pre_install.sh
 RUN bash /tmp/installers/install_bazel.sh
+RUN bash /tmp/installers/install_bazel_packages.sh
+RUN bash /tmp/installers/install_gflags_glog.sh
 RUN bash /tmp/installers/install_glew.sh
 RUN bash /tmp/installers/install_glusterfs.sh
 RUN bash /tmp/installers/install_gpu_caffe.sh
@@ -18,13 +20,16 @@ RUN bash /tmp/installers/install_protobuf.sh
 RUN bash /tmp/installers/install_python_modules.sh
 RUN bash /tmp/installers/install_qp_oases.sh
 RUN bash /tmp/installers/install_ros.sh
+RUN bash /tmp/installers/install_snowboy.sh
 RUN bash /tmp/installers/install_supervisor.sh
 RUN bash /tmp/installers/install_undistort.sh
 RUN bash /tmp/installers/install_yarn.sh
+RUN bash /tmp/installers/post_install.sh
 
 RUN apt-get install -y \
    bc \
    cppcheck \
+   cuda \
    debconf-utils \
    doxygen \
    graphviz \
@@ -38,6 +43,8 @@ RUN apt-get install -y \
    libcurl4-openssl-dev \
    libfreetype6-dev \
    liblapack-dev \
+   libpcap-dev \
+   locate \
    lsof \
    realpath \
    shellcheck \
@@ -45,5 +52,3 @@ RUN apt-get install -y \
    v4l-utils \
    nfs-common \
    zip
-
-RUN bash /tmp/installers/post_install.sh

docker/build/installers/README.md

-# Installers to Build Docker Image
-
-The installers should be standalone, which means it can finish successfully with
-minimum assumption of the base image.
-
-## Minimum assumption
-
-The only thing you can assume is that we have run the `pre_install.sh`. All
-other dependencies should be installed by your own installer.
-
-## Test
-
-Run the following command to test if an installer works well during image
-building process.
-
-```bash
-# Test single or multiple installers.
-./test_installer.sh <install_xxx.sh> [install_yyy.sh ...]
-
-# Test all installers.
-./test_installer.sh install_*.sh
-```

docker/build/installers/install_bazel.sh

@@ -32,3 +32,6 @@ apt-get install -y oracle-java8-installer
 cd "$(dirname "${BASH_SOURCE[0]}")"
 wget https://github.com/bazelbuild/bazel/releases/download/0.5.3/bazel-0.5.3-installer-linux-x86_64.sh
 bash bazel-0.5.3-installer-linux-x86_64.sh
+
+# Clean up.
+rm -fr bazel-0.5.3-installer-linux-x86_64.sh /etc/apt/sources.list.d/bazel.list

docker/build/installers/test_installer.sh → docker/build/installers/install_bazel_packages.sh

@@ -16,10 +16,8 @@
 # limitations under the License.
 ###############################################################################
 
-cd "$(dirname "${BASH_SOURCE[0]}")"
+# Fail on first error.
+set -e
 
-for installer in "$@"; do
-  echo "Start testing ${installer}"
-  docker build --build-arg INSTALLER=${installer} -f test_installer.dockerfile . \
-      2>&1 | tee ${installer}.log
-done
+mkdir -p /home/tmp
+wget -O /home/tmp/opencv-2.4.13.2.zip https://github.com/opencv/opencv/archive/2.4.13.2.zip

docker/build/installers/test_installer.dockerfile → docker/build/installers/install_gflags_glog.sh

+#!/usr/bin/env bash
+
 ###############################################################################
 # Copyright 2018 The Apollo Authors. All Rights Reserved.
 #
@@ -14,16 +16,30 @@
 # limitations under the License.
 ###############################################################################
 
-FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu14.04
+# Fail on first error.
+set -e
 
-ENV DEBIAN_FRONTEND=noninteractive
+cd "$(dirname "${BASH_SOURCE[0]}")"
 
-# Basic tools.
-COPY pre_install.sh /tmp/pre_install.sh
-RUN bash /tmp/pre_install.sh
+# Install gflags.
+wget https://github.com/gflags/gflags/archive/v2.2.0.tar.gz
+tar xzf v2.2.0.tar.gz
+mkdir gflags-2.2.0/build
+pushd gflags-2.2.0/build
+CXXFLAGS="-fPIC" cmake ..
+make -j8
+make install
+popd
 
-# Set it with "--build-arg INSTALLER=<your installer>" when building.
-ARG INSTALLER=not_exist
+# Install glog which also depends on gflags.
+wget https://github.com/google/glog/archive/v0.3.5.tar.gz
+tar xzf v0.3.5.tar.gz
+pushd glog-0.3.5
+./configure
+make -j8 CXXFLAGS='-Wno-sign-compare -Wno-unused-local-typedefs -D_START_GOOGLE_NAMESPACE_="namespace google {" -D_END_GOOGLE_NAMESPACE_="}" -DGOOGLE_NAMESPACE="google" -DHAVE_PTHREAD -DHAVE_SYS_UTSNAME_H -DHAVE_SYS_SYSCALL_H -DHAVE_SYS_TIME_H -DHAVE_STDINT_H -DHAVE_STRING_H -DHAVE_PREAD -DHAVE_FCNTL -DHAVE_SYS_TYPES_H -DHAVE_SYSLOG_H -DHAVE_LIB_GFLAGS -DHAVE_UNISTD_H'
+make install
+popd
 
-COPY ${INSTALLER} /tmp/installer.sh
-RUN bash /tmp/installer.sh
+# Clean up.
+rm -fr /usr/local/lib/libglog.so*
+rm -fr v2.2.0.tar.gz gflags-2.2.0 v0.3.5.tar.gz glog-0.3.5

docker/build/installers/install_glew.sh

@@ -32,8 +32,13 @@ apt-get install -y --force-yes libglfw3 libglfw3-dev freeglut3-dev
 # Install GLEW
 wget https://github.com/nigels-com/glew/releases/download/glew-2.0.0/glew-2.0.0.zip
 unzip glew-2.0.0.zip
-cd glew-2.0.0
-make && make install
+pushd glew-2.0.0
+make -j8
+make install
+popd
 
 ln -s /usr/lib64/libGLEW.so /usr/lib/libGLEW.so
 ln -s /usr/lib64/libGLEW.so.2.0 /usr/lib/libGLEW.so.2.0
+
+# Clean up.
+rm -fr glew-2.0.0.zip glew-2.0.0 /etc/apt/sources.list.d/fillwave_ext.list

docker/build/installers/install_gpu_caffe.sh

@@ -40,4 +40,6 @@ wget https://apollocache.blob.core.windows.net/apollo-docker/caffe_x86.tar.gz
 tar xzf caffe_x86.tar.gz
 mv caffe_x86/output-GPU/include/caffe /usr/include/
 mv caffe_x86/output-GPU/lib/* /usr/lib/x86_64-linux-gnu/
-ldconfig
+
+# Clean up.
+rm -fr caffe_x86.tar.gz caffe_x86

docker/build/installers/install_ipopt.sh

@@ -25,7 +25,18 @@ apt-get install -y libblas-dev liblapack-dev gfortran
 
 wget https://www.coin-or.org/download/source/Ipopt/Ipopt-3.12.8.zip -O Ipopt-3.12.8.zip
 unzip Ipopt-3.12.8.zip
-cd Ipopt-3.12.8/ThirdParty/Mumps && bash get.Mumps
-cd ../..
-./configure --build=x86_64 && make all && make install && mkdir -p /usr/local/ipopt
+
+pushd Ipopt-3.12.8/ThirdParty/Mumps
+bash get.Mumps
+popd
+
+pushd Ipopt-3.12.8
+./configure --build=x86_64
+make -j8 all
+make install
+mkdir -p /usr/local/ipopt
 cp -r include /usr/local/ipopt/ && cp -r lib /usr/local/ipopt/
+popd
+
+# Clean up.
+rm -fr Ipopt-3.12.8.zip Ipopt-3.12.8

docker/build/installers/install_nlopt.sh

@@ -23,9 +23,13 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 wget http://ab-initio.mit.edu/nlopt/nlopt-2.4.2.tar.gz
 tar xzf nlopt-2.4.2.tar.gz
-cd nlopt-2.4.2
-./configure --enable-shared && make && make install
+pushd nlopt-2.4.2
+./configure --enable-shared
+make -j8
+make install
+popd
 
-# Remove unwanted libraries.
+# Clean up.
+rm -fr nlopt-2.4.2.tar.gz nlopt-2.4.2
 rm /usr/local/lib/libnlopt.a
 rm /usr/local/lib/libnlopt.la

docker/build/installers/install_node.sh

@@ -23,6 +23,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 wget https://github.com/tj/n/archive/v2.1.0.tar.gz
 tar xzf v2.1.0.tar.gz
-cd n-2.1.0
+pushd n-2.1.0
 make install
 n 8.0.0
+popd
+
+# Clean up.
+rm -fr v2.1.0.tar.gz n-2.1.0

docker/build/installers/install_ota.sh

@@ -23,5 +23,9 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 wget https://apollocache.blob.core.windows.net/apollo-docker/ota_security.tar.gz
 tar xzf ota_security.tar.gz
-cd ota_security
+pushd ota_security
 bash ota_server_deploy.sh root
+popd
+
+# Clean up.
+rm -fr ota_security.tar.gz ota_security

docker/build/installers/install_pcl.sh

@@ -27,4 +27,6 @@ mkdir -p /usr/local/include/pcl-1.7
 mv pcl-1.7_x86/include/pcl /usr/local/include/pcl-1.7/
 mv pcl-1.7_x86/lib/* /usr/local/lib/
 mv pcl-1.7_x86/share/pcl-1.7 /usr/local/share/
-ldconfig
+
+# Clean up.
+rm -fr pcl-1.7_x86.tar.gz pcl-1.7_x86

docker/build/installers/install_protobuf.sh

@@ -23,5 +23,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 wget https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz
 tar xzf protobuf-cpp-3.3.0.tar.gz
-cd protobuf-3.3.0
-./configure --prefix=/usr && make && make install
+
+pushd protobuf-3.3.0
+./configure --prefix=/usr
+make -j8
+make install
+popd
+
+# Clean up.
+rm -fr protobuf-cpp-3.3.0.tar.gz protobuf-3.3.0

docker/build/installers/install_qp_oases.sh

@@ -23,12 +23,15 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 
 wget https://github.com/startcode/qp-oases/archive/v3.2.1-1.tar.gz
 tar xzf v3.2.1-1.tar.gz
-cd qp-oases-3.2.1-1
 
+pushd qp-oases-3.2.1-1
 mkdir bin
-make CPPFLAGS="-Wall -pedantic -Wshadow -Wfloat-equal -O3 -Wconversion \
-               -Wsign-conversion -fPIC -DLINUX -DSOLVER_NONE -D__NO_COPYRIGHT__"
-
+make -j8 CPPFLAGS="-Wall -pedantic -Wshadow -Wfloat-equal -O3 -Wconversion \
+                   -Wsign-conversion -fPIC -DLINUX -DSOLVER_NONE \
+                   -D__NO_COPYRIGHT__"
 cp bin/libqpOASES.so /usr/local/lib
 cp -r include/* /usr/local/include
-ldconfig
+popd
+
+# Clean up.
+rm -fr v3.2.1-1.tar.gz qp-oases-3.2.1-1

docker/build/installers/install_ros.sh

@@ -49,8 +49,10 @@ echo 'ROSCONSOLE_FORMAT=${file}:${line} ${function}() [${severity}] [${time}]: $
 VERSION=2.1.1
 FILENAME=ros-indigo-apollo-${VERSION}-x86_64.tar.gz
 
-mkdir /home/tmp
+mkdir -p /home/tmp
 cd /home/tmp
 wget https://github.com/ApolloAuto/apollo-platform/releases/download/${VERSION}/${FILENAME}
 tar xzf ${FILENAME}
-rm ${FILENAME}
+
+# Clean up.
+rm -fr ${FILENAME} /etc/apt/sources.list.d/ros-latest.list

docker/build/installers/install_snowboy.sh

@@ -31,3 +31,6 @@ wget https://apollocache.blob.core.windows.net/apollo-docker/snowboy.tar.gz
 tar xzf snowboy.tar.gz
 mkdir -p /usr/local/apollo
 mv snowboy /usr/local/apollo/
+
+# Clean up.
+rm -fr snowboy.tar.gz

docker/build/installers/install_undistort.sh

@@ -25,3 +25,6 @@ wget https://apollocache.blob.core.windows.net/apollo-docker/undistort.tar.gz
 tar xzf undistort.tar.gz
 mkdir -p /usr/local/apollo
 mv undistort /usr/local/apollo/
+
+# Clean up.
+rm -fr undistort.tar.gz

docker/build/installers/install_yarn.sh

@@ -23,3 +23,6 @@ curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
 apt-get update -y
 apt-get install -y yarn
+
+# Clean up.
+rm -fr /etc/apt/sources.list.d/yarn.list

docker/build/installers/post_install.sh

@@ -25,8 +25,3 @@ ln -rs /usr/lib/libtcmalloc_and_profiler.so.4 /usr/lib/libtcmalloc_and_profiler.
 # https://stackoverflow.com/questions/25193161/chfn-pam-system-error-intermittently-in-docker-hub-builds
 ln -s -f /bin/true /usr/bin/chfn
 
-# Clean up.
-apt-get clean autoclean
-apt-get autoremove -y
-rm -fr /var/lib/apt/lists/*
-rm -rf /tmp/*

docker/scripts/dev_start_nvidia_docker.sh

-#!/usr/bin/env bash
-
-###############################################################################
-# Copyright 2018 The Apollo Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
-
-
-if [[ -z $(which nvidia-docker) ]]; then
-  echo "We have switched to nvidia-docker as our basement to leverage the work"
-  echo "of supporting graphics drivers, CUDA, CUDNN, etc. Please install it by"
-  echo "following the official guide:"
-  echo ""
-  echo "   https://github.com/NVIDIA/nvidia-docker/blob/master/README.md"
-  exit 1
-fi
-
-INCHINA="no"
-LOCAL_IMAGE="no"
-VERSION=""
-ARCH=$(uname -m)
-VERSION_X86_64="dev-x86_64-20180405_1744"
-VERSION_AARCH64="dev-aarch64-20170927_1111"
-VERSION_OPT=""
-
-function show_usage()
-{
-cat <<EOF
-Usage: $(basename $0) [options] ...
-OPTIONS:
-    -C                     Pull docker image from China mirror.
-    -h, --help             Display this help and exit.
-    -t, --tag <version>    Specify which version of a docker image to pull.
-    -l, --local            Use local docker image.
-EOF
-exit 0
-}
-
-APOLLO_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
-
-if [ ! -e /apollo ]; then
-    sudo ln -sf ${APOLLO_ROOT_DIR} /apollo
-fi
-
-echo "/apollo/data/core/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null
-
-source ${APOLLO_ROOT_DIR}/scripts/apollo_base.sh
-
-VOLUME_VERSION="latest"
-DEFAULT_MAPS=(
-  sunnyvale_big_loop
-  sunnyvale_loop
-)
-MAP_VOLUME_CONF=""
-
-while [ $# -gt 0 ]
-do
-    case "$1" in
-    -C|--docker-cn-mirror)
-        INCHINA="yes"
-        ;;
-    -image)
-        echo -e "\033[093mWarning\033[0m: This option has been replaced by \"-t\" and \"--tag\", please use the new one.\n"
-        show_usage
-        ;;
-    -t|--tag)
-        VAR=$1
-        [ -z $VERSION_OPT ] || echo -e "\033[093mWarning\033[0m: mixed option $VAR with $VERSION_OPT, only the last one will take effect.\n"
-        shift
-        VERSION_OPT=$1
-        [ -z ${VERSION_OPT// /} ] && echo -e "Missing parameter for $VAR" && exit 2
-        [[ $VERSION_OPT =~ ^-.* ]] && echo -e "Missing parameter for $VAR" && exit 2
-        ;;
-    dev-*) # keep backward compatibility, should be removed from further version.
-        [ -z $VERSION_OPT ] || echo -e "\033[093mWarning\033[0m: mixed option $1 with -t/--tag, only the last one will take effect.\n"
-        VERSION_OPT=$1
-        echo -e "\033[93mWarning\033[0m: You are using an old style command line option which may be removed from"
-        echo -e "further versoin, please use -t <version> instead.\n"
-        ;;
-    -h|--help)
-        show_usage
-        ;;
-    -l|--local)
-        LOCAL_IMAGE="yes"
-        ;;
-    --map)
-        map_name=$2
-        shift
-        source ${APOLLO_ROOT_DIR}/docker/scripts/restart_map_volume.sh \
-            "${map_name}" "${VOLUME_VERSION}"
-    ;;
-    *)
-        echo -e "\033[93mWarning\033[0m: Unknown option: $1"
-        exit 2
-        ;;
-    esac
-    shift
-done
-
-if [ ! -z "$VERSION_OPT" ]; then
-    VERSION=$VERSION_OPT
-elif [ ${ARCH} == "x86_64" ]; then
-    VERSION=${VERSION_X86_64}
-elif [ ${ARCH} == "aarch64" ]; then
-    VERSION=${VERSION_AARCH64}
-else
-    echo "Unknown architecture: ${ARCH}"
-    exit 0
-fi
-
-if [ -z "${DOCKER_REPO}" ]; then
-    DOCKER_REPO=apolloauto/apollo
-fi
-
-if [ "$INCHINA" == "yes" ]; then
-    DOCKER_REPO=registry.docker-cn.com/apolloauto/apollo
-fi
-
-if [ "$LOCAL_IMAGE" == "yes" ] && [ -z "$VERSION_OPT" ]; then
-    VERSION="local_dev"
-fi
-
-# Included default maps.
-for map_name in ${DEFAULT_MAPS[@]}; do
-    source ${APOLLO_ROOT_DIR}/docker/scripts/restart_map_volume.sh ${map_name} "${VOLUME_VERSION}"
-done
-
-IMG=${DOCKER_REPO}:$VERSION
-
-function main(){
-
-    if [ "$LOCAL_IMAGE" = "yes" ];then
-        info "Start docker container based on local image : $IMG"
-    else
-        info "Start pulling docker image $IMG ..."
-        docker pull $IMG
-        if [ $? -ne 0 ];then
-            error "Failed to pull docker image."
-            exit 1
-        fi
-    fi
-
-    docker ps -a --format "{{.Names}}" | grep 'apollo_dev' 1>/dev/null
-    if [ $? == 0 ]; then
-        docker stop apollo_dev 1>/dev/null
-        docker rm -f apollo_dev 1>/dev/null
-    fi
-    local display=""
-    if [[ -z ${DISPLAY} ]];then
-        display=":0"
-    else
-        display="${DISPLAY}"
-    fi
-
-    setup_device
-
-    local devices=" -v /dev:/dev"
-
-    USER_ID=$(id -u)
-    GRP=$(id -g -n)
-    GRP_ID=$(id -g)
-    LOCAL_HOST=`hostname`
-    DOCKER_HOME="/home/$USER"
-    if [ "$USER" == "root" ];then
-        DOCKER_HOME="/root"
-    fi
-    if [ ! -d "$HOME/.cache" ];then
-        mkdir "$HOME/.cache"
-    fi
-
-    LOCALIZATION_VOLUME=apollo_localization_volume
-    docker stop ${LOCALIZATION_VOLUME} > /dev/null 2>&1
-
-    LOCALIZATION_VOLUME_IMAGE=${DOCKER_REPO}:localization_volume-${ARCH}-latest
-    docker pull ${LOCALIZATION_VOLUME_IMAGE}
-    docker run -it -d --rm --name ${LOCALIZATION_VOLUME} ${LOCALIZATION_VOLUME_IMAGE}
-
-    YOLO3D_VOLUME=apollo_yolo3d_volume
-    docker stop ${YOLO3D_VOLUME} > /dev/null 2>&1
-
-    YOLO3D_VOLUME_IMAGE=${DOCKER_REPO}:yolo3d_volume-${ARCH}-latest
-    docker pull ${YOLO3D_VOLUME_IMAGE}
-    docker run -it -d --rm --name ${YOLO3D_VOLUME} ${YOLO3D_VOLUME_IMAGE}
-
-    info "Starting docker container \"apollo_dev\" ..."
-    nvidia-docker run -it \
-        -d \
-        --privileged \
-        --name apollo_dev \
-        ${MAP_VOLUME_CONF} \
-        --volumes-from ${LOCALIZATION_VOLUME} \
-        --volumes-from ${YOLO3D_VOLUME} \
-        -e DISPLAY=$display \
-        -e DOCKER_USER=$USER \
-        -e USER=$USER \
-        -e DOCKER_USER_ID=$USER_ID \
-        -e DOCKER_GRP=$GRP \
-        -e DOCKER_GRP_ID=$GRP_ID \
-        -e DOCKER_IMG=$IMG \
-        -v /tmp/.X11-unix:/tmp/.X11-unix:rw \
-        -v $APOLLO_ROOT_DIR:/apollo \
-        -v /media:/media \
-        -v $HOME/.cache:${DOCKER_HOME}/.cache \
-        -v /etc/localtime:/etc/localtime:ro \
-        -v /usr/src:/usr/src \
-        -v /lib/modules:/lib/modules \
-        --net host \
-        -w /apollo \
-        ${devices} \
-        --add-host in_dev_docker:127.0.0.1 \
-        --add-host ${LOCAL_HOST}:127.0.0.1 \
-        --hostname in_dev_docker \
-        --shm-size 2G \
-        $IMG \
-        /bin/bash
-
-    if [ $? -ne 0 ];then
-        error "Failed to start docker container \"apollo_dev\" based on image: $IMG"
-        exit 1
-    fi
-
-    if [ "${USER}" != "root" ]; then
-        docker exec apollo_dev bash -c '/apollo/scripts/docker_adduser.sh'
-    fi
-
-    ok "Finished setting up Apollo docker environment. Now you can enter with: \nbash docker/scripts/dev_into.sh"
-    ok "Enjoy!"
-}
-
-main