From 3d0ff8eebcfaad1c4b91f354fa7634b6a3475d33 Mon Sep 17 00:00:00 2001
From: chen zhiyu <quby@sina.com>
Date: Mon, 23 Nov 2020 16:40:51 +0800
Subject: [PATCH] optimize musl docker build script (#28974)

* add musl docker build script

* rm space test=document_fix

* fix some docs and types errors test=document_fix

* move install of python requirement to docker build

* add copyright to docker file.

* add extr opts

* format docs
---
 paddle/scripts/musl_build/Dockerfile      | 60 +++++++++++++--
 paddle/scripts/musl_build/README.md       | 72 +++++++++++++-----
 paddle/scripts/musl_build/build_docker.sh | 93 ++++++++++++++++++-----
 paddle/scripts/musl_build/build_inside.sh | 51 ++++++++-----
 paddle/scripts/musl_build/build_paddle.sh | 13 ++--
 paddle/scripts/musl_build/config.sh       | 16 +++-
 paddle/scripts/musl_build/package.txt     |  6 ++
 7 files changed, 236 insertions(+), 75 deletions(-)
 create mode 100644 paddle/scripts/musl_build/package.txt

diff --git a/paddle/scripts/musl_build/Dockerfile b/paddle/scripts/musl_build/Dockerfile
index 649f39b089..21ddbc2b0c 100644
--- a/paddle/scripts/musl_build/Dockerfile
+++ b/paddle/scripts/musl_build/Dockerfile
@@ -1,3 +1,17 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 FROM python:3.7-alpine3.10
 
 WORKDIR /root
@@ -5,11 +19,47 @@ WORKDIR /root
 RUN apk update
 
 RUN apk add --no-cache \
-    g++  gfortran make cmake patchelf git \
-    linux-headers \
-    freetype-dev libjpeg-turbo-dev zlib-dev
+    g++ gfortran make cmake patchelf git ccache
+
+VOLUME /root/.ccache
+
+ARG package
+
+RUN if [ "$package" ]; then \
+        pkgs=$(echo "$package" | base64 -d -); \
+        echo ">>> decode package:"; \
+        echo "$pkgs"; \
+        for nm in $pkgs; do \
+            echo ">>> intall package: $nm"; \
+            apk add --no-cache --force-overwrite "$nm"; \
+        done; \
+    fi
+
+ARG requirement
+ARG requirement_ut
+ARG pip_index
+
+RUN if [ "$requirement" ]; then \
+        echo "$requirement" | base64 -d - > "requirement.txt"; \
+        echo ">>> decode requirement:"; \
+        cat "requirement.txt"; \
+        echo ">>> install python requirement:"; \
+        PIP_ARGS="--timeout 300 --no-cache-dir"; \
+        if [ "$pip_index" ]; then \
+            PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}'); \
+            PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN"; \
+            echo ">>> pip index: $pip_index"; \
+        fi; \
+        pip3 install $PIP_ARGS -r "requirement.txt"; \
+        rm -f "requirement.txt"; \
+        if [ "$requirement_ut" ]; then \
+            echo "$requirement_ut" | base64 -d - > "requirement_ut.txt"; \
+            echo ">>> decode requirement_ut:"; \
+            cat "requirement_ut.txt"; \
+            pip3 install $PIP_ARGS -r "requirement_ut.txt"; \
+            rm -f "requirement_ut.txt"; \
+        fi; \
+    fi
 
-RUN apk add --no-cache --force-overwrite \
-    lapack-dev openblas-dev 
 
 ENTRYPOINT [ "/bin/sh" ]
diff --git a/paddle/scripts/musl_build/README.md b/paddle/scripts/musl_build/README.md
index 99aabfbabb..830215d2d8 100644
--- a/paddle/scripts/musl_build/README.md
+++ b/paddle/scripts/musl_build/README.md
@@ -1,11 +1,11 @@
 Paddle for Linux-musl Usage Guide
 ===========================================
 
-# introduction
+# Introduction
 Paddle can be built for linux-musl such as alpine, and be used in libos-liked SGX TEE environment. Currently supported commericial product TEE Scone, and community maintanced TEE Occlum. We also working on to support open source TEE Graphene.
 
 
-# build automaticly
+# Build Automatically
 1. clone paddle source from github
    
 ```bash
@@ -25,30 +25,32 @@ mkdir -p build && cd build
 3. build docker for compiling. use environment HTTP_PROXY/HTTPS_PROXY for proxy setup.
 
 ```bash
-# setup proxy address
-export HTTP_PROXY='http://127.0.0.1:8080'
-export HTTPS_PROXY='https://127.0.0.1:8080'
+# setup proxy address, when the speed of internet is not good.
+# export HTTP_PROXY='http://127.0.0.1:8080'
+# export HTTPS_PROXY='https://127.0.0.1:8080'
 
 # invoke build script
 ../paddle/scripts/musl_build/build_docker.sh
 ```
 
 4. compile paddle in previous built docker. proxy setup method is same as previous step.
-output wheel package will save to "dist" directory.
+
 
 ```bash
-# setup proxy addresss
-export HTTP_PROXY='http://127.0.0.1:8080'
-export HTTPS_PROXY='https://127.0.0.1:8080'
+# setup proxy addresss, when the speed of internet is not good.
+# export HTTP_PROXY='http://127.0.0.1:8080'
+# export HTTPS_PROXY='https://127.0.0.1:8080'
 
 # invoke build paddle script
-../paddle/scripts/musl_build/build_paddle.sh
+# all arguments, such as -j8 optinal, is past to make procedure.
+../paddle/scripts/musl_build/build_paddle.sh -j8
 
 # find output wheel package
-ls dist/*.whl
+# output wheel packages will save to "./output" directory.
+ls ./output/*.whl
 ```
 
-# build paddle manually  
+# Build Manually  
 
 1. start up the building docker, and enter the shell in the container
 ```bash
@@ -76,15 +78,43 @@ mkdir build && cd build
 pip install -r /paddle/python/requirements.txt
 
 # configure project with cmake
-cmake /paddle -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF
+cmake -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF /paddle
 
-# run the make to build project
-make
+# run the make to build project.
+# the argument -j8 is optional to accelerate compiling.
+make -j8
 ```
 
-# files
-- build_docker.sh: docker building script
-- build_paddle.sh: paddle building script
-- build_inside.sh: build_paddle.sh will invoke this script inside the docker for compiling.
-- config.sh: build config script for configure compiling option setting.
-- Dockerfile: build docker defination file.
+# Scripts
+1. **build_docker.sh**
+   compiling docker building script. it use alpine linux 3.10 as musl linux build enironment. it will try to install all the compiling tools, development packages, and python requirements for paddle musl compiling.
+    
+    environment variables:
+
+   - WITH_PRUNE_DAYS: prune old docker images, with days limitation.
+   - WITH_REBUILD: force to rebuild the image, default=0.
+   - WITH_REQUIREMENT: build with the python requirements, default=1.
+   - WITH_UT_REQUIREMENT: build with the unit test requirements, default=0.
+   - WITH_PIP_INDEX: use custom pip index when pip install packages.
+   - ONLY_NAME: only print the docker name, and exit.
+   - HTTP_PROXY: use http proxy
+   - HTTPS_PROXY: use https proxy
+
+2. **build_paddle.sh** automatically or manually paddle building script. it will mount the root directory of paddle source to /paddle, and run compile procedure in /root/build directory. the output wheel package will save to the ./output directory relative to working directory.
+    
+    environment variables:
+
+    - BUILD_AUTO: build the paddle automatically, save output wheel package to ./output directory, default=1.
+    
+    - HTTP_PROXY: use http proxy
+    - HTTPS_PROXY: use https proxy
+
+
+# Files
+- **build_docker.sh**: docker building script
+- **build_paddle.sh**: paddle building script
+- **build_inside.sh**: build_paddle.sh will invoke this script inside the docker for compiling.
+- **config.sh**: build config script for configure compiling option setting.
+- **Dockerfile**: build docker defination file.
+- **package.txt**: build required develop packages for alpine linux.
+- **REAME.md**: this file.
diff --git a/paddle/scripts/musl_build/build_docker.sh b/paddle/scripts/musl_build/build_docker.sh
index 7abb1031b5..9527939fc9 100755
--- a/paddle/scripts/musl_build/build_docker.sh
+++ b/paddle/scripts/musl_build/build_docker.sh
@@ -20,31 +20,82 @@ CUR_DIR=$(realpath "$CUR_DIR")
 # shellcheck disable=1090
 source "$CUR_DIR/config.sh"
 
+# setup configure to default value
+WITH_REQUIREMENT="${WITH_REQUIREMENT-1}"
+WITH_UT_REQUIREMENT="${WITH_UT_REQUIREMENT-0}"
+WITH_REBUILD="${WITH_REBUILD-0}"
+
 # exit when any command fails
 set -e
 
-declare -a ENV_ARGS
-if [ "$HTTP_PROXY" ]; then
-    ENV_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY")
-    echo "using http proxy: $HTTP_PROXY"
-fi
+remove_image(){
+    echo "clean up docker images: $BUILD_IMAGE"
+    docker rmi -f "$BUILD_IMAGE"
+}
 
-if [ "$HTTPS_PROXY" ]; then
-    ENV_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY")
-    echo "using https proxy: $HTTPS_PROXY"
-fi
+prune_image(){
+    HOURS="$(expr $1 '*' 24)"
+    FILTER="until=${HOURS}h"
+    echo "prune old docker images: $FILTER"
+    docker image prune -f -a --filter "$FILTER"
+}
+
+build_image(){
+    declare -a BUILD_ARGS
+    
+    if [ "$HTTP_PROXY" ]; then
+        BUILD_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY")
+        echo "using http proxy: $HTTP_PROXY"
+    fi
+
+    if [ "$HTTPS_PROXY" ]; then
+        BUILD_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY")
+        echo "using https proxy: $HTTPS_PROXY"
+    fi
+
+    echo "with package requirement: $PACKAGE_REQ"
+    PACKAGE_B64="$(base64 -w0 $PACKAGE_REQ)"
+    BUILD_ARGS+=("--build-arg" package="$PACKAGE_B64")
+
+    if [ "$WITH_REQUIREMENT" == "1" ]; then
+        echo "with python requirement: $PYTHON_REQ"
+        PYTHON_B64="$(base64 -w0 $PYTHON_REQ)"
+        BUILD_ARGS+=("--build-arg" requirement="$PYTHON_B64")
+    fi
 
-echo "clean up docker images: $BUILD_IMAGE"
-docker rmi -f "$BUILD_IMAGE"
+    if [ "$WITH_UT_REQUIREMENT" == "1" ]; then
+        echo "with unittest requirement: $UNITTEST_REQ"
+        UT_B64="$(base64 -w0 $UNITTEST_REQ)"
+        BUILD_ARGS+=("--build-arg" requirement_ut="$UT_B64")
+    fi
 
-echo "build docker image: $BUILD_IMAGE"
+    if [ "$WITH_PIP_INDEX" ]; then
+        echo "with pip index: $WITH_PIP_INDEX"
+        BUILD_ARGS+=("--build-arg" pip_index="$WITH_PIP_INDEX")
+    fi
+        
+    echo "build docker image: $BUILD_IMAGE"
 
-# shellcheck disable=2086
-docker build \
-    -t "$BUILD_IMAGE" \
-    -f "$CUR_DIR/Dockerfile" \
-    --rm=false \
-    --network host \
-    ${ENV_ARGS[*]} \
-    --output type=tar,dest=build.tar \
-    .
+    # shellcheck disable=2086
+    docker build \
+        -t "$BUILD_IMAGE" \
+        -f "$BUILD_DOCKERFILE" \
+        --rm=false \
+        --network host \
+        ${BUILD_ARGS[*]} \
+        $PWD
+}
+
+if [ "$WITH_PRUNE_DAYS" ]; then
+    prune_image "$WITH_PRUNE_DAYS"
+fi
+
+if [ "$WITH_REBUILD" == "1" ]; then
+    remove_image
+fi
+
+if [ "$ONLY_NAME" == "1" ]; then
+    echo "$BUILD_IMAGE"
+else
+    build_image
+fi
diff --git a/paddle/scripts/musl_build/build_inside.sh b/paddle/scripts/musl_build/build_inside.sh
index 65407c7d43..b7eafae267 100755
--- a/paddle/scripts/musl_build/build_inside.sh
+++ b/paddle/scripts/musl_build/build_inside.sh
@@ -15,50 +15,59 @@
 # limitations under the License.
 
 PADDLE_DIR=/paddle
-BUILD_DIR=$PWD
+BUILD_DIR=$PWD/build
 
 echo "paddle: $PADDLE_DIR"
 echo "python: $PYTHON_VERSION"
-echo "http_proxy: $HTTP_PROXY"
-echo "https_proxy: $HTTPS_PROXY"
 
 # exit when any command fails
 set -e
 
-echo "create build dir: $BUILD_DIR"
-mkdir -p "$BUILD_DIR"
+# setup build dir
+echo "setup build dir: $BUILD_DIR"
+mkdir -p $BUILD_DIR
 
-if [ "$HTTP_PROXY" ]; then
+if [ "$HTTP_PROXY" ]; then 
+    echo "http_proxy: $HTTP_PROXY" 
     git config --global http.proxy "$HTTP_PROXY"
 fi
 
-if [ "$HTTP_PROXY" ]; then
+if [ "$HTTP_PROXY" ]; then 
+    echo "https_proxy: $HTTPS_PROXY" 
     git config --global https.proxy "$HTTPS_PROXY"
 fi
 
-PIP_ARGS=""
-if [ "$PIP_INDEX" ]; then
-    PIP_DOMAIN=$(echo "$PIP_INDEX" | awk -F/ '{print $3}')
-    PIP_ARGS="-i $PIP_INDEX --trusted-host $PIP_DOMAIN"
-    echo "pip index: $PIP_INDEX"
+BUILD_ARG=""
+if [ "$WITH_TEST" == "1" ]; then
+    echo "build paddle with testing"
+    BUILD_ARG="-DWITH_TESTING=ON"
+else
+    BUILD_ARG="-DWITH_TESTING=OFF"
 fi
 
-PYTHON_REQS=$PADDLE_DIR/python/requirements.txt
-echo "install python requirements: $PYTHON_REQS"
-
-# shellcheck disable=2086
-pip install $PIP_ARGS --timeout 300 --no-cache-dir -r $PYTHON_REQS
-
 echo "configure with cmake"
 cmake "$PADDLE_DIR" \
     -DWITH_MUSL=ON \
     -DWITH_CRYPTO=OFF \
     -DWITH_MKL=OFF \
-    -DWITH_GPU=OFF
+    -DWITH_GPU=OFF \
+    "$BUILD_ARG"
 
 echo "compile with make: $*"
 # shellcheck disable=2068
 make $@
 
-echo "save python dist directory to /output"
-cp -r python/dist /output/
+OUTPUT_WHL="$(find python/dist/ -type f -name '*.whl'| head -n1)"
+echo "paddle wheel: $OUTPUT_WHL"
+
+echo "save paddle wheel package to /output"
+cp  "$OUTPUT_WHL" /output/
+
+if [ "$WITH_TEST" == "1" ]; then
+
+    echo "install paddle wheel package"
+    pip3 install --no-cache --force-overwrite "$OUTPUT_WHL"
+
+    echo "run ctest"
+    ctest --output-on-failure
+fi
diff --git a/paddle/scripts/musl_build/build_paddle.sh b/paddle/scripts/musl_build/build_paddle.sh
index ecec9182dc..14c3ed1745 100755
--- a/paddle/scripts/musl_build/build_paddle.sh
+++ b/paddle/scripts/musl_build/build_paddle.sh
@@ -38,10 +38,6 @@ if [ "$HTTPS_PROXY" ]; then
     echo "using https proxy: $HTTPS_PROXY"
 fi
 
-if [ "$PIP_INDEX" ]; then
-    ENV_ARGS+=("--env" "PIP_INDEX=$PIP_INDEX")
-fi
-
 echo "compile paddle in docker"
 echo "docker image: $BUILD_IMAGE"
 
@@ -58,6 +54,9 @@ echo "container name: $BUILD_NAME"
 MOUNT_DIR="/paddle"
 echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR"
 
+CCACHE_DIR="${HOME}/.ccache"
+mkdir -p "$CCACHE_DIR"
+echo "ccache dir: $CCACHE_DIR"
 
 if [ "$BUILD_AUTO" -eq "1" ]; then
     echo "enter automatic build mode"
@@ -76,7 +75,8 @@ if [ "$BUILD_AUTO" -eq "1" ]; then
     # shellcheck disable=2086,2068
     docker run \
         -v "$PADDLE_DIR":"$MOUNT_DIR" \
-        -v "$OUTPUT_DIR":/output \
+        -v "$OUTPUT_DIR":"/output" \
+        -v "$CCACHE_DIR":"/root/.ccache" \
         --rm \
         --workdir /root \
         --network host \
@@ -86,7 +86,7 @@ if [ "$BUILD_AUTO" -eq "1" ]; then
         "$BUILD_SCRIPT" $@
 
     echo "list output: $OUTPUT_DIR"
-    ls "$OUTPUT_DIR"
+    find "$OUTPUT_DIR" -type f
 else
     echo "enter manual build mode"
 
@@ -94,6 +94,7 @@ else
     docker run \
         -it \
         -v "$PADDLE_DIR":"$MOUNT_DIR" \
+        -v "$CCACHE_DIR":"/root/.ccache" \
         --workdir /root \
         --network host ${ENV_ARGS[*]}\
         --name "$BUILD_NAME" \
diff --git a/paddle/scripts/musl_build/config.sh b/paddle/scripts/musl_build/config.sh
index d7ec3a8dbb..69214213e2 100755
--- a/paddle/scripts/musl_build/config.sh
+++ b/paddle/scripts/musl_build/config.sh
@@ -20,5 +20,19 @@ CUR_DIR=$(realpath "$CUR_DIR")
 # shellcheck disable=2034
 PADDLE_DIR=$(realpath "$CUR_DIR/../../../")
 
+BUILD_DOCKERFILE="$CUR_DIR/Dockerfile"
+
+PYTHON_REQ="$PADDLE_DIR/python/requirements.txt"
+UNITTEST_REQ="$PADDLE_DIR/python/unittest_py/requirements.txt"
+
+PACKAGE_REQ="$CUR_DIR/package.txt"
+
+image_tag(){
+    CHKSUM=$(cat "$BUILD_DOCKERFILE" "$PACKAGE_REQ" "$PYTHON_REQ" "$UNITTEST_REQ"| md5sum - | cut -b-8)
+    echo "$CHKSUM"
+}
+
 # shellcheck disable=2034
-BUILD_IMAGE="paddle-musl-build:2.0"
+BUILD_TAG="$(image_tag)"
+BUILD_NAME="paddle-musl-build"
+BUILD_IMAGE="$BUILD_NAME:$BUILD_TAG"
diff --git a/paddle/scripts/musl_build/package.txt b/paddle/scripts/musl_build/package.txt
new file mode 100644
index 0000000000..21843e5f81
--- /dev/null
+++ b/paddle/scripts/musl_build/package.txt
@@ -0,0 +1,6 @@
+linux-headers=4.19.36-r0 
+freetype-dev=2.10.0-r1
+libjpeg-turbo-dev=2.0.4-r1
+zlib-dev=1.2.11-r1
+lapack-dev=3.8.0-r1
+openblas-dev=0.3.6-r0
-- 
GitLab