Add unittest in musl build (#29099)

* add musl docker build script

* rm space test=document_fix

* fix some docs and types errors test=document_fix

* move install of python requirement to docker build

* add copyright to docker file.

* add extr opts

* format docs

* add ut test add pip cache

* add more args description in readme

* add stack backtrace in ctest

* fix readme bugs

paddle/scripts/musl_build/Dockerfile

@@ -14,18 +14,23 @@
 
 FROM python:3.7-alpine3.10
 
+USER root
+
 WORKDIR /root
 
+VOLUME /root/.ccache
+
+VOLUME /root/.cache
+
 RUN apk update
 
 RUN apk add --no-cache \
     g++ gfortran make cmake patchelf git ccache
 
-VOLUME /root/.ccache
-
 ARG package
 
 RUN if [ "$package" ]; then \
+        set -e; \
         pkgs=$(echo "$package" | base64 -d -); \
         echo ">>> decode package:"; \
         echo "$pkgs"; \
@@ -40,11 +45,12 @@ ARG requirement_ut
 ARG pip_index
 
 RUN if [ "$requirement" ]; then \
+        set -e; \
         echo "$requirement" | base64 -d - > "requirement.txt"; \
         echo ">>> decode requirement:"; \
         cat "requirement.txt"; \
         echo ">>> install python requirement:"; \
-        PIP_ARGS="--timeout 300 --no-cache-dir"; \
+        PIP_ARGS="--timeout 300"; \
         if [ "$pip_index" ]; then \
             PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}'); \
             PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN"; \
@@ -54,7 +60,6 @@ RUN if [ "$requirement" ]; then \
         rm -f "requirement.txt"; \
         if [ "$requirement_ut" ]; then \
             echo "$requirement_ut" | base64 -d - > "requirement_ut.txt"; \
-            echo ">>> decode requirement_ut:"; \
             cat "requirement_ut.txt"; \
             pip3 install $PIP_ARGS -r "requirement_ut.txt"; \
             rm -f "requirement_ut.txt"; \

paddle/scripts/musl_build/README.md

@@ -64,10 +64,10 @@ cd ./Paddle
 ../paddle/scripts/musl_build/build_docker.sh
 
 # enter the container interactive shell
-BUILD_AUTO=0 ../paddle/scripts/musl_build/build_paddle.sh
+BUILD_MAN=1 ../paddle/scripts/musl_build/build_paddle.sh
 ```
 
-2. Type commands to compile source manually
+2. type commands and compile source manually
 ```sh
 # compile paddle by commands
 # paddle is mount to /paddle directory
@@ -78,7 +78,7 @@ mkdir build && cd build
 pip install -r /paddle/python/requirements.txt
 
 # configure project with cmake
-cmake -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF /paddle
+cmake -DWITH_MUSL=ON -DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF /paddle
 
 # run the make to build project.
 # the argument -j8 is optional to accelerate compiling.
@@ -97,18 +97,18 @@ make -j8
    - WITH_UT_REQUIREMENT: build with the unit test requirements, default=0.
    - WITH_PIP_INDEX: use custom pip index when pip install packages.
    - ONLY_NAME: only print the docker name, and exit.
-   - HTTP_PROXY: use http proxy
-   - HTTPS_PROXY: use https proxy
+   - HTTP_PROXY: use http proxy.
+   - HTTPS_PROXY: use https proxy.
 
 2. **build_paddle.sh** automatically or manually paddle building script. it will mount the root directory of paddle source to /paddle, and run compile procedure in /root/build directory. the output wheel package will save to the ./output directory relative to working directory.
     
     environment variables:
 
-    - BUILD_AUTO: build the paddle automatically, save output wheel package to ./output directory, default=1.
-    
-    - HTTP_PROXY: use http proxy
-    - HTTPS_PROXY: use https proxy
-
+    - BUILD_MAN: build the paddle manually, default=0.
+    - WITH_TEST: build with unitest, and run unitest check, default=0.
+    - WITH_PRUNE_CONTAINER: remove the container after building, default=0.
+    - HTTP_PROXY: use http proxy.
+    - HTTPS_PROXY: use https proxy.
 
 # Files
 - **build_docker.sh**: docker building script

paddle/scripts/musl_build/build_docker.sh

@@ -21,61 +21,65 @@ CUR_DIR=$(realpath "$CUR_DIR")
 source "$CUR_DIR/config.sh"
 
 # setup configure to default value
-WITH_REQUIREMENT="${WITH_REQUIREMENT-1}"
+WITH_REQUIREMENT="${WITH_REQUIREMENT-0}"
 WITH_UT_REQUIREMENT="${WITH_UT_REQUIREMENT-0}"
 WITH_REBUILD="${WITH_REBUILD-0}"
 
 # exit when any command fails
 set -e
 
-remove_image(){
-    echo "clean up docker images: $BUILD_IMAGE"
+function remove_image(){
+    echo ">>> clean up docker images: $BUILD_IMAGE"
     docker rmi -f "$BUILD_IMAGE"
 }
 
-prune_image(){
+function prune_image(){
     HOURS="$(expr $1 '*' 24)"
     FILTER="until=${HOURS}h"
-    echo "prune old docker images: $FILTER"
+
+    echo ">>> prune old docker images: $FILTER"
     docker image prune -f -a --filter "$FILTER"
 }
 
-build_image(){
+function build_image(){
     declare -a BUILD_ARGS
     
     if [ "$HTTP_PROXY" ]; then
         BUILD_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY")
-        echo "using http proxy: $HTTP_PROXY"
+        echo ">>> using http proxy: $HTTP_PROXY"
     fi
 
     if [ "$HTTPS_PROXY" ]; then
         BUILD_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY")
-        echo "using https proxy: $HTTPS_PROXY"
+        echo ">>> using https proxy: $HTTPS_PROXY"
     fi
 
-    echo "with package requirement: $PACKAGE_REQ"
+    echo ">>> with package requirement: $PACKAGE_REQ"
     PACKAGE_B64="$(base64 -w0 $PACKAGE_REQ)"
     BUILD_ARGS+=("--build-arg" package="$PACKAGE_B64")
 
     if [ "$WITH_REQUIREMENT" == "1" ]; then
-        echo "with python requirement: $PYTHON_REQ"
-        PYTHON_B64="$(base64 -w0 $PYTHON_REQ)"
+        FULL_PYTHON_REQ="$PADDLE_DIR/$PYTHON_REQ"
+        echo ">>> with python requirement: $FULL_PYTHON_REQ"
+
+        PYTHON_B64="$(base64 -w0 $FULL_PYTHON_REQ)"
         BUILD_ARGS+=("--build-arg" requirement="$PYTHON_B64")
     fi
 
     if [ "$WITH_UT_REQUIREMENT" == "1" ]; then
-        echo "with unittest requirement: $UNITTEST_REQ"
+        FULL_UT_REQ="$PADDLE_DIR/$UNITTEST_REQ"
+        echo ">>> with unittest requirement: $FULL_UT_REQ"
+
         UT_B64="$(base64 -w0 $UNITTEST_REQ)"
         BUILD_ARGS+=("--build-arg" requirement_ut="$UT_B64")
     fi
 
     if [ "$WITH_PIP_INDEX" ]; then
-        echo "with pip index: $WITH_PIP_INDEX"
+        echo ">>> with pip index: $WITH_PIP_INDEX"
         BUILD_ARGS+=("--build-arg" pip_index="$WITH_PIP_INDEX")
     fi
         
-    echo "build docker image: $BUILD_IMAGE"
-
+    echo ">>> build docker image: $BUILD_IMAGE"
     # shellcheck disable=2086
     docker build \
         -t "$BUILD_IMAGE" \

paddle/scripts/musl_build/build_inside.sh

@@ -17,35 +17,55 @@
 PADDLE_DIR=/paddle
 BUILD_DIR=$PWD/build
 
-echo "paddle: $PADDLE_DIR"
-echo "python: $PYTHON_VERSION"
+echo ">>> paddle: $PADDLE_DIR"
+echo ">>> python: $PYTHON_VERSION"
 
 # exit when any command fails
 set -e
 
 # setup build dir
-echo "setup build dir: $BUILD_DIR"
-mkdir -p $BUILD_DIR
+echo ">>> setup build dir: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
+
+# setup root dir
+chown -R root:root /root
 
 if [ "$HTTP_PROXY" ]; then 
-    echo "http_proxy: $HTTP_PROXY" 
+    echo ">>> http_proxy: $HTTP_PROXY" 
     git config --global http.proxy "$HTTP_PROXY"
 fi
 
 if [ "$HTTP_PROXY" ]; then 
-    echo "https_proxy: $HTTPS_PROXY" 
+    echo ">>> https_proxy: $HTTPS_PROXY" 
     git config --global https.proxy "$HTTPS_PROXY"
 fi
 
+PIP_ARGS="--timeout 300"
+if [ "$pip_index" ]; then
+    PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}')
+    PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN"
+    echo ">>> pip index: $pip_index"
+fi
+
+if [ "$WITH_REQUIREMENT" ]; then
+    echo ">>> install python requirement: $WITH_REQUIREMENT";
+    pip install $PIP_ARGS -r "$WITH_REQUIREMENT";
+fi
+
 BUILD_ARG=""
 if [ "$WITH_TEST" == "1" ]; then
-    echo "build paddle with testing"
+    echo ">>> build paddle with testing"
     BUILD_ARG="-DWITH_TESTING=ON"
 else
     BUILD_ARG="-DWITH_TESTING=OFF"
 fi
 
-echo "configure with cmake"
+echo ">>> compile source code"
+set -x
+
+export FLAGS_call_stack_level=2
+
 cmake "$PADDLE_DIR" \
     -DWITH_MUSL=ON \
     -DWITH_CRYPTO=OFF \
@@ -53,21 +73,26 @@ cmake "$PADDLE_DIR" \
     -DWITH_GPU=OFF \
     "$BUILD_ARG"
 
-echo "compile with make: $*"
 # shellcheck disable=2068
 make $@
+set +x
 
 OUTPUT_WHL="$(find python/dist/ -type f -name '*.whl'| head -n1)"
-echo "paddle wheel: $OUTPUT_WHL"
+echo ">>> paddle wheel: $OUTPUT_WHL"
 
-echo "save paddle wheel package to /output"
-cp  "$OUTPUT_WHL" /output/
+echo ">>> save paddle wheel package to /output"
+cp -f "$OUTPUT_WHL" /output/
 
 if [ "$WITH_TEST" == "1" ]; then
 
-    echo "install paddle wheel package"
-    pip3 install --no-cache --force-overwrite "$OUTPUT_WHL"
+    if [ "$WITH_UT_REQUIREMENT" ]; then
+        echo ">>> install unittest requirement: $WITH_UT_REQUIREMENT"
+        pip install $PIP_ARGS -r "$WITH_UT_REQUIREMENT"
+    fi
+
+    echo ">>> install paddle wheel package"
+    pip install "$OUTPUT_WHL"
 
-    echo "run ctest"
+    echo ">>> run ctest"
     ctest --output-on-failure
 fi

paddle/scripts/musl_build/build_paddle.sh

@@ -23,80 +23,99 @@ source "$CUR_DIR/config.sh"
 # exit when any command fails
 set -e
 
-# check build mode auto/man
-BUILD_AUTO=${BUILD_AUTO:-1}
+# setup default arguments
+BUILD_MAN="${BUILD_MAN-0}"
+WITH_PRUNE_CONTAINER="${WITH_PRUNE_CONTAINER-1}"
+WITH_TEST="${WITH_TEST-0}"
 
-
-declare -a ENV_ARGS
+declare -a RUN_ARGS
 if [ "$HTTP_PROXY" ]; then
-    ENV_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY")
-    echo "using http proxy: $HTTP_PROXY"
+    RUN_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY")
+    echo ">>> using http proxy: $HTTP_PROXY"
 fi
 
 if [ "$HTTPS_PROXY" ]; then
-    ENV_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY")
-    echo "using https proxy: $HTTPS_PROXY"
+    RUN_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY")
+    echo ">>> using https proxy: $HTTPS_PROXY"
 fi
 
-echo "compile paddle in docker"
-echo "docker image: $BUILD_IMAGE"
+echo ">>> compile paddle in docker"
+echo ">>> docker image: $BUILD_IMAGE"
 
 BUILD_ID=$(docker images -q "$BUILD_IMAGE")
 if [ ! "$BUILD_ID" ]; then
-    echo "docker image is not existed, and try to build."
-
-    "$CUR_DIR/build_docker.sh"
+    echo ">>> docker image is not existed, and try to build."
+    WITH_REQUIREMENT=0 WITH_UT_REQUIREMENT=0 "$CUR_DIR/build_docker.sh"
 fi
 
-BUILD_NAME="paddle-musl-build-$(date +%Y%m%d-%H%M%S)"
-echo "container name: $BUILD_NAME"
-
-MOUNT_DIR="/paddle"
-echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR"
+echo ">>> container name: $BUILD_CONTAINER"
+echo ">>> mount paddle: $PADDLE_DIR => $MOUNT_DIR"
 
-CCACHE_DIR="${HOME}/.ccache"
 mkdir -p "$CCACHE_DIR"
-echo "ccache dir: $CCACHE_DIR"
+echo ">>> ccache dir: $CCACHE_DIR"
+
+mkdir -p "$CACHE_DIR"
+echo ">>> local cache dir: $CACHE_DIR"
+
+RUN_ARGS+=("--env" "WITH_REQUIREMENT=$MOUNT_DIR/$PYTHON_REQ")
+echo ">>> install python requirement"
 
-if [ "$BUILD_AUTO" -eq "1" ]; then
-    echo "enter automatic build mode"
 
-    # no exit when fails
-    set +e
+if [ "$BUILD_MAN" != "1" ]; then
+    echo ">>> ========================================"
+    echo ">>> automatic build mode"
+    echo ">>> ========================================"
 
     BUILD_SCRIPT=$MOUNT_DIR/paddle/scripts/musl_build/build_inside.sh
-    echo "build script: $BUILD_SCRIPT"
+    echo ">>> build script: $BUILD_SCRIPT"
 
     OUTPUT_DIR="output"
     mkdir -p $OUTPUT_DIR
     OUTPUT_DIR=$(realpath $OUTPUT_DIR)
-    echo "build output: $OUTPUT_DIR"
+    echo ">>> build output: $OUTPUT_DIR"
+
+    if [ "$WITH_TEST" == "1" ]; then
+        RUN_ARGS+=("--env" "WITH_TEST=1")
+        echo ">>> run with unit test"
+
+        RUN_ARGS+=("--env" "WITH_UT_REQUIREMENT=$MOUNT_DIR/$UNITTEST_REQ")
+        echo ">>> install unit test requirement"
+    fi
+  
+    if [ "$WITH_PRUNE_CONTAINER" == "1" ]; then
+        echo ">>> with prune container"
+        RUN_ARGS+=("--rm")
+    fi
 
     # shellcheck disable=2086,2068
     docker run \
         -v "$PADDLE_DIR":"$MOUNT_DIR" \
         -v "$OUTPUT_DIR":"/output" \
         -v "$CCACHE_DIR":"/root/.ccache" \
-        --rm \
+        -v "$CACHE_DIR":"/root/.cache" \
         --workdir /root \
         --network host \
-        ${ENV_ARGS[*]} \
-        --name "$BUILD_NAME" \
+        ${RUN_ARGS[*]} \
+        --name "$BUILD_CONTAINER" \
         "$BUILD_IMAGE" \
         "$BUILD_SCRIPT" $@
 
-    echo "list output: $OUTPUT_DIR"
+    echo ">>> list output: $OUTPUT_DIR"
     find "$OUTPUT_DIR" -type f
 else
-    echo "enter manual build mode"
+    echo ">>> ========================================"
+    echo ">>> manual build mode"
+    echo ">>> ========================================"
 
     # shellcheck disable=2086
     docker run \
         -it \
         -v "$PADDLE_DIR":"$MOUNT_DIR" \
         -v "$CCACHE_DIR":"/root/.ccache" \
+        -v "$CACHE_DIR":"/root/.cache" \
         --workdir /root \
-        --network host ${ENV_ARGS[*]}\
-        --name "$BUILD_NAME" \
+        --network host \
+        ${RUN_ARGS[*]} \
+        --name "$BUILD_CONTAINER" \
         "$BUILD_IMAGE"
 fi

paddle/scripts/musl_build/config.sh

@@ -19,20 +19,23 @@ CUR_DIR=$(realpath "$CUR_DIR")
 
 # shellcheck disable=2034
 PADDLE_DIR=$(realpath "$CUR_DIR/../../../")
+MOUNT_DIR="/paddle"
 
 BUILD_DOCKERFILE="$CUR_DIR/Dockerfile"
-
-PYTHON_REQ="$PADDLE_DIR/python/requirements.txt"
-UNITTEST_REQ="$PADDLE_DIR/python/unittest_py/requirements.txt"
-
 PACKAGE_REQ="$CUR_DIR/package.txt"
 
-image_tag(){
-    CHKSUM=$(cat "$BUILD_DOCKERFILE" "$PACKAGE_REQ" "$PYTHON_REQ" "$UNITTEST_REQ"| md5sum - | cut -b-8)
-    echo "$CHKSUM"
+PYTHON_REQ="python/requirements.txt"
+UNITTEST_REQ="python/unittest_py/requirements.txt"
+
+function chksum(){
+    cat $* | md5sum - | cut -b-8
 }
 
 # shellcheck disable=2034
-BUILD_TAG="$(image_tag)"
 BUILD_NAME="paddle-musl-build"
+BUILD_TAG=$(chksum "$BUILD_DOCKERFILE" "$PACKAGE_REQ")
 BUILD_IMAGE="$BUILD_NAME:$BUILD_TAG"
+BUILD_CONTAINER="$BUILD_NAME-$(date +%Y%m%d-%H%M%S)"
+
+CCACHE_DIR="${CCACHE_DIR-${HOME}/.paddle-musl/ccache}"
+CACHE_DIR="${CACHE_DIR-${HOME}/.paddle-musl/cache}"

paddle/scripts/musl_build/package.txt

@@ -4,3 +4,6 @@ libjpeg-turbo-dev=2.0.4-r1
 zlib-dev=1.2.11-r1
 lapack-dev=3.8.0-r1
 openblas-dev=0.3.6-r0
+openssl-dev=1.1.1g-r0
+libuv-dev=1.29.1-r0
+graphviz