diff --git a/paddle/scripts/musl_build/Dockerfile b/paddle/scripts/musl_build/Dockerfile index 21ddbc2b0cf6469da06724c7785c0c6a7a74cdfd..120b47b21a761f2f34347c7be18b0e331306c835 100644 --- a/paddle/scripts/musl_build/Dockerfile +++ b/paddle/scripts/musl_build/Dockerfile @@ -14,18 +14,23 @@ FROM python:3.7-alpine3.10 +USER root + WORKDIR /root +VOLUME /root/.ccache + +VOLUME /root/.cache + RUN apk update RUN apk add --no-cache \ g++ gfortran make cmake patchelf git ccache -VOLUME /root/.ccache - ARG package RUN if [ "$package" ]; then \ + set -e; \ pkgs=$(echo "$package" | base64 -d -); \ echo ">>> decode package:"; \ echo "$pkgs"; \ @@ -40,11 +45,12 @@ ARG requirement_ut ARG pip_index RUN if [ "$requirement" ]; then \ + set -e; \ echo "$requirement" | base64 -d - > "requirement.txt"; \ echo ">>> decode requirement:"; \ cat "requirement.txt"; \ echo ">>> install python requirement:"; \ - PIP_ARGS="--timeout 300 --no-cache-dir"; \ + PIP_ARGS="--timeout 300"; \ if [ "$pip_index" ]; then \ PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}'); \ PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN"; \ @@ -54,7 +60,6 @@ RUN if [ "$requirement" ]; then \ rm -f "requirement.txt"; \ if [ "$requirement_ut" ]; then \ echo "$requirement_ut" | base64 -d - > "requirement_ut.txt"; \ - echo ">>> decode requirement_ut:"; \ cat "requirement_ut.txt"; \ pip3 install $PIP_ARGS -r "requirement_ut.txt"; \ rm -f "requirement_ut.txt"; \ diff --git a/paddle/scripts/musl_build/README.md b/paddle/scripts/musl_build/README.md index 830215d2d821ff12bcfcc5aec2fac951158a33c6..d80e9d8b6fcb74e9f321c88a233091909fd0413c 100644 --- a/paddle/scripts/musl_build/README.md +++ b/paddle/scripts/musl_build/README.md @@ -64,10 +64,10 @@ cd ./Paddle ../paddle/scripts/musl_build/build_docker.sh # enter the container interactive shell -BUILD_AUTO=0 ../paddle/scripts/musl_build/build_paddle.sh +BUILD_MAN=1 ../paddle/scripts/musl_build/build_paddle.sh ``` -2. Type commands to compile source manually +2. type commands and compile source manually ```sh # compile paddle by commands # paddle is mount to /paddle directory @@ -78,7 +78,7 @@ mkdir build && cd build pip install -r /paddle/python/requirements.txt # configure project with cmake -cmake -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF /paddle +cmake -DWITH_MUSL=ON -DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF /paddle # run the make to build project. # the argument -j8 is optional to accelerate compiling. @@ -97,18 +97,18 @@ make -j8 - WITH_UT_REQUIREMENT: build with the unit test requirements, default=0. - WITH_PIP_INDEX: use custom pip index when pip install packages. - ONLY_NAME: only print the docker name, and exit. - - HTTP_PROXY: use http proxy - - HTTPS_PROXY: use https proxy + - HTTP_PROXY: use http proxy. + - HTTPS_PROXY: use https proxy. 2. **build_paddle.sh** automatically or manually paddle building script. it will mount the root directory of paddle source to /paddle, and run compile procedure in /root/build directory. the output wheel package will save to the ./output directory relative to working directory. environment variables: - - BUILD_AUTO: build the paddle automatically, save output wheel package to ./output directory, default=1. - - - HTTP_PROXY: use http proxy - - HTTPS_PROXY: use https proxy - + - BUILD_MAN: build the paddle manually, default=0. + - WITH_TEST: build with unitest, and run unitest check, default=0. + - WITH_PRUNE_CONTAINER: remove the container after building, default=0. + - HTTP_PROXY: use http proxy. + - HTTPS_PROXY: use https proxy. # Files - **build_docker.sh**: docker building script diff --git a/paddle/scripts/musl_build/build_docker.sh b/paddle/scripts/musl_build/build_docker.sh index 9527939fc9d14cf4697edce75b560be9dab15693..8f6e5b07bb1236289b68b5fdbaa5b3e245e37e4a 100755 --- a/paddle/scripts/musl_build/build_docker.sh +++ b/paddle/scripts/musl_build/build_docker.sh @@ -21,61 +21,65 @@ CUR_DIR=$(realpath "$CUR_DIR") source "$CUR_DIR/config.sh" # setup configure to default value -WITH_REQUIREMENT="${WITH_REQUIREMENT-1}" +WITH_REQUIREMENT="${WITH_REQUIREMENT-0}" WITH_UT_REQUIREMENT="${WITH_UT_REQUIREMENT-0}" WITH_REBUILD="${WITH_REBUILD-0}" # exit when any command fails set -e -remove_image(){ - echo "clean up docker images: $BUILD_IMAGE" +function remove_image(){ + echo ">>> clean up docker images: $BUILD_IMAGE" docker rmi -f "$BUILD_IMAGE" } -prune_image(){ +function prune_image(){ HOURS="$(expr $1 '*' 24)" FILTER="until=${HOURS}h" - echo "prune old docker images: $FILTER" + + echo ">>> prune old docker images: $FILTER" docker image prune -f -a --filter "$FILTER" } -build_image(){ +function build_image(){ declare -a BUILD_ARGS if [ "$HTTP_PROXY" ]; then BUILD_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY") - echo "using http proxy: $HTTP_PROXY" + echo ">>> using http proxy: $HTTP_PROXY" fi if [ "$HTTPS_PROXY" ]; then BUILD_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY") - echo "using https proxy: $HTTPS_PROXY" + echo ">>> using https proxy: $HTTPS_PROXY" fi - echo "with package requirement: $PACKAGE_REQ" + echo ">>> with package requirement: $PACKAGE_REQ" PACKAGE_B64="$(base64 -w0 $PACKAGE_REQ)" BUILD_ARGS+=("--build-arg" package="$PACKAGE_B64") if [ "$WITH_REQUIREMENT" == "1" ]; then - echo "with python requirement: $PYTHON_REQ" - PYTHON_B64="$(base64 -w0 $PYTHON_REQ)" + FULL_PYTHON_REQ="$PADDLE_DIR/$PYTHON_REQ" + echo ">>> with python requirement: $FULL_PYTHON_REQ" + + PYTHON_B64="$(base64 -w0 $FULL_PYTHON_REQ)" BUILD_ARGS+=("--build-arg" requirement="$PYTHON_B64") fi if [ "$WITH_UT_REQUIREMENT" == "1" ]; then - echo "with unittest requirement: $UNITTEST_REQ" + FULL_UT_REQ="$PADDLE_DIR/$UNITTEST_REQ" + echo ">>> with unittest requirement: $FULL_UT_REQ" + UT_B64="$(base64 -w0 $UNITTEST_REQ)" BUILD_ARGS+=("--build-arg" requirement_ut="$UT_B64") fi if [ "$WITH_PIP_INDEX" ]; then - echo "with pip index: $WITH_PIP_INDEX" + echo ">>> with pip index: $WITH_PIP_INDEX" BUILD_ARGS+=("--build-arg" pip_index="$WITH_PIP_INDEX") fi - echo "build docker image: $BUILD_IMAGE" - + echo ">>> build docker image: $BUILD_IMAGE" # shellcheck disable=2086 docker build \ -t "$BUILD_IMAGE" \ diff --git a/paddle/scripts/musl_build/build_inside.sh b/paddle/scripts/musl_build/build_inside.sh index b7eafae267472b527c8c4d6919ad875d6d1edfad..32a6d5c3f33e3da1e334a37b12282cb0451454e2 100755 --- a/paddle/scripts/musl_build/build_inside.sh +++ b/paddle/scripts/musl_build/build_inside.sh @@ -17,35 +17,55 @@ PADDLE_DIR=/paddle BUILD_DIR=$PWD/build -echo "paddle: $PADDLE_DIR" -echo "python: $PYTHON_VERSION" +echo ">>> paddle: $PADDLE_DIR" +echo ">>> python: $PYTHON_VERSION" # exit when any command fails set -e # setup build dir -echo "setup build dir: $BUILD_DIR" -mkdir -p $BUILD_DIR +echo ">>> setup build dir: $BUILD_DIR" +mkdir -p "$BUILD_DIR" +cd "$BUILD_DIR" + +# setup root dir +chown -R root:root /root if [ "$HTTP_PROXY" ]; then - echo "http_proxy: $HTTP_PROXY" + echo ">>> http_proxy: $HTTP_PROXY" git config --global http.proxy "$HTTP_PROXY" fi if [ "$HTTP_PROXY" ]; then - echo "https_proxy: $HTTPS_PROXY" + echo ">>> https_proxy: $HTTPS_PROXY" git config --global https.proxy "$HTTPS_PROXY" fi +PIP_ARGS="--timeout 300" +if [ "$pip_index" ]; then + PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}') + PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN" + echo ">>> pip index: $pip_index" +fi + +if [ "$WITH_REQUIREMENT" ]; then + echo ">>> install python requirement: $WITH_REQUIREMENT"; + pip install $PIP_ARGS -r "$WITH_REQUIREMENT"; +fi + BUILD_ARG="" if [ "$WITH_TEST" == "1" ]; then - echo "build paddle with testing" + echo ">>> build paddle with testing" BUILD_ARG="-DWITH_TESTING=ON" else BUILD_ARG="-DWITH_TESTING=OFF" fi -echo "configure with cmake" +echo ">>> compile source code" +set -x + +export FLAGS_call_stack_level=2 + cmake "$PADDLE_DIR" \ -DWITH_MUSL=ON \ -DWITH_CRYPTO=OFF \ @@ -53,21 +73,26 @@ cmake "$PADDLE_DIR" \ -DWITH_GPU=OFF \ "$BUILD_ARG" -echo "compile with make: $*" # shellcheck disable=2068 make $@ +set +x OUTPUT_WHL="$(find python/dist/ -type f -name '*.whl'| head -n1)" -echo "paddle wheel: $OUTPUT_WHL" +echo ">>> paddle wheel: $OUTPUT_WHL" -echo "save paddle wheel package to /output" -cp "$OUTPUT_WHL" /output/ +echo ">>> save paddle wheel package to /output" +cp -f "$OUTPUT_WHL" /output/ if [ "$WITH_TEST" == "1" ]; then - echo "install paddle wheel package" - pip3 install --no-cache --force-overwrite "$OUTPUT_WHL" + if [ "$WITH_UT_REQUIREMENT" ]; then + echo ">>> install unittest requirement: $WITH_UT_REQUIREMENT" + pip install $PIP_ARGS -r "$WITH_UT_REQUIREMENT" + fi + + echo ">>> install paddle wheel package" + pip install "$OUTPUT_WHL" - echo "run ctest" + echo ">>> run ctest" ctest --output-on-failure fi diff --git a/paddle/scripts/musl_build/build_paddle.sh b/paddle/scripts/musl_build/build_paddle.sh index 14c3ed17456fc6c5fe68254eae322752cc31e5b1..19d64d91501f2a60d6995d2cd281d9cab4cf6e70 100755 --- a/paddle/scripts/musl_build/build_paddle.sh +++ b/paddle/scripts/musl_build/build_paddle.sh @@ -23,80 +23,99 @@ source "$CUR_DIR/config.sh" # exit when any command fails set -e -# check build mode auto/man -BUILD_AUTO=${BUILD_AUTO:-1} +# setup default arguments +BUILD_MAN="${BUILD_MAN-0}" +WITH_PRUNE_CONTAINER="${WITH_PRUNE_CONTAINER-1}" +WITH_TEST="${WITH_TEST-0}" - -declare -a ENV_ARGS +declare -a RUN_ARGS if [ "$HTTP_PROXY" ]; then - ENV_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY") - echo "using http proxy: $HTTP_PROXY" + RUN_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY") + echo ">>> using http proxy: $HTTP_PROXY" fi if [ "$HTTPS_PROXY" ]; then - ENV_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY") - echo "using https proxy: $HTTPS_PROXY" + RUN_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY") + echo ">>> using https proxy: $HTTPS_PROXY" fi -echo "compile paddle in docker" -echo "docker image: $BUILD_IMAGE" +echo ">>> compile paddle in docker" +echo ">>> docker image: $BUILD_IMAGE" BUILD_ID=$(docker images -q "$BUILD_IMAGE") if [ ! "$BUILD_ID" ]; then - echo "docker image is not existed, and try to build." - - "$CUR_DIR/build_docker.sh" + echo ">>> docker image is not existed, and try to build." + WITH_REQUIREMENT=0 WITH_UT_REQUIREMENT=0 "$CUR_DIR/build_docker.sh" fi -BUILD_NAME="paddle-musl-build-$(date +%Y%m%d-%H%M%S)" -echo "container name: $BUILD_NAME" +echo ">>> container name: $BUILD_CONTAINER" +echo ">>> mount paddle: $PADDLE_DIR => $MOUNT_DIR" -MOUNT_DIR="/paddle" -echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR" - -CCACHE_DIR="${HOME}/.ccache" mkdir -p "$CCACHE_DIR" -echo "ccache dir: $CCACHE_DIR" +echo ">>> ccache dir: $CCACHE_DIR" + +mkdir -p "$CACHE_DIR" +echo ">>> local cache dir: $CACHE_DIR" -if [ "$BUILD_AUTO" -eq "1" ]; then - echo "enter automatic build mode" +RUN_ARGS+=("--env" "WITH_REQUIREMENT=$MOUNT_DIR/$PYTHON_REQ") +echo ">>> install python requirement" - # no exit when fails - set +e + +if [ "$BUILD_MAN" != "1" ]; then + echo ">>> ========================================" + echo ">>> automatic build mode" + echo ">>> ========================================" BUILD_SCRIPT=$MOUNT_DIR/paddle/scripts/musl_build/build_inside.sh - echo "build script: $BUILD_SCRIPT" + echo ">>> build script: $BUILD_SCRIPT" OUTPUT_DIR="output" mkdir -p $OUTPUT_DIR OUTPUT_DIR=$(realpath $OUTPUT_DIR) - echo "build output: $OUTPUT_DIR" + echo ">>> build output: $OUTPUT_DIR" + + if [ "$WITH_TEST" == "1" ]; then + RUN_ARGS+=("--env" "WITH_TEST=1") + echo ">>> run with unit test" + + RUN_ARGS+=("--env" "WITH_UT_REQUIREMENT=$MOUNT_DIR/$UNITTEST_REQ") + echo ">>> install unit test requirement" + fi + + if [ "$WITH_PRUNE_CONTAINER" == "1" ]; then + echo ">>> with prune container" + RUN_ARGS+=("--rm") + fi # shellcheck disable=2086,2068 docker run \ -v "$PADDLE_DIR":"$MOUNT_DIR" \ -v "$OUTPUT_DIR":"/output" \ -v "$CCACHE_DIR":"/root/.ccache" \ - --rm \ + -v "$CACHE_DIR":"/root/.cache" \ --workdir /root \ --network host \ - ${ENV_ARGS[*]} \ - --name "$BUILD_NAME" \ + ${RUN_ARGS[*]} \ + --name "$BUILD_CONTAINER" \ "$BUILD_IMAGE" \ "$BUILD_SCRIPT" $@ - echo "list output: $OUTPUT_DIR" + echo ">>> list output: $OUTPUT_DIR" find "$OUTPUT_DIR" -type f else - echo "enter manual build mode" + echo ">>> ========================================" + echo ">>> manual build mode" + echo ">>> ========================================" # shellcheck disable=2086 docker run \ -it \ -v "$PADDLE_DIR":"$MOUNT_DIR" \ -v "$CCACHE_DIR":"/root/.ccache" \ + -v "$CACHE_DIR":"/root/.cache" \ --workdir /root \ - --network host ${ENV_ARGS[*]}\ - --name "$BUILD_NAME" \ + --network host \ + ${RUN_ARGS[*]} \ + --name "$BUILD_CONTAINER" \ "$BUILD_IMAGE" fi diff --git a/paddle/scripts/musl_build/config.sh b/paddle/scripts/musl_build/config.sh index 69214213e26fe01ade0d0fe4be2d88aa27df99f0..ded239a2a4da7dae4ca7004f7bdf1ea7ca6d3183 100755 --- a/paddle/scripts/musl_build/config.sh +++ b/paddle/scripts/musl_build/config.sh @@ -19,20 +19,23 @@ CUR_DIR=$(realpath "$CUR_DIR") # shellcheck disable=2034 PADDLE_DIR=$(realpath "$CUR_DIR/../../../") +MOUNT_DIR="/paddle" BUILD_DOCKERFILE="$CUR_DIR/Dockerfile" - -PYTHON_REQ="$PADDLE_DIR/python/requirements.txt" -UNITTEST_REQ="$PADDLE_DIR/python/unittest_py/requirements.txt" - PACKAGE_REQ="$CUR_DIR/package.txt" -image_tag(){ - CHKSUM=$(cat "$BUILD_DOCKERFILE" "$PACKAGE_REQ" "$PYTHON_REQ" "$UNITTEST_REQ"| md5sum - | cut -b-8) - echo "$CHKSUM" +PYTHON_REQ="python/requirements.txt" +UNITTEST_REQ="python/unittest_py/requirements.txt" + +function chksum(){ + cat $* | md5sum - | cut -b-8 } # shellcheck disable=2034 -BUILD_TAG="$(image_tag)" BUILD_NAME="paddle-musl-build" +BUILD_TAG=$(chksum "$BUILD_DOCKERFILE" "$PACKAGE_REQ") BUILD_IMAGE="$BUILD_NAME:$BUILD_TAG" +BUILD_CONTAINER="$BUILD_NAME-$(date +%Y%m%d-%H%M%S)" + +CCACHE_DIR="${CCACHE_DIR-${HOME}/.paddle-musl/ccache}" +CACHE_DIR="${CACHE_DIR-${HOME}/.paddle-musl/cache}" diff --git a/paddle/scripts/musl_build/package.txt b/paddle/scripts/musl_build/package.txt index 21843e5f8144801b3cadf564b761c7fa7fd88276..ed6796a0d3cc3e0694ff22f1e63fec47a1db4616 100644 --- a/paddle/scripts/musl_build/package.txt +++ b/paddle/scripts/musl_build/package.txt @@ -1,6 +1,9 @@ -linux-headers=4.19.36-r0 +linux-headers=4.19.36-r0 freetype-dev=2.10.0-r1 libjpeg-turbo-dev=2.0.4-r1 zlib-dev=1.2.11-r1 lapack-dev=3.8.0-r1 openblas-dev=0.3.6-r0 +openssl-dev=1.1.1g-r0 +libuv-dev=1.29.1-r0 +graphviz