diff --git a/paddle/scripts/musl_build/Dockerfile b/paddle/scripts/musl_build/Dockerfile index 649f39b08932ba4b6354af489759311d152985ec..21ddbc2b0cf6469da06724c7785c0c6a7a74cdfd 100644 --- a/paddle/scripts/musl_build/Dockerfile +++ b/paddle/scripts/musl_build/Dockerfile @@ -1,3 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + FROM python:3.7-alpine3.10 WORKDIR /root @@ -5,11 +19,47 @@ WORKDIR /root RUN apk update RUN apk add --no-cache \ - g++ gfortran make cmake patchelf git \ - linux-headers \ - freetype-dev libjpeg-turbo-dev zlib-dev + g++ gfortran make cmake patchelf git ccache + +VOLUME /root/.ccache + +ARG package + +RUN if [ "$package" ]; then \ + pkgs=$(echo "$package" | base64 -d -); \ + echo ">>> decode package:"; \ + echo "$pkgs"; \ + for nm in $pkgs; do \ + echo ">>> intall package: $nm"; \ + apk add --no-cache --force-overwrite "$nm"; \ + done; \ + fi + +ARG requirement +ARG requirement_ut +ARG pip_index + +RUN if [ "$requirement" ]; then \ + echo "$requirement" | base64 -d - > "requirement.txt"; \ + echo ">>> decode requirement:"; \ + cat "requirement.txt"; \ + echo ">>> install python requirement:"; \ + PIP_ARGS="--timeout 300 --no-cache-dir"; \ + if [ "$pip_index" ]; then \ + PIP_DOMAIN=$(echo "$pip_index" | awk -F/ '{print $3}'); \ + PIP_ARGS="$PIP_ARGS -i $pip_index --trusted-host $PIP_DOMAIN"; \ + echo ">>> pip index: $pip_index"; \ + fi; \ + pip3 install $PIP_ARGS -r "requirement.txt"; \ + rm -f "requirement.txt"; \ + if [ "$requirement_ut" ]; then \ + echo "$requirement_ut" | base64 -d - > "requirement_ut.txt"; \ + echo ">>> decode requirement_ut:"; \ + cat "requirement_ut.txt"; \ + pip3 install $PIP_ARGS -r "requirement_ut.txt"; \ + rm -f "requirement_ut.txt"; \ + fi; \ + fi -RUN apk add --no-cache --force-overwrite \ - lapack-dev openblas-dev ENTRYPOINT [ "/bin/sh" ] diff --git a/paddle/scripts/musl_build/README.md b/paddle/scripts/musl_build/README.md index 99aabfbabb79364c2b4b1082d07e1128d726b9a1..830215d2d821ff12bcfcc5aec2fac951158a33c6 100644 --- a/paddle/scripts/musl_build/README.md +++ b/paddle/scripts/musl_build/README.md @@ -1,11 +1,11 @@ Paddle for Linux-musl Usage Guide =========================================== -# introduction +# Introduction Paddle can be built for linux-musl such as alpine, and be used in libos-liked SGX TEE environment. Currently supported commericial product TEE Scone, and community maintanced TEE Occlum. We also working on to support open source TEE Graphene. -# build automaticly +# Build Automatically 1. clone paddle source from github ```bash @@ -25,30 +25,32 @@ mkdir -p build && cd build 3. build docker for compiling. use environment HTTP_PROXY/HTTPS_PROXY for proxy setup. ```bash -# setup proxy address -export HTTP_PROXY='http://127.0.0.1:8080' -export HTTPS_PROXY='https://127.0.0.1:8080' +# setup proxy address, when the speed of internet is not good. +# export HTTP_PROXY='http://127.0.0.1:8080' +# export HTTPS_PROXY='https://127.0.0.1:8080' # invoke build script ../paddle/scripts/musl_build/build_docker.sh ``` 4. compile paddle in previous built docker. proxy setup method is same as previous step. -output wheel package will save to "dist" directory. + ```bash -# setup proxy addresss -export HTTP_PROXY='http://127.0.0.1:8080' -export HTTPS_PROXY='https://127.0.0.1:8080' +# setup proxy addresss, when the speed of internet is not good. +# export HTTP_PROXY='http://127.0.0.1:8080' +# export HTTPS_PROXY='https://127.0.0.1:8080' # invoke build paddle script -../paddle/scripts/musl_build/build_paddle.sh +# all arguments, such as -j8 optinal, is past to make procedure. +../paddle/scripts/musl_build/build_paddle.sh -j8 # find output wheel package -ls dist/*.whl +# output wheel packages will save to "./output" directory. +ls ./output/*.whl ``` -# build paddle manually +# Build Manually 1. start up the building docker, and enter the shell in the container ```bash @@ -76,15 +78,43 @@ mkdir build && cd build pip install -r /paddle/python/requirements.txt # configure project with cmake -cmake /paddle -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF +cmake -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF /paddle -# run the make to build project -make +# run the make to build project. +# the argument -j8 is optional to accelerate compiling. +make -j8 ``` -# files -- build_docker.sh: docker building script -- build_paddle.sh: paddle building script -- build_inside.sh: build_paddle.sh will invoke this script inside the docker for compiling. -- config.sh: build config script for configure compiling option setting. -- Dockerfile: build docker defination file. +# Scripts +1. **build_docker.sh** + compiling docker building script. it use alpine linux 3.10 as musl linux build enironment. it will try to install all the compiling tools, development packages, and python requirements for paddle musl compiling. + + environment variables: + + - WITH_PRUNE_DAYS: prune old docker images, with days limitation. + - WITH_REBUILD: force to rebuild the image, default=0. + - WITH_REQUIREMENT: build with the python requirements, default=1. + - WITH_UT_REQUIREMENT: build with the unit test requirements, default=0. + - WITH_PIP_INDEX: use custom pip index when pip install packages. + - ONLY_NAME: only print the docker name, and exit. + - HTTP_PROXY: use http proxy + - HTTPS_PROXY: use https proxy + +2. **build_paddle.sh** automatically or manually paddle building script. it will mount the root directory of paddle source to /paddle, and run compile procedure in /root/build directory. the output wheel package will save to the ./output directory relative to working directory. + + environment variables: + + - BUILD_AUTO: build the paddle automatically, save output wheel package to ./output directory, default=1. + + - HTTP_PROXY: use http proxy + - HTTPS_PROXY: use https proxy + + +# Files +- **build_docker.sh**: docker building script +- **build_paddle.sh**: paddle building script +- **build_inside.sh**: build_paddle.sh will invoke this script inside the docker for compiling. +- **config.sh**: build config script for configure compiling option setting. +- **Dockerfile**: build docker defination file. +- **package.txt**: build required develop packages for alpine linux. +- **REAME.md**: this file. diff --git a/paddle/scripts/musl_build/build_docker.sh b/paddle/scripts/musl_build/build_docker.sh index 7abb1031b52824707ca3c881fa61108ad6b19759..9527939fc9d14cf4697edce75b560be9dab15693 100755 --- a/paddle/scripts/musl_build/build_docker.sh +++ b/paddle/scripts/musl_build/build_docker.sh @@ -20,31 +20,82 @@ CUR_DIR=$(realpath "$CUR_DIR") # shellcheck disable=1090 source "$CUR_DIR/config.sh" +# setup configure to default value +WITH_REQUIREMENT="${WITH_REQUIREMENT-1}" +WITH_UT_REQUIREMENT="${WITH_UT_REQUIREMENT-0}" +WITH_REBUILD="${WITH_REBUILD-0}" + # exit when any command fails set -e -declare -a ENV_ARGS -if [ "$HTTP_PROXY" ]; then - ENV_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY") - echo "using http proxy: $HTTP_PROXY" -fi +remove_image(){ + echo "clean up docker images: $BUILD_IMAGE" + docker rmi -f "$BUILD_IMAGE" +} -if [ "$HTTPS_PROXY" ]; then - ENV_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY") - echo "using https proxy: $HTTPS_PROXY" -fi +prune_image(){ + HOURS="$(expr $1 '*' 24)" + FILTER="until=${HOURS}h" + echo "prune old docker images: $FILTER" + docker image prune -f -a --filter "$FILTER" +} + +build_image(){ + declare -a BUILD_ARGS + + if [ "$HTTP_PROXY" ]; then + BUILD_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY") + echo "using http proxy: $HTTP_PROXY" + fi + + if [ "$HTTPS_PROXY" ]; then + BUILD_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY") + echo "using https proxy: $HTTPS_PROXY" + fi + + echo "with package requirement: $PACKAGE_REQ" + PACKAGE_B64="$(base64 -w0 $PACKAGE_REQ)" + BUILD_ARGS+=("--build-arg" package="$PACKAGE_B64") + + if [ "$WITH_REQUIREMENT" == "1" ]; then + echo "with python requirement: $PYTHON_REQ" + PYTHON_B64="$(base64 -w0 $PYTHON_REQ)" + BUILD_ARGS+=("--build-arg" requirement="$PYTHON_B64") + fi -echo "clean up docker images: $BUILD_IMAGE" -docker rmi -f "$BUILD_IMAGE" + if [ "$WITH_UT_REQUIREMENT" == "1" ]; then + echo "with unittest requirement: $UNITTEST_REQ" + UT_B64="$(base64 -w0 $UNITTEST_REQ)" + BUILD_ARGS+=("--build-arg" requirement_ut="$UT_B64") + fi -echo "build docker image: $BUILD_IMAGE" + if [ "$WITH_PIP_INDEX" ]; then + echo "with pip index: $WITH_PIP_INDEX" + BUILD_ARGS+=("--build-arg" pip_index="$WITH_PIP_INDEX") + fi + + echo "build docker image: $BUILD_IMAGE" -# shellcheck disable=2086 -docker build \ - -t "$BUILD_IMAGE" \ - -f "$CUR_DIR/Dockerfile" \ - --rm=false \ - --network host \ - ${ENV_ARGS[*]} \ - --output type=tar,dest=build.tar \ - . + # shellcheck disable=2086 + docker build \ + -t "$BUILD_IMAGE" \ + -f "$BUILD_DOCKERFILE" \ + --rm=false \ + --network host \ + ${BUILD_ARGS[*]} \ + $PWD +} + +if [ "$WITH_PRUNE_DAYS" ]; then + prune_image "$WITH_PRUNE_DAYS" +fi + +if [ "$WITH_REBUILD" == "1" ]; then + remove_image +fi + +if [ "$ONLY_NAME" == "1" ]; then + echo "$BUILD_IMAGE" +else + build_image +fi diff --git a/paddle/scripts/musl_build/build_inside.sh b/paddle/scripts/musl_build/build_inside.sh index 65407c7d433bada6e64fb195dfeebb957b8edc06..b7eafae267472b527c8c4d6919ad875d6d1edfad 100755 --- a/paddle/scripts/musl_build/build_inside.sh +++ b/paddle/scripts/musl_build/build_inside.sh @@ -15,50 +15,59 @@ # limitations under the License. PADDLE_DIR=/paddle -BUILD_DIR=$PWD +BUILD_DIR=$PWD/build echo "paddle: $PADDLE_DIR" echo "python: $PYTHON_VERSION" -echo "http_proxy: $HTTP_PROXY" -echo "https_proxy: $HTTPS_PROXY" # exit when any command fails set -e -echo "create build dir: $BUILD_DIR" -mkdir -p "$BUILD_DIR" +# setup build dir +echo "setup build dir: $BUILD_DIR" +mkdir -p $BUILD_DIR -if [ "$HTTP_PROXY" ]; then +if [ "$HTTP_PROXY" ]; then + echo "http_proxy: $HTTP_PROXY" git config --global http.proxy "$HTTP_PROXY" fi -if [ "$HTTP_PROXY" ]; then +if [ "$HTTP_PROXY" ]; then + echo "https_proxy: $HTTPS_PROXY" git config --global https.proxy "$HTTPS_PROXY" fi -PIP_ARGS="" -if [ "$PIP_INDEX" ]; then - PIP_DOMAIN=$(echo "$PIP_INDEX" | awk -F/ '{print $3}') - PIP_ARGS="-i $PIP_INDEX --trusted-host $PIP_DOMAIN" - echo "pip index: $PIP_INDEX" +BUILD_ARG="" +if [ "$WITH_TEST" == "1" ]; then + echo "build paddle with testing" + BUILD_ARG="-DWITH_TESTING=ON" +else + BUILD_ARG="-DWITH_TESTING=OFF" fi -PYTHON_REQS=$PADDLE_DIR/python/requirements.txt -echo "install python requirements: $PYTHON_REQS" - -# shellcheck disable=2086 -pip install $PIP_ARGS --timeout 300 --no-cache-dir -r $PYTHON_REQS - echo "configure with cmake" cmake "$PADDLE_DIR" \ -DWITH_MUSL=ON \ -DWITH_CRYPTO=OFF \ -DWITH_MKL=OFF \ - -DWITH_GPU=OFF + -DWITH_GPU=OFF \ + "$BUILD_ARG" echo "compile with make: $*" # shellcheck disable=2068 make $@ -echo "save python dist directory to /output" -cp -r python/dist /output/ +OUTPUT_WHL="$(find python/dist/ -type f -name '*.whl'| head -n1)" +echo "paddle wheel: $OUTPUT_WHL" + +echo "save paddle wheel package to /output" +cp "$OUTPUT_WHL" /output/ + +if [ "$WITH_TEST" == "1" ]; then + + echo "install paddle wheel package" + pip3 install --no-cache --force-overwrite "$OUTPUT_WHL" + + echo "run ctest" + ctest --output-on-failure +fi diff --git a/paddle/scripts/musl_build/build_paddle.sh b/paddle/scripts/musl_build/build_paddle.sh index ecec9182dc24856692e3a00cd13d40f2fbd79bf8..14c3ed17456fc6c5fe68254eae322752cc31e5b1 100755 --- a/paddle/scripts/musl_build/build_paddle.sh +++ b/paddle/scripts/musl_build/build_paddle.sh @@ -38,10 +38,6 @@ if [ "$HTTPS_PROXY" ]; then echo "using https proxy: $HTTPS_PROXY" fi -if [ "$PIP_INDEX" ]; then - ENV_ARGS+=("--env" "PIP_INDEX=$PIP_INDEX") -fi - echo "compile paddle in docker" echo "docker image: $BUILD_IMAGE" @@ -58,6 +54,9 @@ echo "container name: $BUILD_NAME" MOUNT_DIR="/paddle" echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR" +CCACHE_DIR="${HOME}/.ccache" +mkdir -p "$CCACHE_DIR" +echo "ccache dir: $CCACHE_DIR" if [ "$BUILD_AUTO" -eq "1" ]; then echo "enter automatic build mode" @@ -76,7 +75,8 @@ if [ "$BUILD_AUTO" -eq "1" ]; then # shellcheck disable=2086,2068 docker run \ -v "$PADDLE_DIR":"$MOUNT_DIR" \ - -v "$OUTPUT_DIR":/output \ + -v "$OUTPUT_DIR":"/output" \ + -v "$CCACHE_DIR":"/root/.ccache" \ --rm \ --workdir /root \ --network host \ @@ -86,7 +86,7 @@ if [ "$BUILD_AUTO" -eq "1" ]; then "$BUILD_SCRIPT" $@ echo "list output: $OUTPUT_DIR" - ls "$OUTPUT_DIR" + find "$OUTPUT_DIR" -type f else echo "enter manual build mode" @@ -94,6 +94,7 @@ else docker run \ -it \ -v "$PADDLE_DIR":"$MOUNT_DIR" \ + -v "$CCACHE_DIR":"/root/.ccache" \ --workdir /root \ --network host ${ENV_ARGS[*]}\ --name "$BUILD_NAME" \ diff --git a/paddle/scripts/musl_build/config.sh b/paddle/scripts/musl_build/config.sh index d7ec3a8dbb2e1779ebda04fdb48c7804906ead85..69214213e26fe01ade0d0fe4be2d88aa27df99f0 100755 --- a/paddle/scripts/musl_build/config.sh +++ b/paddle/scripts/musl_build/config.sh @@ -20,5 +20,19 @@ CUR_DIR=$(realpath "$CUR_DIR") # shellcheck disable=2034 PADDLE_DIR=$(realpath "$CUR_DIR/../../../") +BUILD_DOCKERFILE="$CUR_DIR/Dockerfile" + +PYTHON_REQ="$PADDLE_DIR/python/requirements.txt" +UNITTEST_REQ="$PADDLE_DIR/python/unittest_py/requirements.txt" + +PACKAGE_REQ="$CUR_DIR/package.txt" + +image_tag(){ + CHKSUM=$(cat "$BUILD_DOCKERFILE" "$PACKAGE_REQ" "$PYTHON_REQ" "$UNITTEST_REQ"| md5sum - | cut -b-8) + echo "$CHKSUM" +} + # shellcheck disable=2034 -BUILD_IMAGE="paddle-musl-build:2.0" +BUILD_TAG="$(image_tag)" +BUILD_NAME="paddle-musl-build" +BUILD_IMAGE="$BUILD_NAME:$BUILD_TAG" diff --git a/paddle/scripts/musl_build/package.txt b/paddle/scripts/musl_build/package.txt new file mode 100644 index 0000000000000000000000000000000000000000..21843e5f8144801b3cadf564b761c7fa7fd88276 --- /dev/null +++ b/paddle/scripts/musl_build/package.txt @@ -0,0 +1,6 @@ +linux-headers=4.19.36-r0 +freetype-dev=2.10.0-r1 +libjpeg-turbo-dev=2.0.4-r1 +zlib-dev=1.2.11-r1 +lapack-dev=3.8.0-r1 +openblas-dev=0.3.6-r0