diff --git a/paddle/scripts/musl_build/Dockerfile b/paddle/scripts/musl_build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..649f39b08932ba4b6354af489759311d152985ec --- /dev/null +++ b/paddle/scripts/musl_build/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-alpine3.10 + +WORKDIR /root + +RUN apk update + +RUN apk add --no-cache \ + g++ gfortran make cmake patchelf git \ + linux-headers \ + freetype-dev libjpeg-turbo-dev zlib-dev + +RUN apk add --no-cache --force-overwrite \ + lapack-dev openblas-dev + +ENTRYPOINT [ "/bin/sh" ] diff --git a/paddle/scripts/musl_build/README.md b/paddle/scripts/musl_build/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99aabfbabb79364c2b4b1082d07e1128d726b9a1 --- /dev/null +++ b/paddle/scripts/musl_build/README.md @@ -0,0 +1,90 @@ +Paddle for Linux-musl Usage Guide +=========================================== + +# introduction +Paddle can be built for linux-musl such as alpine, and be used in libos-liked SGX TEE environment. Currently supported commericial product TEE Scone, and community maintanced TEE Occlum. We also working on to support open source TEE Graphene. + + +# build automaticly +1. clone paddle source from github + +```bash +git clone https://github.com/PaddlePaddle/Paddle.git +``` + +2. setup build directory + +```bash +# enter paddle directory +cd ./Paddle + +# create and enter building directory +mkdir -p build && cd build +``` + +3. build docker for compiling. use environment HTTP_PROXY/HTTPS_PROXY for proxy setup. + +```bash +# setup proxy address +export HTTP_PROXY='http://127.0.0.1:8080' +export HTTPS_PROXY='https://127.0.0.1:8080' + +# invoke build script +../paddle/scripts/musl_build/build_docker.sh +``` + +4. compile paddle in previous built docker. proxy setup method is same as previous step. +output wheel package will save to "dist" directory. + +```bash +# setup proxy addresss +export HTTP_PROXY='http://127.0.0.1:8080' +export HTTPS_PROXY='https://127.0.0.1:8080' + +# invoke build paddle script +../paddle/scripts/musl_build/build_paddle.sh + +# find output wheel package +ls dist/*.whl +``` + +# build paddle manually + +1. start up the building docker, and enter the shell in the container +```bash +# checkout paddle source code +git clone https://github.com/PaddlePaddle/Paddle.git + +# entery paddle directory +cd ./Paddle + +# build docker image +../paddle/scripts/musl_build/build_docker.sh + +# enter the container interactive shell +BUILD_AUTO=0 ../paddle/scripts/musl_build/build_paddle.sh +``` + +2. Type commands to compile source manually +```sh +# compile paddle by commands +# paddle is mount to /paddle directory +# working directory is /root +mkdir build && cd build + +# install python requirement +pip install -r /paddle/python/requirements.txt + +# configure project with cmake +cmake /paddle -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF + +# run the make to build project +make +``` + +# files +- build_docker.sh: docker building script +- build_paddle.sh: paddle building script +- build_inside.sh: build_paddle.sh will invoke this script inside the docker for compiling. +- config.sh: build config script for configure compiling option setting. +- Dockerfile: build docker defination file. diff --git a/paddle/scripts/musl_build/build_docker.sh b/paddle/scripts/musl_build/build_docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..7abb1031b52824707ca3c881fa61108ad6b19759 --- /dev/null +++ b/paddle/scripts/musl_build/build_docker.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUR_DIR=$(dirname "${BASH_SOURCE[0]}") +CUR_DIR=$(realpath "$CUR_DIR") + +# shellcheck disable=1090 +source "$CUR_DIR/config.sh" + +# exit when any command fails +set -e + +declare -a ENV_ARGS +if [ "$HTTP_PROXY" ]; then + ENV_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY") + echo "using http proxy: $HTTP_PROXY" +fi + +if [ "$HTTPS_PROXY" ]; then + ENV_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY") + echo "using https proxy: $HTTPS_PROXY" +fi + +echo "clean up docker images: $BUILD_IMAGE" +docker rmi -f "$BUILD_IMAGE" + +echo "build docker image: $BUILD_IMAGE" + +# shellcheck disable=2086 +docker build \ + -t "$BUILD_IMAGE" \ + -f "$CUR_DIR/Dockerfile" \ + --rm=false \ + --network host \ + ${ENV_ARGS[*]} \ + --output type=tar,dest=build.tar \ + . diff --git a/paddle/scripts/musl_build/build_inside.sh b/paddle/scripts/musl_build/build_inside.sh new file mode 100755 index 0000000000000000000000000000000000000000..65407c7d433bada6e64fb195dfeebb957b8edc06 --- /dev/null +++ b/paddle/scripts/musl_build/build_inside.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PADDLE_DIR=/paddle +BUILD_DIR=$PWD + +echo "paddle: $PADDLE_DIR" +echo "python: $PYTHON_VERSION" +echo "http_proxy: $HTTP_PROXY" +echo "https_proxy: $HTTPS_PROXY" + +# exit when any command fails +set -e + +echo "create build dir: $BUILD_DIR" +mkdir -p "$BUILD_DIR" + +if [ "$HTTP_PROXY" ]; then + git config --global http.proxy "$HTTP_PROXY" +fi + +if [ "$HTTP_PROXY" ]; then + git config --global https.proxy "$HTTPS_PROXY" +fi + +PIP_ARGS="" +if [ "$PIP_INDEX" ]; then + PIP_DOMAIN=$(echo "$PIP_INDEX" | awk -F/ '{print $3}') + PIP_ARGS="-i $PIP_INDEX --trusted-host $PIP_DOMAIN" + echo "pip index: $PIP_INDEX" +fi + +PYTHON_REQS=$PADDLE_DIR/python/requirements.txt +echo "install python requirements: $PYTHON_REQS" + +# shellcheck disable=2086 +pip install $PIP_ARGS --timeout 300 --no-cache-dir -r $PYTHON_REQS + +echo "configure with cmake" +cmake "$PADDLE_DIR" \ + -DWITH_MUSL=ON \ + -DWITH_CRYPTO=OFF \ + -DWITH_MKL=OFF \ + -DWITH_GPU=OFF + +echo "compile with make: $*" +# shellcheck disable=2068 +make $@ + +echo "save python dist directory to /output" +cp -r python/dist /output/ diff --git a/paddle/scripts/musl_build/build_paddle.sh b/paddle/scripts/musl_build/build_paddle.sh new file mode 100755 index 0000000000000000000000000000000000000000..ecec9182dc24856692e3a00cd13d40f2fbd79bf8 --- /dev/null +++ b/paddle/scripts/musl_build/build_paddle.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUR_DIR=$(dirname "${BASH_SOURCE[0]}") +CUR_DIR=$(realpath "$CUR_DIR") + +# shellcheck disable=1090 +source "$CUR_DIR/config.sh" + +# exit when any command fails +set -e + +# check build mode auto/man +BUILD_AUTO=${BUILD_AUTO:-1} + + +declare -a ENV_ARGS +if [ "$HTTP_PROXY" ]; then + ENV_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY") + echo "using http proxy: $HTTP_PROXY" +fi + +if [ "$HTTPS_PROXY" ]; then + ENV_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY") + echo "using https proxy: $HTTPS_PROXY" +fi + +if [ "$PIP_INDEX" ]; then + ENV_ARGS+=("--env" "PIP_INDEX=$PIP_INDEX") +fi + +echo "compile paddle in docker" +echo "docker image: $BUILD_IMAGE" + +BUILD_ID=$(docker images -q "$BUILD_IMAGE") +if [ ! "$BUILD_ID" ]; then + echo "docker image is not existed, and try to build." + + "$CUR_DIR/build_docker.sh" +fi + +BUILD_NAME="paddle-musl-build-$(date +%Y%m%d-%H%M%S)" +echo "container name: $BUILD_NAME" + +MOUNT_DIR="/paddle" +echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR" + + +if [ "$BUILD_AUTO" -eq "1" ]; then + echo "enter automatic build mode" + + # no exit when fails + set +e + + BUILD_SCRIPT=$MOUNT_DIR/paddle/scripts/musl_build/build_inside.sh + echo "build script: $BUILD_SCRIPT" + + OUTPUT_DIR="output" + mkdir -p $OUTPUT_DIR + OUTPUT_DIR=$(realpath $OUTPUT_DIR) + echo "build output: $OUTPUT_DIR" + + # shellcheck disable=2086,2068 + docker run \ + -v "$PADDLE_DIR":"$MOUNT_DIR" \ + -v "$OUTPUT_DIR":/output \ + --rm \ + --workdir /root \ + --network host \ + ${ENV_ARGS[*]} \ + --name "$BUILD_NAME" \ + "$BUILD_IMAGE" \ + "$BUILD_SCRIPT" $@ + + echo "list output: $OUTPUT_DIR" + ls "$OUTPUT_DIR" +else + echo "enter manual build mode" + + # shellcheck disable=2086 + docker run \ + -it \ + -v "$PADDLE_DIR":"$MOUNT_DIR" \ + --workdir /root \ + --network host ${ENV_ARGS[*]}\ + --name "$BUILD_NAME" \ + "$BUILD_IMAGE" +fi diff --git a/paddle/scripts/musl_build/config.sh b/paddle/scripts/musl_build/config.sh new file mode 100755 index 0000000000000000000000000000000000000000..d7ec3a8dbb2e1779ebda04fdb48c7804906ead85 --- /dev/null +++ b/paddle/scripts/musl_build/config.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUR_DIR=$(dirname "${BASH_SOURCE[0]}") +CUR_DIR=$(realpath "$CUR_DIR") + +# shellcheck disable=2034 +PADDLE_DIR=$(realpath "$CUR_DIR/../../../") + +# shellcheck disable=2034 +BUILD_IMAGE="paddle-musl-build:2.0"