From 95d2651bc2dc75b7420d7d9155da60e97f156503 Mon Sep 17 00:00:00 2001
From: Lei Wang <bestwanglei@gmail.com>
Date: Tue, 1 May 2018 10:22:40 -0700
Subject: [PATCH] Build: simplify travis CI script. (#10245)

* Build: simplify travis CI script.

* Add linkcheck and run gen_doc in host machine not in docker.

* Add LinkCheck python package.
---
 .travis.yml                           | 19 +++++++--------
 paddle/scripts/paddle_build.sh        | 13 +++++++---
 paddle/scripts/paddle_docker_build.sh | 35 +++++++++++----------------
 python/requirements.txt               |  1 +
 4 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 929c847bd36..79055651db6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ services:
 os:
   - linux
 env:
-  - JOB=build_doc
+  - JOB=doc
   - JOB=check_style
   - JOB=build_android
 addons:
@@ -36,21 +36,20 @@ addons:
       - ccache
   ssh_known_hosts: 13.229.163.131
 before_install:
-  - if [[ "$JOB" == "check_style" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python
-  # protobuf version.
-  - sudo pip install -r $TRAVIS_BUILD_DIR/python/requirements.txt
-  - sudo pip install wheel sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit LinkChecker
   - |
     function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
 script:
   - |
     # 43min timeout
-    if [[ "$JOB" == "build_android" ]]; then timeout 2580 docker run -it --rm -v "$TRAVIS_BUILD_DIR:/paddle" paddlepaddle/paddle:latest-dev-android;
-    else timeout 2580 paddle/scripts/travis/${JOB}.sh; fi;
-    RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else exit 1; fi;
+    if [[ "$JOB" != "doc" ]]; then
+      timeout 2580 paddle/scripts/paddle_docker_build.sh ${JOB}
+    else
+      timeout 2580 paddle/scritps/paddle_build.sh doc 
+    fi
+    if [ $? -eq 0 ] || [ $? -eq 142 ]; then true; else exit 1; fi;
   - |
-    if [[ "$JOB" != "build_doc" ]]; then exit 0; fi;
+    if [[ "$JOB" != "doc" ]]; then exit 0; fi;
+    # For document only
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi;
     if [[ "$TRAVIS_BRANCH" != "develop"  && ! "$TRAVIS_BRANCH" =~ ^v[[:digit:]]+\.[[:digit:]]+(\.[[:digit:]]+)?(-\S*)?$ ]]; then exit 0; fi;
     export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/master/scripts/deploy/deploy_docs.sh
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 654c8272a18..53455fd8604 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -208,8 +208,8 @@ EOF
           --platform=android-$ANDROID_API \
           --install-dir=$ANDROID_STANDALONE_TOOLCHAIN
     
-    BUILD_ROOT=${PADDLE_ROOT}/build
-    DEST_ROOT={PADDLE_ROOT}/install
+    BUILD_ROOT=${PADDLE_ROOT}/build_android
+    DEST_ROOT=${PADDLE_ROOT}/install_android
     
     mkdir -p $BUILD_ROOT
     cd $BUILD_ROOT
@@ -349,13 +349,18 @@ function gen_docs() {
     ========================================
 EOF
     cmake .. \
+        -DCMAKE_BUILD_TYPE=Release \
         -DWITH_DOC=ON \
         -DWITH_GPU=OFF \
-        -DWITH_AVX=${WITH_AVX:-ON} \
-        -DWITH_SWIG_PY=ON \
+        -DWITH_MKL=OFF \
         -DWITH_STYLE_CHECK=OFF
 
     make -j `nproc` paddle_docs paddle_apis
+
+    # check websites for broken links
+    linkchecker doc/v2/en/html/index.html
+    linkchecker doc/v2/cn/html/index.html
+    linkchecker doc/v2/api/en/html/index.html
 }
 
 function gen_html() {
diff --git a/paddle/scripts/paddle_docker_build.sh b/paddle/scripts/paddle_docker_build.sh
index 252227ef88a..311eb576014 100755
--- a/paddle/scripts/paddle_docker_build.sh
+++ b/paddle/scripts/paddle_docker_build.sh
@@ -28,11 +28,16 @@ function start_build_docker() {
         docker rm -f "${CONTAINER_ID}" 1>/dev/null
     fi
 
+    apt_mirror='s#http://archive.ubuntu.com/ubuntu#mirror://mirrors.ubuntu.com/mirrors.txt#g'
     DOCKER_ENV=$(cat <<EOL
         -e FLAGS_fraction_of_gpu_memory_to_use=0.15 \
         -e CTEST_OUTPUT_ON_FAILURE=1 \
         -e CTEST_PARALLEL_LEVEL=5 \
+        -e APT_MIRROR=${apt_mirror} \
         -e WITH_GPU=ON \
+        -e CUDA_ARCH_NAME=Auto \
+        -e WITH_AVX=ON \
+        -e WITH_GOLANG=OFF \
         -e WITH_TESTING=ON \
         -e WITH_C_API=OFF \
         -e WITH_COVERAGE=ON \
@@ -42,18 +47,23 @@ function start_build_docker() {
         -e PADDLE_FRACTION_GPU_MEMORY_TO_USE=0.15 \
         -e CUDA_VISIBLE_DEVICES=0,1 \
         -e WITH_DISTRIBUTE=ON \
+        -e WITH_FLUID_ONLY=ON \
         -e RUN_TEST=ON
 EOL
     )
+
+    DOCKER_CMD="nvidia-docker"
+    if ! [ -x "$(command -v ${DOCKER_CMD})" ]; then
+        DOCKER_CMD="docker"
+    fi
     set -x
-    nvidia-docker run -it \
-        -d \
+    ${DOCKER_CMD} run -it \
         --name $CONTAINER_ID \
         ${DOCKER_ENV} \
         -v $PADDLE_ROOT:/paddle \
         -w /paddle \
         $IMG \
-        /bin/bash
+        paddle/scripts/paddle_build.sh $@
     set +x
 }
 
@@ -67,24 +77,7 @@ function main() {
         VERSION="latest-dev-android"
     fi
     IMG=${DOCKER_REPO}:${VERSION}
-
-    case $1 in
-      start)
-        start_build_docker
-        ;;
-      build_android)
-        start_build_docker
-        docker exec ${CONTAINER_ID} bash -c "./paddle/scripts/paddle_build.sh $@"
-        ;;
-      *)
-        if container_running "${CONTAINER_ID}"; then
-            docker exec ${CONTAINER_ID} bash -c "./paddle/scripts/paddle_build.sh $@"
-        else
-            echo "Please start container first, with command:"
-            echo "$0 start"
-        fi
-        ;;
-    esac
+    start_build_docker $@
 }
 
 main $@
diff --git a/python/requirements.txt b/python/requirements.txt
index daf3f368b92..ea827e9d5a0 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -8,3 +8,4 @@ scipy>=0.19.0
 Pillow
 nltk>=3.2.2
 graphviz
+LinkChecker
-- 
GitLab