From 041c4bca832ef342679b17783c67f5d7294b1f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=20Wei=20=28=E4=BB=BB=E5=8D=AB=29?= Date: Wed, 9 Mar 2022 10:29:37 +0800 Subject: [PATCH] build documents if public apis modified, meanwhile their samplecodes should be tested (#39728) * run document_preview when samplecodes be tested * run document_preview when samplecodes be tested * sphinx-build symbol link; and build-doc default * FLUIDDOCDIR typo * download the required configirations and some other scripts * install required python packages. * clone specified branch of docs repo, and if failed, clone the default branch * clean workspace for docs repo * use the conf.py imported by https://github.com/PaddlePaddle/docs/pull/4222/ * download and install the boscmd * Optimaze the code comments. * specify the pypi index server * only do doc-build when running in cpu mode * pull docs pr git log paddle_pr_info * install jq * force using sphinx-build under py3.7 * using our new domain name for preview * install python package error * don't build doc default --- tools/document_preview.sh | 170 ++++++++++++++++++++++++++++++++++---- tools/sampcd_processor.py | 45 ++++++++++ 2 files changed, 198 insertions(+), 17 deletions(-) diff --git a/tools/document_preview.sh b/tools/document_preview.sh index 83c758d0aa8..424169bbc51 100755 --- a/tools/document_preview.sh +++ b/tools/document_preview.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,19 +14,155 @@ # See the License for the specific language governing permissions and # limitations under the License. -PADDLE_ROOT=/home -mkdir ${PADDLE_ROOT} -cd ${PADDLE_ROOT} -pip install /paddle/build/opt/paddle/share/wheels/*.whl -git clone https://github.com/PaddlePaddle/FluidDoc -git clone https://github.com/tianshuo78520a/PaddlePaddle.org.git -cd ${PADDLE_ROOT}/PaddlePaddle.org -git reset 3feaa68376d8423e41d076814e901e6bf108c705 -cd ${PADDLE_ROOT}/FluidDoc/doc/fluid/api -sh gen_doc.sh -apt-get update && apt-get install -y python-dev build-essential -cd ${PADDLE_ROOT}/PaddlePaddle.org/portal -pip install -r requirements.txt -#If the default port is not occupied, you can use port 8000, you need to replace it with a random port on the CI. -sed -i "s#8000#$1#g" runserver -nohup ./runserver --paddle ${PADDLE_ROOT}/FluidDoc & +is_shell_attribute_set() { # attribute, like "x" + case "$-" in + *"$1"*) return 0 ;; + *) return 1 ;; + esac +} +function get_docs_pr_num_from_paddle_pr_info(){ + # get_repo_pr_info's output + pr_info_file=$1 + if [ ! -r ${pr_info_file} ] ; then + return 1 + fi + + declare -A arr_kv + while read line + do + echo "$line" | grep '^\w\+\s*=\s*.*' > /dev/null + if [ $? = 0 ] ; then + kv=($(echo $line | sed 's/=/\n/g')) + k=($(echo "${kv[0]}" | sed 's/\s//g')) + v=($(echo "${kv[1]}" | sed 's/^\s*//g' | sed 's/\s*$//g')) + # arr_kv[${kv[1]}]=${kv[2]} + arr_kv[${k}]=${v} + fi + done < <(jq -r '.body' ${pr_info_file}) + + echo ${arr_kv[PADDLEDOCS_PR]} + return 0 +} + +# Attention: +# 1. /FluidDoc will be used as the workspace of PaddlePaddle/docs. +# 2. And /docs is used as the output of doc-build process. +# 3. If conflicted with yours, please modify the defination of FLUIDDOCDIR and +# OUTPUTDIR in the subsequent codes. +# 4. The doc-build process is controlled under EnvVar BUILD_DOC and UPLOAD_DOC. +# All the Chinese and English docs will be generated, and then uploaded. + +PREVIEW_URL_PROMPT="ipipe_log_param_preview_url: None" +BUILD_DOC=${BUILD_DOC:=false} +UPLOAD_DOC=${UPLOAD_DOC:=false} + +CURPWD=${PWD} + +if [ -f /usr/local/python3.7.0/bin/sphinx-build ] ; then + if [ -f /usr/local/bin/sphinx-build ] ; then + rm /usr/local/bin/sphinx-build + fi + ln -s /usr/local/python3.7.0/bin/sphinx-build /usr/local/bin/sphinx-build +fi + +if [ "${BUILD_DOC}" = "true" ] && [ -x /usr/local/bin/sphinx-build ] ; then + export FLUIDDOCDIR=${FLUIDDOCDIR:=/FluidDoc} + export OUTPUTDIR=${OUTPUTDIR:=/docs} + export VERSIONSTR=$(echo ${BRANCH} | sed 's@release/@@g') + + if [ -d ${FLUIDDOCDIR} ] ; then + echo "${FLUIDDOCDIR} exists, git clone will be skipped, but git clean will be done." + cd ${FLUIDDOCDIR} + git reset --hard + git clean -dfx + cd ${CURPWD} + else + git clone -b ${BRANCH} --depth=1 https://github.com/PaddlePaddle/docs.git ${FLUIDDOCDIR} + if [ ! "$?" = "0" ] ; then + git clone --depth=1 https://github.com/PaddlePaddle/docs.git ${FLUIDDOCDIR} + fi + fi + if [ -d ${OUTPUTDIR} ] ; then + echo "$0: rm -rf ${OUTPUTDIR}" + rm -rf ${OUTPUTDIR} + mkdir -p ${OUTPUTDIR} + fi + + # install requirements + export no_proxy=mirror.baidu.com,${no_proxy} + apt-get install -y --no-install-recommends doxygen jq + echo 'beautifulsoup4 +Markdown +sphinx-sitemap +sphinx-markdown-tables +breathe +exhale +sphinx_design +nbsphinx +' >/tmp/doc-build.requirements && \ + pip install --no-cache-dir -i https://mirror.baidu.com/pypi/simple -r /tmp/doc-build.requirements && \ + rm /tmp/doc-build.requirements + + + source ${FLUIDDOCDIR}/ci_scripts/utils.sh + paddle_pr_info=$(get_repo_pr_info "PaddlePaddle/Paddle" ${GIT_PR_ID}) + docs_pr_id=$(get_docs_pr_num_from_paddle_pr_info ${paddle_pr_info}) + if [ -n "${docs_pr_id}" ] ; then + cd ${FLUIDDOCDIR} + git fetch --depth=1 origin pull/${docs_pr_id}/head + git checkout -b "pr${docs_pr_id}" FETCH_HEAD + git log --pretty=oneline -10 + fi + echo "docs_pr_id=${docs_pr_id}" + + + # build doc + /bin/bash -x ${FLUIDDOCDIR}/ci_scripts/gendoc.sh + if [ $? -ne 0 ];then + echo 'gendoc error' + exit 1 + fi + + if [ "${UPLOAD_DOC}" = "true" ] ; then + curl -o /tmp/linux-bcecmd-0.3.0.zip https://sdk.bce.baidu.com/console-sdk/linux-bcecmd-0.3.0.zip && \ + python -m zipfile -e /tmp/linux-bcecmd-0.3.0.zip /opt && \ + chmod +x /opt/linux-bcecmd-0.3.0/bcecmd && \ + rm /tmp/linux-bcecmd-0.3.0.zip && \ + curl -o /tmp/boscmdconfig.tgz https://paddle-dev-tools-open.bj.bcebos.com/fluiddoc-preview/boscmdconfig.tgz && \ + tar xzf /tmp/boscmdconfig.tgz -C /opt/linux-bcecmd-0.3.0/ && \ + rm /tmp/boscmdconfig.tgz + + # credentials file is empty, please build it if need. + BCECMD=/opt/linux-bcecmd-0.3.0/bcecmd + BCECMD_CONFIG=/opt/linux-bcecmd-0.3.0/boscmdconfig + + is_shell_attribute_set x + xdebug_setted=$? + if [ $xdebug_setted ] ; then + set +x + fi + if [ -n "${BOS_CREDENTIAL_AK}" ] && [ -n "${BOS_CREDENTIAL_SK}" ] ; then + echo "Ak = ${BOS_CREDENTIAL_AK}" >> ${BCECMD_CONFIG}/credentials + echo "Sk = ${BOS_CREDENTIAL_SK}" >> ${BCECMD_CONFIG}/credentials + fi + if [ $xdebug_setted ] ; then + set -x + fi + + PREVIEW_JOB_NAME="preview-paddle-pr-${GIT_PR_ID}" + BOSBUCKET=${BOSBUCKET:=paddle-site-web-dev} + ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/en/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/en/${PREVIEW_JOB_NAME}" \ + --delete --yes --exclude "${OUTPUTDIR}/en/${VERSIONSTR}/_sources/" + ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/en/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/en/${PREVIEW_JOB_NAME}" \ + --delete --yes --exclude "${OUTPUTDIR}/en/${VERSIONSTR}/_sources/" + ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/zh/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/zh/${PREVIEW_JOB_NAME}" \ + --delete --yes --exclude "${OUTPUTDIR}/zh/${VERSIONSTR}/_sources/" + ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/zh/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/zh/${PREVIEW_JOB_NAME}" \ + --delete --yes --exclude "${OUTPUTDIR}/zh/${VERSIONSTR}/_sources/" + PREVIEW_URL_PROMPT="ipipe_log_param_preview_url: http://${PREVIEW_JOB_NAME}.${PREVIEW_SITE:-paddle.run}/documentation/docs/zh/api/index_cn.html" + fi +fi + +cd ${CURPWD} +# print the preview url +echo "${PREVIEW_URL_PROMPT}" diff --git a/tools/sampcd_processor.py b/tools/sampcd_processor.py index d8cb70c9dd1..2d8692c5bc7 100644 --- a/tools/sampcd_processor.py +++ b/tools/sampcd_processor.py @@ -550,6 +550,42 @@ def get_incrementapi(): f.write('\n') +def exec_gen_doc(): + result = True + cmd = ["bash", "document_preview.sh"] + logger.info("----exec gen_doc----") + start_time = time.time() + subprc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = subprc.communicate() + msg = "".join(output.decode(encoding='utf-8')) + err = "".join(error.decode(encoding='utf-8')) + end_time = time.time() + + if subprc.returncode != 0: + logger.info("----gen_doc msg----") + logger.info(msg) + logger.error("----gen_doc error msg----") + logger.error(err) + logger.error("----exec gen_doc failed----") + result = False + else: + logger.info("----gen_doc msg----") + logger.info(msg) + logger.info("----exec gen_doc success----") + + for fn in [ + '/docs/en/develop/index_en.html', '/docs/zh/develop/index_cn.html' + ]: + if os.path.exists(fn): + logger.info('%s exists.', fn) + else: + logger.error('%s not exists.', fn) + + # msg is the returned code execution report + return result, msg, end_time - start_time + + arguments = [ # flags, dest, type, default, help ['--gpu_id', 'gpu_id', int, 0, 'GPU device id to use [0]'], @@ -570,6 +606,11 @@ def parse_args(): parser.add_argument('--debug', dest='debug', action="store_true") parser.add_argument('--full-test', dest='full_test', action="store_true") parser.add_argument('mode', type=str, help='run on device', default='cpu') + parser.add_argument( + '--build-doc', + dest='build_doc', + action='store_true', + help='build doc if need.') for item in arguments: parser.add_argument( item[0], dest=item[1], help=item[4], type=item[2], default=item[3]) @@ -702,3 +743,7 @@ if __name__ == '__main__': exit(1) logger.info("Sample code check is successful!") + + if args.mode == "cpu": + # As cpu mode is also run with the GPU whl, so skip it in gpu mode. + exec_gen_doc() -- GitLab