From 71bc8e9993f610216e7eed079ad8c680e313b279 Mon Sep 17 00:00:00 2001 From: risemeup1 <515586620@qq.com> Date: Mon, 2 Jan 2023 14:20:23 +0000 Subject: [PATCH] modify setup.py and paddle_build.sh --- paddle/scripts/paddle_build.sh | 40 +++++++-- setup.py | 156 ++++++++++++++++++--------------- 2 files changed, 118 insertions(+), 78 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index a234e4906ff..32fc317b1fb 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -3476,13 +3476,16 @@ function check_coverage_build() { set -x } function run_setup(){ + startTime_s=`date +%s` + mkdir -p ${PADDLE_ROOT}/build + cd ${PADDLE_ROOT}/build # Build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true # Delete previous built egg packages rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true # Delete previous built paddle cache rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true - startTime_s=`date +%s` + SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then echo "Using python abi: $1" @@ -3640,7 +3643,7 @@ function run_setup(){ export WITH_MLU=${WITH_MLU:-OFF} export WITH_IPU=${WITH_IPU:-OFF} export WITH_CNCL=${WITH_CNCL:-OFF} - export XPU_SDK_ROOT=${XPU_SDK_ROOT:-} + export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""} export WITH_LITE=${WITH_LITE:-OFF} export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF} export WITH_ARM=${WITH_ARM:-OFF} @@ -3657,12 +3660,28 @@ function run_setup(){ export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF} + if [ "$SYSTEM" == "Linux" ];then + if [ `nproc` -gt 16 ];then + parallel_number=$(expr `nproc` - 8) + else + parallel_number=`nproc` + fi + else + parallel_number=8 + fi + if [ "$2" != "" ]; then + parallel_number=$2 + fi + export MAX_JOBS=${parallel_number} + # reset ccache zero stats for collect PR's actual hit rate + ccache -z + cd .. if [ "${PYTHON_EXECUTABLE}" != "" ];then - ${PYTHON_EXECUTABLE} setup.py $2;build_error=$? + ${PYTHON_EXECUTABLE} setup.py $3;build_error=$? else - python setup.py $2;build_error=$? + python setup.py $3;build_error=$? fi # ci will collect ccache hit rate @@ -3672,14 +3691,19 @@ function run_setup(){ exit 7; fi + endTime_s=`date +%s` + [ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild + echo "Build Time: $[ $endTime_s - $startTime_s ]s" + echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt } + function main() { local CMD=$1 local parallel_number=$2 init case $CMD in build_only) - run_setup ${PYTHON_ABI:-""} bdist_wheel + run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel ;; build_pr_dev) build_pr_and_develop @@ -3796,7 +3820,7 @@ function main() { ;; cicheck_coverage) check_diff_file_for_coverage - run_setup ${PYTHON_ABI:-""} install + run_setup ${PYTHON_ABI:-""} ${parallel_number} install enable_unused_var_check parallel_test check_coverage @@ -3883,7 +3907,7 @@ function main() { build_mac ;; cicheck_py37) - run_setup ${PYTHON_ABI:-""} bdist_wheel + run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} ;; test_cicheck_py37) @@ -3896,7 +3920,7 @@ function main() { parallel_test ;; build_gpubox) - run_setup ${PYTHON_ABI:-""} install + run_setup ${PYTHON_ABI:-""} ${parallel_number} install ;; check_xpu) cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} diff --git a/setup.py b/setup.py index 092af614010..00a72c5a3b1 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ import platform import re import shutil import subprocess +import sysconfig import sys from contextlib import contextmanager from distutils.spawn import find_executable @@ -49,6 +50,21 @@ assert ( CMAKE ), 'The "cmake" executable is not found. Please check if Cmake is installed.' +#CMAKE: full path to python library +if (platform.system() == "Windows"): + cmake_python_library = "{}/libs/python{}.lib".format( + sysconfig.get_config_var("prefix"), + sysconfig.get_config_var("VERSION")) + # Fix virtualenv builds + if not os.path.exists(cmake_python_library): + cmake_python_library = "{}/libs/python{}.lib".format( + sys.base_prefix, + sysconfig.get_config_var("VERSION")) +else: + cmake_python_library = "{}/{}".format( + sysconfig.get_config_var("LIBDIR"), + sysconfig.get_config_var("INSTSONAME")) + TOP_DIR = os.path.dirname(os.path.realpath(__file__)) IS_WINDOWS = os.name == 'nt' @@ -555,7 +571,14 @@ def options_process(args, build_options): for key, value in sorted(build_options.items()): if value is not None: args.append("-D{}={}".format(key, value)) - + if ('PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys()): + args.append("-D{}={}".format('PYTHON_EXECUTABLE',sys.executable)) + if ('PYTHON_INCLUDE_DIR:PATH' not in build_options.keys()): + args.append('-D{}={}'.format('PYTHON_INCLUDE_DIR',sysconfig.get_path("include"))) + if ('PYTHON_LIBRARY:FILEPATH' not in build_options.keys()): + args.append('-D{}={}'.format('PYTHON_LIBRARY',cmake_python_library)) + + def get_cmake_generator(): if os.getenv("CMAKE_GENERATOR"): @@ -566,7 +589,61 @@ def get_cmake_generator(): return cmake_generator -def cmake_run(args, build_path): +def cmake_run(build_path): + args = [] + env_var = os.environ.copy() # get env variables + paddle_build_options = {} + other_options = {} + other_options.update( + { + option: option + for option in ( + # "PYTHON_LIBRARY", + "INFERENCE_DEMO_INSTALL_DIR", + "ON_INFER", + # "PYTHON_EXECUTABLE", + "TENSORRT_ROOT", + "CUDA_ARCH_NAME", + "CUDA_ARCH_BIN", + # "PYTHON_INCLUDE_DIR", + # "PYTHON_LIBRARIES", + "PY_VERSION", + "CUB_PATH", + "NEW_RELEASE_PYPI", + "CUDNN_ROOT", + "THIRD_PARTY_PATH", + "NOAVX_CORE_FILE", + "LITE_GIT_TAG", + "CUDA_TOOLKIT_ROOT_DIR", + "NEW_RELEASE_JIT", + "XPU_SDK_ROOT", + "MSVC_STATIC_CRT", + "NEW_RELEASE_ALL", + "CMAKE_GENERATOR", + ) + } + ) + # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options + for option_key, option_value in env_var.items(): + if option_key.startswith(("CMAKE_", "WITH_")): + paddle_build_options[option_key] = option_value + if option_key in other_options: + if ( + option_key == 'PYTHON_EXECUTABLE' + or option_key == 'PYTHON_LIBRARY' + or option_key == 'PYTHON_LIBRARIES' + ): + key = option_key + ":FILEPATH" + print(key) + elif option_key == 'PYTHON_INCLUDE_DIR': + key = key = option_key + ':PATH' + print(key) + else: + key = other_options[option_key] + if key not in paddle_build_options: + paddle_build_options[key] = option_value + options_process(args, paddle_build_options) + print("args:", args) with cd(build_path): cmake_args = [] cmake_args.append(CMAKE) @@ -621,74 +698,14 @@ def build_steps(): # if rerun_cmake is True,remove CMakeCache.txt and rerun camke if os.path.isfile(cmake_cache_file_path) and rerun_cmake is True: os.remove(cmake_cache_file_path) - + CMAKE_GENERATOR = get_cmake_generator() - - if CMAKE_GENERATOR == "Ninja": - build_ninja_file_path = os.path.join(build_path, "build.ninja") - # if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)): - if os.path.exists(cmake_cache_file_path) and os.path.exists( - build_ninja_file_path - ): - print("Do not need rerun camke,everything is ready,run build now") - run_cmake_build(build_path) - return - - args = [] - env_var = os.environ.copy() # get env variables - paddle_build_options = {} - other_options = {} - other_options.update( - { - option: option - for option in ( - "PYTHON_LIBRARY", - "INFERENCE_DEMO_INSTALL_DIR", - "ON_INFER", - "PYTHON_EXECUTABLE", - "TENSORRT_ROOT", - "CUDA_ARCH_NAME", - "CUDA_ARCH_BIN", - "PYTHON_INCLUDE_DIR", - "PYTHON_LIBRARIES", - "PY_VERSION", - "CUB_PATH", - "NEW_RELEASE_PYPI", - "CUDNN_ROOT", - "THIRD_PARTY_PATH", - "NOAVX_CORE_FILE", - "LITE_GIT_TAG", - "CUDA_TOOLKIT_ROOT_DIR", - "NEW_RELEASE_JIT", - "XPU_SDK_ROOT", - "MSVC_STATIC_CRT", - "NEW_RELEASE_ALL", - "CMAKE_GENERATOR", - ) - } - ) - # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options - for option_key, option_value in env_var.items(): - if option_key.startswith(("CMAKE_", "WITH_")): - paddle_build_options[option_key] = option_value - if option_key in other_options: - if ( - option_key == 'PYTHON_EXECUTABLE' - or option_key == 'PYTHON_LIBRARY' - or option_key == 'PYTHON_LIBRARIES' - ): - key = option_key + ":FILEPATH" - print(key) - elif option_key == 'PYTHON_INCLUDE_DIR': - key = key = option_key + ':PATH' - print(key) - else: - key = other_options[option_key] - if key not in paddle_build_options: - paddle_build_options[key] = option_value - options_process(args, paddle_build_options) - print("args:", args) - cmake_run(args, build_path) + bool_ninja = (CMAKE_GENERATOR == "Ninja") + build_ninja_file_path = os.path.join(build_path, "build.ninja") + if os.path.exists(cmake_cache_file_path) and not (bool_ninja and not os.path.exists(build_ninja_file_path)): + print("Do not need rerun camke,everything is ready,run build now") + else: + cmake_run(build_path) # make if only_cmake: print( @@ -1342,7 +1359,6 @@ def main(): paddle_binary_dir ) ) - ( setup_requires, packages, -- GitLab