From 1228bad073bdff03dcadb759ce74f37ee5967044 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Thu, 5 Jan 2023 10:01:06 +0800 Subject: [PATCH] keep run_setup and cmake_gen_and_build same (#46957) * modify setup.py and paddle_build.sh * modify setup.py and paddle_build.sh * modify setup.py and paddle_build.sh * modify setup.py * modify run_setup * modify setup.py * fix make_clean * modify setup.py * modify setup.py * delete setting python_libary * debug * debug * debug * debug --- paddle/scripts/paddle_build.sh | 35 +++++-- setup.py | 167 +++++++++++++++++++-------------- 2 files changed, 126 insertions(+), 76 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index a234e4906f..1587ff83fb 100644 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -3476,13 +3476,16 @@ function check_coverage_build() { set -x } function run_setup(){ + startTime_s=`date +%s` + mkdir -p ${PADDLE_ROOT}/build + cd ${PADDLE_ROOT}/build # Build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true # Delete previous built egg packages rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true # Delete previous built paddle cache rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true - startTime_s=`date +%s` + SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then echo "Using python abi: $1" @@ -3640,7 +3643,7 @@ function run_setup(){ export WITH_MLU=${WITH_MLU:-OFF} export WITH_IPU=${WITH_IPU:-OFF} export WITH_CNCL=${WITH_CNCL:-OFF} - export XPU_SDK_ROOT=${XPU_SDK_ROOT:-} + export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""} export WITH_LITE=${WITH_LITE:-OFF} export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF} export WITH_ARM=${WITH_ARM:-OFF} @@ -3657,8 +3660,23 @@ function run_setup(){ export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF} + if [ "$SYSTEM" == "Linux" ];then + if [ `nproc` -gt 16 ];then + parallel_number=$(expr `nproc` - 8) + else + parallel_number=`nproc` + fi + else + parallel_number=8 + fi + if [ "$3" != "" ]; then + parallel_number=$3 + fi + export MAX_JOBS=${parallel_number} # reset ccache zero stats for collect PR's actual hit rate + ccache -z + cd .. if [ "${PYTHON_EXECUTABLE}" != "" ];then ${PYTHON_EXECUTABLE} setup.py $2;build_error=$? else @@ -3672,14 +3690,19 @@ function run_setup(){ exit 7; fi + endTime_s=`date +%s` + [ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild + echo "Build Time: $[ $endTime_s - $startTime_s ]s" + echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt } + function main() { local CMD=$1 local parallel_number=$2 init case $CMD in build_only) - run_setup ${PYTHON_ABI:-""} bdist_wheel + run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number} ;; build_pr_dev) build_pr_and_develop @@ -3796,7 +3819,7 @@ function main() { ;; cicheck_coverage) check_diff_file_for_coverage - run_setup ${PYTHON_ABI:-""} install + run_setup ${PYTHON_ABI:-""} install ${parallel_number} enable_unused_var_check parallel_test check_coverage @@ -3883,7 +3906,7 @@ function main() { build_mac ;; cicheck_py37) - run_setup ${PYTHON_ABI:-""} bdist_wheel + run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number} run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} ;; test_cicheck_py37) @@ -3896,7 +3919,7 @@ function main() { parallel_test ;; build_gpubox) - run_setup ${PYTHON_ABI:-""} install + run_setup ${PYTHON_ABI:-""} install ${parallel_number} ;; check_xpu) cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} diff --git a/setup.py b/setup.py index 879adb4935..39b33ba686 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ import re import shutil import subprocess import sys +import sysconfig from contextlib import contextmanager from distutils.spawn import find_executable from subprocess import CalledProcessError @@ -49,6 +50,28 @@ assert ( CMAKE ), 'The "cmake" executable is not found. Please check if Cmake is installed.' +# CMAKE: full path to python library +if platform.system() == "Windows": + cmake_python_library = "{}/libs/python{}.lib".format( + sysconfig.get_config_var("prefix"), sysconfig.get_config_var("VERSION") + ) + # Fix virtualenv builds + if not os.path.exists(cmake_python_library): + cmake_python_library = "{}/libs/python{}.lib".format( + sys.base_prefix, sysconfig.get_config_var("VERSION") + ) +else: + cmake_python_library = "{}/{}".format( + sysconfig.get_config_var("LIBDIR"), + sysconfig.get_config_var("INSTSONAME"), + ) + if not os.path.exists(cmake_python_library): + libname = sysconfig.get_config_var("INSTSONAME") + libdir = sysconfig.get_config_var('LIBDIR') + ( + sysconfig.get_config_var("multiarchsubdir") or "" + ) + cmake_python_library = os.path.join(libdir, libname) + TOP_DIR = os.path.dirname(os.path.realpath(__file__)) IS_WINDOWS = os.name == 'nt' @@ -555,6 +578,16 @@ def options_process(args, build_options): for key, value in sorted(build_options.items()): if value is not None: args.append("-D{}={}".format(key, value)) + if 'PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys(): + args.append("-D{}={}".format('PYTHON_EXECUTABLE', sys.executable)) + if 'PYTHON_INCLUDE_DIR:PATH' not in build_options.keys(): + args.append( + '-D{}={}'.format( + 'PYTHON_INCLUDE_DIR', sysconfig.get_path("include") + ) + ) + if 'PYTHON_LIBRARY:FILEPATH' not in build_options.keys(): + args.append('-D{}={}'.format('PYTHON_LIBRARY', cmake_python_library)) def get_cmake_generator(): @@ -562,11 +595,64 @@ def get_cmake_generator(): cmake_generator = os.getenv("CMAKE_GENERATOR") else: cmake_generator = "Unix Makefiles" - return cmake_generator -def cmake_run(args, build_path): +def cmake_run(build_path): + args = [] + env_var = os.environ.copy() # get env variables + paddle_build_options = {} + other_options = {} + other_options.update( + { + option: option + for option in ( + "PYTHON_LIBRARY", + "INFERENCE_DEMO_INSTALL_DIR", + "ON_INFER", + "PYTHON_EXECUTABLE", + "TENSORRT_ROOT", + "CUDA_ARCH_NAME", + "CUDA_ARCH_BIN", + "PYTHON_INCLUDE_DIR", + "PYTHON_LIBRARIES", + "PY_VERSION", + "CUB_PATH", + "NEW_RELEASE_PYPI", + "CUDNN_ROOT", + "THIRD_PARTY_PATH", + "NOAVX_CORE_FILE", + "LITE_GIT_TAG", + "CUDA_TOOLKIT_ROOT_DIR", + "NEW_RELEASE_JIT", + "XPU_SDK_ROOT", + "MSVC_STATIC_CRT", + "NEW_RELEASE_ALL", + "CMAKE_GENERATOR", + ) + } + ) + # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options + for option_key, option_value in env_var.items(): + if option_key.startswith(("CMAKE_", "WITH_")): + paddle_build_options[option_key] = option_value + if option_key in other_options: + if ( + option_key == 'PYTHON_EXECUTABLE' + or option_key == 'PYTHON_LIBRARY' + or option_key == 'PYTHON_LIBRARIES' + ): + key = option_key + ":FILEPATH" + print(key) + elif option_key == 'PYTHON_INCLUDE_DIR': + key = option_key + ':PATH' + print(key) + else: + key = other_options[option_key] + if key not in paddle_build_options: + paddle_build_options[key] = option_value + options_process(args, paddle_build_options) + print("args:", args) with cd(build_path): cmake_args = [] cmake_args.append(CMAKE) @@ -623,72 +709,14 @@ def build_steps(): os.remove(cmake_cache_file_path) CMAKE_GENERATOR = get_cmake_generator() - - if CMAKE_GENERATOR == "Ninja": - build_ninja_file_path = os.path.join(build_path, "build.ninja") - # if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)): - if os.path.exists(cmake_cache_file_path) and os.path.exists( - build_ninja_file_path - ): - print("Do not need rerun camke,everything is ready,run build now") - run_cmake_build(build_path) - return - - args = [] - env_var = os.environ.copy() # get env variables - paddle_build_options = {} - other_options = {} - other_options.update( - { - option: option - for option in ( - "PYTHON_LIBRARY", - "INFERENCE_DEMO_INSTALL_DIR", - "ON_INFER", - "PYTHON_EXECUTABLE", - "TENSORRT_ROOT", - "CUDA_ARCH_NAME", - "CUDA_ARCH_BIN", - "PYTHON_INCLUDE_DIR", - "PYTHON_LIBRARIES", - "PY_VERSION", - "CUB_PATH", - "NEW_RELEASE_PYPI", - "CUDNN_ROOT", - "THIRD_PARTY_PATH", - "NOAVX_CORE_FILE", - "LITE_GIT_TAG", - "CUDA_TOOLKIT_ROOT_DIR", - "NEW_RELEASE_JIT", - "XPU_SDK_ROOT", - "MSVC_STATIC_CRT", - "NEW_RELEASE_ALL", - "CMAKE_GENERATOR", - ) - } - ) - # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options - for option_key, option_value in env_var.items(): - if option_key.startswith(("CMAKE_", "WITH_")): - paddle_build_options[option_key] = option_value - if option_key in other_options: - if ( - option_key == 'PYTHON_EXECUTABLE' - or option_key == 'PYTHON_LIBRARY' - or option_key == 'PYTHON_LIBRARIES' - ): - key = option_key + ":FILEPATH" - print(key) - elif option_key == 'PYTHON_INCLUDE_DIR': - key = key = option_key + ':PATH' - print(key) - else: - key = other_options[option_key] - if key not in paddle_build_options: - paddle_build_options[key] = option_value - options_process(args, paddle_build_options) - print("args:", args) - cmake_run(args, build_path) + bool_ninja = CMAKE_GENERATOR == "Ninja" + build_ninja_file_path = os.path.join(build_path, "build.ninja") + if os.path.exists(cmake_cache_file_path) and not ( + bool_ninja and not os.path.exists(build_ninja_file_path) + ): + print("Do not need rerun camke, everything is ready, run build now") + else: + cmake_run(build_path) # make if only_cmake: print( @@ -1317,7 +1345,7 @@ def main(): env_dict_path = TOP_DIR + '/' + build_dir + '/python' else: env_dict_path = TOP_DIR + "/build/python/" - sys.path.append(env_dict_path) + sys.path.insert(1, env_dict_path) from env_dict import env_dict as env_dict global env_dict @@ -1341,7 +1369,6 @@ def main(): paddle_binary_dir ) ) - ( setup_requires, packages, -- GitLab