未验证 提交 1228bad0 编写于 作者: R risemeup1 提交者: GitHub

keep run_setup and cmake_gen_and_build same (#46957)

* modify setup.py and paddle_build.sh

* modify setup.py and paddle_build.sh

* modify setup.py and paddle_build.sh

* modify setup.py

* modify run_setup

* modify setup.py

* fix make_clean

* modify setup.py

* modify setup.py

* delete setting python_libary

* debug

* debug

* debug

* debug
上级 da3e9d66
...@@ -3476,13 +3476,16 @@ function check_coverage_build() { ...@@ -3476,13 +3476,16 @@ function check_coverage_build() {
set -x set -x
} }
function run_setup(){ function run_setup(){
startTime_s=`date +%s`
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
# Build script will not fail if *.deb does not exist # Build script will not fail if *.deb does not exist
rm *.deb 2>/dev/null || true rm *.deb 2>/dev/null || true
# Delete previous built egg packages # Delete previous built egg packages
rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true
# Delete previous built paddle cache # Delete previous built paddle cache
rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true
startTime_s=`date +%s`
SYSTEM=`uname -s` SYSTEM=`uname -s`
if [ "$SYSTEM" == "Darwin" ]; then if [ "$SYSTEM" == "Darwin" ]; then
echo "Using python abi: $1" echo "Using python abi: $1"
...@@ -3640,7 +3643,7 @@ function run_setup(){ ...@@ -3640,7 +3643,7 @@ function run_setup(){
export WITH_MLU=${WITH_MLU:-OFF} export WITH_MLU=${WITH_MLU:-OFF}
export WITH_IPU=${WITH_IPU:-OFF} export WITH_IPU=${WITH_IPU:-OFF}
export WITH_CNCL=${WITH_CNCL:-OFF} export WITH_CNCL=${WITH_CNCL:-OFF}
export XPU_SDK_ROOT=${XPU_SDK_ROOT:-} export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""}
export WITH_LITE=${WITH_LITE:-OFF} export WITH_LITE=${WITH_LITE:-OFF}
export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF} export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF}
export WITH_ARM=${WITH_ARM:-OFF} export WITH_ARM=${WITH_ARM:-OFF}
...@@ -3657,8 +3660,23 @@ function run_setup(){ ...@@ -3657,8 +3660,23 @@ function run_setup(){
export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF}
export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF} export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF}
if [ "$SYSTEM" == "Linux" ];then
if [ `nproc` -gt 16 ];then
parallel_number=$(expr `nproc` - 8)
else
parallel_number=`nproc`
fi
else
parallel_number=8
fi
if [ "$3" != "" ]; then
parallel_number=$3
fi
export MAX_JOBS=${parallel_number}
# reset ccache zero stats for collect PR's actual hit rate # reset ccache zero stats for collect PR's actual hit rate
ccache -z ccache -z
cd ..
if [ "${PYTHON_EXECUTABLE}" != "" ];then if [ "${PYTHON_EXECUTABLE}" != "" ];then
${PYTHON_EXECUTABLE} setup.py $2;build_error=$? ${PYTHON_EXECUTABLE} setup.py $2;build_error=$?
else else
...@@ -3672,14 +3690,19 @@ function run_setup(){ ...@@ -3672,14 +3690,19 @@ function run_setup(){
exit 7; exit 7;
fi fi
endTime_s=`date +%s`
[ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
} }
function main() { function main() {
local CMD=$1 local CMD=$1
local parallel_number=$2 local parallel_number=$2
init init
case $CMD in case $CMD in
build_only) build_only)
run_setup ${PYTHON_ABI:-""} bdist_wheel run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number}
;; ;;
build_pr_dev) build_pr_dev)
build_pr_and_develop build_pr_and_develop
...@@ -3796,7 +3819,7 @@ function main() { ...@@ -3796,7 +3819,7 @@ function main() {
;; ;;
cicheck_coverage) cicheck_coverage)
check_diff_file_for_coverage check_diff_file_for_coverage
run_setup ${PYTHON_ABI:-""} install run_setup ${PYTHON_ABI:-""} install ${parallel_number}
enable_unused_var_check enable_unused_var_check
parallel_test parallel_test
check_coverage check_coverage
...@@ -3883,7 +3906,7 @@ function main() { ...@@ -3883,7 +3906,7 @@ function main() {
build_mac build_mac
;; ;;
cicheck_py37) cicheck_py37)
run_setup ${PYTHON_ABI:-""} bdist_wheel run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number}
run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
;; ;;
test_cicheck_py37) test_cicheck_py37)
...@@ -3896,7 +3919,7 @@ function main() { ...@@ -3896,7 +3919,7 @@ function main() {
parallel_test parallel_test
;; ;;
build_gpubox) build_gpubox)
run_setup ${PYTHON_ABI:-""} install run_setup ${PYTHON_ABI:-""} install ${parallel_number}
;; ;;
check_xpu) check_xpu)
cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
......
...@@ -22,6 +22,7 @@ import re ...@@ -22,6 +22,7 @@ import re
import shutil import shutil
import subprocess import subprocess
import sys import sys
import sysconfig
from contextlib import contextmanager from contextlib import contextmanager
from distutils.spawn import find_executable from distutils.spawn import find_executable
from subprocess import CalledProcessError from subprocess import CalledProcessError
...@@ -49,6 +50,28 @@ assert ( ...@@ -49,6 +50,28 @@ assert (
CMAKE CMAKE
), 'The "cmake" executable is not found. Please check if Cmake is installed.' ), 'The "cmake" executable is not found. Please check if Cmake is installed.'
# CMAKE: full path to python library
if platform.system() == "Windows":
cmake_python_library = "{}/libs/python{}.lib".format(
sysconfig.get_config_var("prefix"), sysconfig.get_config_var("VERSION")
)
# Fix virtualenv builds
if not os.path.exists(cmake_python_library):
cmake_python_library = "{}/libs/python{}.lib".format(
sys.base_prefix, sysconfig.get_config_var("VERSION")
)
else:
cmake_python_library = "{}/{}".format(
sysconfig.get_config_var("LIBDIR"),
sysconfig.get_config_var("INSTSONAME"),
)
if not os.path.exists(cmake_python_library):
libname = sysconfig.get_config_var("INSTSONAME")
libdir = sysconfig.get_config_var('LIBDIR') + (
sysconfig.get_config_var("multiarchsubdir") or ""
)
cmake_python_library = os.path.join(libdir, libname)
TOP_DIR = os.path.dirname(os.path.realpath(__file__)) TOP_DIR = os.path.dirname(os.path.realpath(__file__))
IS_WINDOWS = os.name == 'nt' IS_WINDOWS = os.name == 'nt'
...@@ -555,6 +578,16 @@ def options_process(args, build_options): ...@@ -555,6 +578,16 @@ def options_process(args, build_options):
for key, value in sorted(build_options.items()): for key, value in sorted(build_options.items()):
if value is not None: if value is not None:
args.append("-D{}={}".format(key, value)) args.append("-D{}={}".format(key, value))
if 'PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys():
args.append("-D{}={}".format('PYTHON_EXECUTABLE', sys.executable))
if 'PYTHON_INCLUDE_DIR:PATH' not in build_options.keys():
args.append(
'-D{}={}'.format(
'PYTHON_INCLUDE_DIR', sysconfig.get_path("include")
)
)
if 'PYTHON_LIBRARY:FILEPATH' not in build_options.keys():
args.append('-D{}={}'.format('PYTHON_LIBRARY', cmake_python_library))
def get_cmake_generator(): def get_cmake_generator():
...@@ -562,11 +595,64 @@ def get_cmake_generator(): ...@@ -562,11 +595,64 @@ def get_cmake_generator():
cmake_generator = os.getenv("CMAKE_GENERATOR") cmake_generator = os.getenv("CMAKE_GENERATOR")
else: else:
cmake_generator = "Unix Makefiles" cmake_generator = "Unix Makefiles"
return cmake_generator return cmake_generator
def cmake_run(args, build_path): def cmake_run(build_path):
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
with cd(build_path): with cd(build_path):
cmake_args = [] cmake_args = []
cmake_args.append(CMAKE) cmake_args.append(CMAKE)
...@@ -623,72 +709,14 @@ def build_steps(): ...@@ -623,72 +709,14 @@ def build_steps():
os.remove(cmake_cache_file_path) os.remove(cmake_cache_file_path)
CMAKE_GENERATOR = get_cmake_generator() CMAKE_GENERATOR = get_cmake_generator()
bool_ninja = CMAKE_GENERATOR == "Ninja"
if CMAKE_GENERATOR == "Ninja": build_ninja_file_path = os.path.join(build_path, "build.ninja")
build_ninja_file_path = os.path.join(build_path, "build.ninja") if os.path.exists(cmake_cache_file_path) and not (
# if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)): bool_ninja and not os.path.exists(build_ninja_file_path)
if os.path.exists(cmake_cache_file_path) and os.path.exists( ):
build_ninja_file_path print("Do not need rerun camke, everything is ready, run build now")
): else:
print("Do not need rerun camke,everything is ready,run build now") cmake_run(build_path)
run_cmake_build(build_path)
return
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
cmake_run(args, build_path)
# make # make
if only_cmake: if only_cmake:
print( print(
...@@ -1317,7 +1345,7 @@ def main(): ...@@ -1317,7 +1345,7 @@ def main():
env_dict_path = TOP_DIR + '/' + build_dir + '/python' env_dict_path = TOP_DIR + '/' + build_dir + '/python'
else: else:
env_dict_path = TOP_DIR + "/build/python/" env_dict_path = TOP_DIR + "/build/python/"
sys.path.append(env_dict_path) sys.path.insert(1, env_dict_path)
from env_dict import env_dict as env_dict from env_dict import env_dict as env_dict
global env_dict global env_dict
...@@ -1341,7 +1369,6 @@ def main(): ...@@ -1341,7 +1369,6 @@ def main():
paddle_binary_dir paddle_binary_dir
) )
) )
( (
setup_requires, setup_requires,
packages, packages,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册