Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 1228bad0 编写于 作者: R risemeup1 提交者： GitHub
浏览文件
操作

keep run_setup and cmake_gen_and_build same (#46957) 

* modify setup.py and paddle_build.sh

* modify setup.py and paddle_build.sh

* modify setup.py and paddle_build.sh

* modify setup.py

* modify run_setup

* modify setup.py

* fix make_clean

* modify setup.py

* modify setup.py

* delete setting python_libary

* debug

* debug

* debug

* debug
上级 da3e9d66
显示空白变更内容
内联 并排
Showing with 126 addition and 76 deletion
paddle/scripts/paddle_build.sh
浏览文件 @ 1228bad0
......@@ -3476,13 +3476,16 @@ function check_coverage_build() {
set -x
}
function run_setup(){
startTime_s=`date +%s`
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
# Build script will not fail if *.deb does not exist
rm *.deb 2>/dev/null || true
# Delete previous built egg packages
rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true
# Delete previous built paddle cache
rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true
startTime_s=`date +%s`
SYSTEM=`uname -s`
if [ "$SYSTEM" == "Darwin" ]; then
echo "Using python abi: $1"
......@@ -3640,7 +3643,7 @@ function run_setup(){
export WITH_MLU=${WITH_MLU:-OFF}
export WITH_IPU=${WITH_IPU:-OFF}
export WITH_CNCL=${WITH_CNCL:-OFF}
export XPU_SDK_ROOT=${XPU_SDK_ROOT:-}
export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""}
export WITH_LITE=${WITH_LITE:-OFF}
export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF}
export WITH_ARM=${WITH_ARM:-OFF}
......@@ -3657,8 +3660,23 @@ function run_setup(){
export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF}
export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF}
if [ "$SYSTEM" == "Linux" ];then
if [ `nproc` -gt 16 ];then
parallel_number=$(expr `nproc` - 8)
else
parallel_number=`nproc`
fi
else
parallel_number=8
fi
if [ "$3" != "" ]; then
parallel_number=$3
fi
export MAX_JOBS=${parallel_number}
# reset ccache zero stats for collect PR's actual hit rate
ccache -z
cd ..
if [ "${PYTHON_EXECUTABLE}" != "" ];then
${PYTHON_EXECUTABLE} setup.py $2;build_error=$?
else
......@@ -3672,14 +3690,19 @@ function run_setup(){
exit 7;
fi
endTime_s=`date +%s`
[ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
}
function main() {
local CMD=$1
local parallel_number=$2
init
case $CMD in
build_only)
run_setup ${PYTHON_ABI:-""} bdist_wheel
run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number}
;;
build_pr_dev)
build_pr_and_develop
......@@ -3796,7 +3819,7 @@ function main() {
;;
cicheck_coverage)
check_diff_file_for_coverage
run_setup ${PYTHON_ABI:-""} install
run_setup ${PYTHON_ABI:-""} install ${parallel_number}
enable_unused_var_check
parallel_test
check_coverage
......@@ -3883,7 +3906,7 @@ function main() {
build_mac
;;
cicheck_py37)
run_setup ${PYTHON_ABI:-""} bdist_wheel
run_setup ${PYTHON_ABI:-""} bdist_wheel ${parallel_number}
run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
;;
test_cicheck_py37)
......@@ -3896,7 +3919,7 @@ function main() {
parallel_test
;;
build_gpubox)
run_setup ${PYTHON_ABI:-""} install
run_setup ${PYTHON_ABI:-""} install ${parallel_number}
;;
check_xpu)
cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
......
setup.py
浏览文件 @ 1228bad0
......@@ -22,6 +22,7 @@ import re
import shutil
import subprocess
import sys
import sysconfig
from contextlib import contextmanager
from distutils.spawn import find_executable
from subprocess import CalledProcessError
......@@ -49,6 +50,28 @@ assert (
CMAKE
), 'The "cmake" executable is not found. Please check if Cmake is installed.'
# CMAKE: full path to python library
if platform.system() == "Windows":
cmake_python_library = "{}/libs/python{}.lib".format(
sysconfig.get_config_var("prefix"), sysconfig.get_config_var("VERSION")
)
# Fix virtualenv builds
if not os.path.exists(cmake_python_library):
cmake_python_library = "{}/libs/python{}.lib".format(
sys.base_prefix, sysconfig.get_config_var("VERSION")
)
else:
cmake_python_library = "{}/{}".format(
sysconfig.get_config_var("LIBDIR"),
sysconfig.get_config_var("INSTSONAME"),
)
if not os.path.exists(cmake_python_library):
libname = sysconfig.get_config_var("INSTSONAME")
libdir = sysconfig.get_config_var('LIBDIR') + (
sysconfig.get_config_var("multiarchsubdir") or ""
)
cmake_python_library = os.path.join(libdir, libname)
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
IS_WINDOWS = os.name == 'nt'
......@@ -555,6 +578,16 @@ def options_process(args, build_options):
for key, value in sorted(build_options.items()):
if value is not None:
args.append("-D{}={}".format(key, value))
if 'PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys():
args.append("-D{}={}".format('PYTHON_EXECUTABLE', sys.executable))
if 'PYTHON_INCLUDE_DIR:PATH' not in build_options.keys():
args.append(
'-D{}={}'.format(
'PYTHON_INCLUDE_DIR', sysconfig.get_path("include")
)
)
if 'PYTHON_LIBRARY:FILEPATH' not in build_options.keys():
args.append('-D{}={}'.format('PYTHON_LIBRARY', cmake_python_library))
def get_cmake_generator():
......@@ -562,11 +595,64 @@ def get_cmake_generator():
cmake_generator = os.getenv("CMAKE_GENERATOR")
else:
cmake_generator = "Unix Makefiles"
return cmake_generator
def cmake_run(args, build_path):
def cmake_run(build_path):
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
with cd(build_path):
cmake_args = []
cmake_args.append(CMAKE)
......@@ -623,72 +709,14 @@ def build_steps():
os.remove(cmake_cache_file_path)
CMAKE_GENERATOR = get_cmake_generator()
if CMAKE_GENERATOR == "Ninja":
bool_ninja = CMAKE_GENERATOR == "Ninja"
build_ninja_file_path = os.path.join(build_path, "build.ninja")
# if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)):
if os.path.exists(cmake_cache_file_path) and os.path.exists(
build_ninja_file_path
if os.path.exists(cmake_cache_file_path) and not (
bool_ninja and not os.path.exists(build_ninja_file_path)
):
print("Do not need rerun camke,everything is ready,run build now")
run_cmake_build(build_path)
return
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
print("Do not need rerun camke, everything is ready, run build now")
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
cmake_run(args, build_path)
cmake_run(build_path)
# make
if only_cmake:
print(
......@@ -1317,7 +1345,7 @@ def main():
env_dict_path = TOP_DIR + '/' + build_dir + '/python'
else:
env_dict_path = TOP_DIR + "/build/python/"
sys.path.append(env_dict_path)
sys.path.insert(1, env_dict_path)
from env_dict import env_dict as env_dict
global env_dict
......@@ -1341,7 +1369,6 @@ def main():
paddle_binary_dir
)
)
(
setup_requires,
packages,
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或