提交 71bc8e99 编写于 作者: 5991120's avatar 5991120

modify setup.py and paddle_build.sh

上级 18c0a002
...@@ -3476,13 +3476,16 @@ function check_coverage_build() { ...@@ -3476,13 +3476,16 @@ function check_coverage_build() {
set -x set -x
} }
function run_setup(){ function run_setup(){
startTime_s=`date +%s`
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
# Build script will not fail if *.deb does not exist # Build script will not fail if *.deb does not exist
rm *.deb 2>/dev/null || true rm *.deb 2>/dev/null || true
# Delete previous built egg packages # Delete previous built egg packages
rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true
# Delete previous built paddle cache # Delete previous built paddle cache
rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true
startTime_s=`date +%s`
SYSTEM=`uname -s` SYSTEM=`uname -s`
if [ "$SYSTEM" == "Darwin" ]; then if [ "$SYSTEM" == "Darwin" ]; then
echo "Using python abi: $1" echo "Using python abi: $1"
...@@ -3640,7 +3643,7 @@ function run_setup(){ ...@@ -3640,7 +3643,7 @@ function run_setup(){
export WITH_MLU=${WITH_MLU:-OFF} export WITH_MLU=${WITH_MLU:-OFF}
export WITH_IPU=${WITH_IPU:-OFF} export WITH_IPU=${WITH_IPU:-OFF}
export WITH_CNCL=${WITH_CNCL:-OFF} export WITH_CNCL=${WITH_CNCL:-OFF}
export XPU_SDK_ROOT=${XPU_SDK_ROOT:-} export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""}
export WITH_LITE=${WITH_LITE:-OFF} export WITH_LITE=${WITH_LITE:-OFF}
export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF} export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF}
export WITH_ARM=${WITH_ARM:-OFF} export WITH_ARM=${WITH_ARM:-OFF}
...@@ -3657,12 +3660,28 @@ function run_setup(){ ...@@ -3657,12 +3660,28 @@ function run_setup(){
export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF} export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF}
export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF} export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF}
if [ "$SYSTEM" == "Linux" ];then
if [ `nproc` -gt 16 ];then
parallel_number=$(expr `nproc` - 8)
else
parallel_number=`nproc`
fi
else
parallel_number=8
fi
if [ "$2" != "" ]; then
parallel_number=$2
fi
export MAX_JOBS=${parallel_number}
# reset ccache zero stats for collect PR's actual hit rate # reset ccache zero stats for collect PR's actual hit rate
ccache -z ccache -z
cd ..
if [ "${PYTHON_EXECUTABLE}" != "" ];then if [ "${PYTHON_EXECUTABLE}" != "" ];then
${PYTHON_EXECUTABLE} setup.py $2;build_error=$? ${PYTHON_EXECUTABLE} setup.py $3;build_error=$?
else else
python setup.py $2;build_error=$? python setup.py $3;build_error=$?
fi fi
# ci will collect ccache hit rate # ci will collect ccache hit rate
...@@ -3672,14 +3691,19 @@ function run_setup(){ ...@@ -3672,14 +3691,19 @@ function run_setup(){
exit 7; exit 7;
fi fi
endTime_s=`date +%s`
[ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
} }
function main() { function main() {
local CMD=$1 local CMD=$1
local parallel_number=$2 local parallel_number=$2
init init
case $CMD in case $CMD in
build_only) build_only)
run_setup ${PYTHON_ABI:-""} bdist_wheel run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel
;; ;;
build_pr_dev) build_pr_dev)
build_pr_and_develop build_pr_and_develop
...@@ -3796,7 +3820,7 @@ function main() { ...@@ -3796,7 +3820,7 @@ function main() {
;; ;;
cicheck_coverage) cicheck_coverage)
check_diff_file_for_coverage check_diff_file_for_coverage
run_setup ${PYTHON_ABI:-""} install run_setup ${PYTHON_ABI:-""} ${parallel_number} install
enable_unused_var_check enable_unused_var_check
parallel_test parallel_test
check_coverage check_coverage
...@@ -3883,7 +3907,7 @@ function main() { ...@@ -3883,7 +3907,7 @@ function main() {
build_mac build_mac
;; ;;
cicheck_py37) cicheck_py37)
run_setup ${PYTHON_ABI:-""} bdist_wheel run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel
run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1} run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
;; ;;
test_cicheck_py37) test_cicheck_py37)
...@@ -3896,7 +3920,7 @@ function main() { ...@@ -3896,7 +3920,7 @@ function main() {
parallel_test parallel_test
;; ;;
build_gpubox) build_gpubox)
run_setup ${PYTHON_ABI:-""} install run_setup ${PYTHON_ABI:-""} ${parallel_number} install
;; ;;
check_xpu) check_xpu)
cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
......
...@@ -21,6 +21,7 @@ import platform ...@@ -21,6 +21,7 @@ import platform
import re import re
import shutil import shutil
import subprocess import subprocess
import sysconfig
import sys import sys
from contextlib import contextmanager from contextlib import contextmanager
from distutils.spawn import find_executable from distutils.spawn import find_executable
...@@ -49,6 +50,21 @@ assert ( ...@@ -49,6 +50,21 @@ assert (
CMAKE CMAKE
), 'The "cmake" executable is not found. Please check if Cmake is installed.' ), 'The "cmake" executable is not found. Please check if Cmake is installed.'
#CMAKE: full path to python library
if (platform.system() == "Windows"):
cmake_python_library = "{}/libs/python{}.lib".format(
sysconfig.get_config_var("prefix"),
sysconfig.get_config_var("VERSION"))
# Fix virtualenv builds
if not os.path.exists(cmake_python_library):
cmake_python_library = "{}/libs/python{}.lib".format(
sys.base_prefix,
sysconfig.get_config_var("VERSION"))
else:
cmake_python_library = "{}/{}".format(
sysconfig.get_config_var("LIBDIR"),
sysconfig.get_config_var("INSTSONAME"))
TOP_DIR = os.path.dirname(os.path.realpath(__file__)) TOP_DIR = os.path.dirname(os.path.realpath(__file__))
IS_WINDOWS = os.name == 'nt' IS_WINDOWS = os.name == 'nt'
...@@ -555,6 +571,13 @@ def options_process(args, build_options): ...@@ -555,6 +571,13 @@ def options_process(args, build_options):
for key, value in sorted(build_options.items()): for key, value in sorted(build_options.items()):
if value is not None: if value is not None:
args.append("-D{}={}".format(key, value)) args.append("-D{}={}".format(key, value))
if ('PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys()):
args.append("-D{}={}".format('PYTHON_EXECUTABLE',sys.executable))
if ('PYTHON_INCLUDE_DIR:PATH' not in build_options.keys()):
args.append('-D{}={}'.format('PYTHON_INCLUDE_DIR',sysconfig.get_path("include")))
if ('PYTHON_LIBRARY:FILEPATH' not in build_options.keys()):
args.append('-D{}={}'.format('PYTHON_LIBRARY',cmake_python_library))
def get_cmake_generator(): def get_cmake_generator():
...@@ -566,7 +589,61 @@ def get_cmake_generator(): ...@@ -566,7 +589,61 @@ def get_cmake_generator():
return cmake_generator return cmake_generator
def cmake_run(args, build_path): def cmake_run(build_path):
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
# "PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
# "PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
# "PYTHON_INCLUDE_DIR",
# "PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
with cd(build_path): with cd(build_path):
cmake_args = [] cmake_args = []
cmake_args.append(CMAKE) cmake_args.append(CMAKE)
...@@ -623,72 +700,12 @@ def build_steps(): ...@@ -623,72 +700,12 @@ def build_steps():
os.remove(cmake_cache_file_path) os.remove(cmake_cache_file_path)
CMAKE_GENERATOR = get_cmake_generator() CMAKE_GENERATOR = get_cmake_generator()
bool_ninja = (CMAKE_GENERATOR == "Ninja")
if CMAKE_GENERATOR == "Ninja":
build_ninja_file_path = os.path.join(build_path, "build.ninja") build_ninja_file_path = os.path.join(build_path, "build.ninja")
# if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)): if os.path.exists(cmake_cache_file_path) and not (bool_ninja and not os.path.exists(build_ninja_file_path)):
if os.path.exists(cmake_cache_file_path) and os.path.exists(
build_ninja_file_path
):
print("Do not need rerun camke,everything is ready,run build now") print("Do not need rerun camke,everything is ready,run build now")
run_cmake_build(build_path)
return
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
else: else:
key = other_options[option_key] cmake_run(build_path)
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
cmake_run(args, build_path)
# make # make
if only_cmake: if only_cmake:
print( print(
...@@ -1342,7 +1359,6 @@ def main(): ...@@ -1342,7 +1359,6 @@ def main():
paddle_binary_dir paddle_binary_dir
) )
) )
( (
setup_requires, setup_requires,
packages, packages,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册