modify setup.py and paddle_build.sh

paddle/scripts/paddle_build.sh

@@ -3476,13 +3476,16 @@ function check_coverage_build() {
     set -x
 }
 function run_setup(){
+    startTime_s=`date +%s`
+    mkdir -p ${PADDLE_ROOT}/build
+    cd ${PADDLE_ROOT}/build
     # Build script will not fail if *.deb does not exist
     rm *.deb 2>/dev/null || true
     # Delete previous built egg packages
     rm -rf ${PADDLE_ROOT}/dist 2>/dev/null || true
     # Delete previous built paddle cache
     rm -rf ${PADDLE_ROOT}/build/python/paddle 2>/dev/null || true
-    startTime_s=`date +%s`
+
     SYSTEM=`uname -s`
     if [ "$SYSTEM" == "Darwin" ]; then
         echo "Using python abi: $1"
@@ -3640,7 +3643,7 @@ function run_setup(){
     export WITH_MLU=${WITH_MLU:-OFF}
     export WITH_IPU=${WITH_IPU:-OFF}
     export WITH_CNCL=${WITH_CNCL:-OFF}
-    export XPU_SDK_ROOT=${XPU_SDK_ROOT:-}
+    export XPU_SDK_ROOT=${XPU_SDK_ROOT:-""}
     export WITH_LITE=${WITH_LITE:-OFF}
     export WITH_XPU_BKCL=${WITH_XPU_BKCL:-OFF}
     export WITH_ARM=${WITH_ARM:-OFF}
@@ -3657,12 +3660,28 @@ function run_setup(){
     export WITH_ONNXRUNTIME=${WITH_ONNXRUNTIME:-OFF}
     export WITH_CUDNN_FRONTEND=${WITH_CUDNN_FRONTEND:-OFF}
 
+    if [ "$SYSTEM" == "Linux" ];then
+      if [ `nproc` -gt 16 ];then
+          parallel_number=$(expr `nproc` - 8)
+      else
+          parallel_number=`nproc`
+      fi
+    else
+      parallel_number=8
+    fi
+    if [ "$2" != "" ]; then
+      parallel_number=$2
+    fi
+    export MAX_JOBS=${parallel_number}
+
     # reset ccache zero stats for collect PR's actual hit rate
+
     ccache -z
+    cd ..
     if [ "${PYTHON_EXECUTABLE}" != "" ];then
-        ${PYTHON_EXECUTABLE} setup.py $2;build_error=$?
+        ${PYTHON_EXECUTABLE} setup.py $3;build_error=$?
     else
-        python setup.py $2;build_error=$?
+        python setup.py $3;build_error=$?
     fi
     
     # ci will collect ccache hit rate
@@ -3672,14 +3691,19 @@ function run_setup(){
         exit 7;
     fi
 
+    endTime_s=`date +%s`
+    [ -n "$startTime_firstBuild" ] && startTime_s=$startTime_firstBuild
+    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
+    echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
 }
+
 function main() {
     local CMD=$1
     local parallel_number=$2
     init
     case $CMD in
       build_only)
-        run_setup ${PYTHON_ABI:-""} bdist_wheel
+        run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel
         ;;
       build_pr_dev)
         build_pr_and_develop
@@ -3796,7 +3820,7 @@ function main() {
         ;;
       cicheck_coverage)
         check_diff_file_for_coverage
-        run_setup ${PYTHON_ABI:-""} install
+        run_setup ${PYTHON_ABI:-""} ${parallel_number} install
         enable_unused_var_check
         parallel_test
         check_coverage
@@ -3883,7 +3907,7 @@ function main() {
         build_mac
         ;;
       cicheck_py37)
-        run_setup ${PYTHON_ABI:-""} bdist_wheel
+        run_setup ${PYTHON_ABI:-""} ${parallel_number} bdist_wheel
         run_linux_cpu_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
         ;;
       test_cicheck_py37)
@@ -3896,7 +3920,7 @@ function main() {
         parallel_test
         ;;
       build_gpubox)
-        run_setup ${PYTHON_ABI:-""} install
+        run_setup ${PYTHON_ABI:-""} ${parallel_number} install
         ;;
       check_xpu)
         cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}

setup.py

@@ -21,6 +21,7 @@ import platform
 import re
 import shutil
 import subprocess
+import sysconfig
 import sys
 from contextlib import contextmanager
 from distutils.spawn import find_executable
@@ -49,6 +50,21 @@ assert (
     CMAKE
 ), 'The "cmake" executable is not found. Please check if Cmake is installed.'
 
+#CMAKE: full path to python library
+if (platform.system() == "Windows"):
+    cmake_python_library = "{}/libs/python{}.lib".format(
+        sysconfig.get_config_var("prefix"),
+        sysconfig.get_config_var("VERSION"))
+    # Fix virtualenv builds
+    if not os.path.exists(cmake_python_library):
+        cmake_python_library = "{}/libs/python{}.lib".format(
+            sys.base_prefix,
+            sysconfig.get_config_var("VERSION"))
+else:
+    cmake_python_library = "{}/{}".format(
+        sysconfig.get_config_var("LIBDIR"),
+        sysconfig.get_config_var("INSTSONAME"))
+
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
 
 IS_WINDOWS = os.name == 'nt'
@@ -555,7 +571,14 @@ def options_process(args, build_options):
     for key, value in sorted(build_options.items()):
         if value is not None:
             args.append("-D{}={}".format(key, value))
-
+    if ('PYTHON_EXECUTABLE:FILEPATH' not in build_options.keys()):
+        args.append("-D{}={}".format('PYTHON_EXECUTABLE',sys.executable))
+    if ('PYTHON_INCLUDE_DIR:PATH' not in build_options.keys()):
+        args.append('-D{}={}'.format('PYTHON_INCLUDE_DIR',sysconfig.get_path("include")))
+    if ('PYTHON_LIBRARY:FILEPATH' not in build_options.keys()):
+        args.append('-D{}={}'.format('PYTHON_LIBRARY',cmake_python_library))
+        
+        
 
 def get_cmake_generator():
     if os.getenv("CMAKE_GENERATOR"):
@@ -566,7 +589,61 @@ def get_cmake_generator():
     return cmake_generator
 
 
-def cmake_run(args, build_path):
+def cmake_run(build_path):
+    args = []
+    env_var = os.environ.copy()  # get env variables
+    paddle_build_options = {}
+    other_options = {}
+    other_options.update(
+        {
+            option: option
+            for option in (
+                # "PYTHON_LIBRARY",
+                "INFERENCE_DEMO_INSTALL_DIR",
+                "ON_INFER",
+                # "PYTHON_EXECUTABLE",
+                "TENSORRT_ROOT",
+                "CUDA_ARCH_NAME",
+                "CUDA_ARCH_BIN",
+                # "PYTHON_INCLUDE_DIR",
+                # "PYTHON_LIBRARIES",
+                "PY_VERSION",
+                "CUB_PATH",
+                "NEW_RELEASE_PYPI",
+                "CUDNN_ROOT",
+                "THIRD_PARTY_PATH",
+                "NOAVX_CORE_FILE",
+                "LITE_GIT_TAG",
+                "CUDA_TOOLKIT_ROOT_DIR",
+                "NEW_RELEASE_JIT",
+                "XPU_SDK_ROOT",
+                "MSVC_STATIC_CRT",
+                "NEW_RELEASE_ALL",
+                "CMAKE_GENERATOR",
+            )
+        }
+    )
+    # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
+    for option_key, option_value in env_var.items():
+        if option_key.startswith(("CMAKE_", "WITH_")):
+            paddle_build_options[option_key] = option_value
+        if option_key in other_options:
+            if (
+                option_key == 'PYTHON_EXECUTABLE'
+                or option_key == 'PYTHON_LIBRARY'
+                or option_key == 'PYTHON_LIBRARIES'
+            ):
+                key = option_key + ":FILEPATH"
+                print(key)
+            elif option_key == 'PYTHON_INCLUDE_DIR':
+                key = key = option_key + ':PATH'
+                print(key)
+            else:
+                key = other_options[option_key]
+            if key not in paddle_build_options:
+                paddle_build_options[key] = option_value
+    options_process(args, paddle_build_options)
+    print("args:", args)
     with cd(build_path):
         cmake_args = []
         cmake_args.append(CMAKE)
@@ -621,74 +698,14 @@ def build_steps():
     # if rerun_cmake is True,remove CMakeCache.txt and rerun camke
     if os.path.isfile(cmake_cache_file_path) and rerun_cmake is True:
         os.remove(cmake_cache_file_path)
-
+        
     CMAKE_GENERATOR = get_cmake_generator()
-
-    if CMAKE_GENERATOR == "Ninja":
-        build_ninja_file_path = os.path.join(build_path, "build.ninja")
-        # if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)):
-        if os.path.exists(cmake_cache_file_path) and os.path.exists(
-            build_ninja_file_path
-        ):
-            print("Do not need rerun camke,everything is ready,run build now")
-            run_cmake_build(build_path)
-            return
-
-    args = []
-    env_var = os.environ.copy()  # get env variables
-    paddle_build_options = {}
-    other_options = {}
-    other_options.update(
-        {
-            option: option
-            for option in (
-                "PYTHON_LIBRARY",
-                "INFERENCE_DEMO_INSTALL_DIR",
-                "ON_INFER",
-                "PYTHON_EXECUTABLE",
-                "TENSORRT_ROOT",
-                "CUDA_ARCH_NAME",
-                "CUDA_ARCH_BIN",
-                "PYTHON_INCLUDE_DIR",
-                "PYTHON_LIBRARIES",
-                "PY_VERSION",
-                "CUB_PATH",
-                "NEW_RELEASE_PYPI",
-                "CUDNN_ROOT",
-                "THIRD_PARTY_PATH",
-                "NOAVX_CORE_FILE",
-                "LITE_GIT_TAG",
-                "CUDA_TOOLKIT_ROOT_DIR",
-                "NEW_RELEASE_JIT",
-                "XPU_SDK_ROOT",
-                "MSVC_STATIC_CRT",
-                "NEW_RELEASE_ALL",
-                "CMAKE_GENERATOR",
-            )
-        }
-    )
-    # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
-    for option_key, option_value in env_var.items():
-        if option_key.startswith(("CMAKE_", "WITH_")):
-            paddle_build_options[option_key] = option_value
-        if option_key in other_options:
-            if (
-                option_key == 'PYTHON_EXECUTABLE'
-                or option_key == 'PYTHON_LIBRARY'
-                or option_key == 'PYTHON_LIBRARIES'
-            ):
-                key = option_key + ":FILEPATH"
-                print(key)
-            elif option_key == 'PYTHON_INCLUDE_DIR':
-                key = key = option_key + ':PATH'
-                print(key)
-            else:
-                key = other_options[option_key]
-            if key not in paddle_build_options:
-                paddle_build_options[key] = option_value
-    options_process(args, paddle_build_options)
-    print("args:", args)
-    cmake_run(args, build_path)
+    bool_ninja = (CMAKE_GENERATOR == "Ninja")
+    build_ninja_file_path = os.path.join(build_path, "build.ninja")
+    if os.path.exists(cmake_cache_file_path) and not (bool_ninja and not os.path.exists(build_ninja_file_path)):
+        print("Do not need rerun camke,everything is ready,run build now")
+    else:
+        cmake_run(build_path)
     # make
     if only_cmake:
         print(
@@ -1342,7 +1359,6 @@ def main():
             paddle_binary_dir
         )
     )
-
     (
         setup_requires,
         packages,