diff --git a/scripts/cmake-build/cross_build_android_arm_inference.sh b/scripts/cmake-build/cross_build_android_arm_inference.sh index 1f5236e91e25b5604a396bae1bd2e4d30470ef51..9dee931c6cdd3265818e02b6ffab4a7a0ed47c4f 100755 --- a/scripts/cmake-build/cross_build_android_arm_inference.sh +++ b/scripts/cmake-build/cross_build_android_arm_inference.sh @@ -10,6 +10,16 @@ REMOVE_OLD_BUILD=false NINJA_VERBOSE=OFF NINJA_DRY_RUN=OFF SPECIFIED_TARGET="install/strip" +READLINK=readlink +OS=$(uname -s) + +if [ $OS = "Darwin" ];then + READLINK=greadlink +fi + +SRC_DIR=$($READLINK -f "`dirname $0`/../../") +source $SRC_DIR/scripts/cmake-build/utils/utils.sh +config_ninja_default_max_jobs echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" @@ -23,16 +33,27 @@ function usage() { echo "-r : remove old build dir before make, default off" echo "-v : ninja with verbose and explain, default off" echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" + echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2" echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)" + echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS" echo "-h : show usage" - echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." - echo "example: $0 -d" + echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:" + echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d" exit -1 } -while getopts "nvrkhdfa:e:" arg +while getopts "lnvrkhdfa:e:j:" arg do case $arg in + j) + NINJA_MAX_JOBS=$OPTARG + echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}" + ;; + l) + echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS" + show_cmakelist_options + exit 0 + ;; d) echo "Build with Debug mode" BUILD_TYPE=Debug @@ -95,18 +116,10 @@ echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}" echo "ARCH: $ARCH" echo "----------------------------------------------------" -READLINK=readlink -OS=$(uname -s) - -if [ $OS = "Darwin" ];then - READLINK=greadlink -elif [[ $OS =~ "NT" ]]; then +if [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." fi -SRC_DIR=$($READLINK -f "`dirname $0`/../../") -source $SRC_DIR/scripts/cmake-build/utils/utils.sh - if [ -z $NDK_ROOT ];then echo "can not find NDK_ROOT env, pls export you NDK root dir to NDK_ROOT" exit -1 @@ -147,7 +160,7 @@ function cmake_build() { ${EXTRA_CMAKE_ARGS} \ $SRC_DIR " - config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS} bash -c "${NINJA_CMD}" } diff --git a/scripts/cmake-build/cross_build_ios_arm_inference.sh b/scripts/cmake-build/cross_build_ios_arm_inference.sh index d4a86571d110499cba8389344e5a68b1dce90fdb..d70a928f8dafe8938957412184b489e4a9996a16 100755 --- a/scripts/cmake-build/cross_build_ios_arm_inference.sh +++ b/scripts/cmake-build/cross_build_ios_arm_inference.sh @@ -10,6 +10,16 @@ REMOVE_OLD_BUILD=false NINJA_VERBOSE=OFF NINJA_DRY_RUN=OFF SPECIFIED_TARGET="install" +READLINK=readlink +OS=$(uname -s) + +if [ $OS = "Darwin" ];then + READLINK=greadlink +fi + +SRC_DIR=$($READLINK -f "`dirname $0`/../../") +source $SRC_DIR/scripts/cmake-build/utils/utils.sh +config_ninja_default_max_jobs echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" @@ -23,16 +33,27 @@ function usage() { echo "-r : remove old build dir before make, default off" echo "-v : ninja with verbose and explain, default off" echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" + echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2" echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)" + echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS" echo "-h : show usage" - echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." - echo "example: $0 -d" + echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:" + echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d" exit -1 } -while getopts "nvrkhdfa:e:" arg +while getopts "lnvrkhdfa:e:j:" arg do case $arg in + j) + NINJA_MAX_JOBS=$OPTARG + echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}" + ;; + l) + echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS" + show_cmakelist_options + exit 0 + ;; d) echo "Build with Debug mode" BUILD_TYPE=Debug @@ -92,22 +113,17 @@ echo "BUILD_TYPE: $BUILD_TYPE" echo "MGE_ARMV8_2_FEATURE_FP16: $MGE_ARMV8_2_FEATURE_FP16" echo "MGE_DISABLE_FLOAT16: $MGE_DISABLE_FLOAT16" echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}" +echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}" echo "ARCH: $ARCH" echo "----------------------------------------------------" -READLINK=readlink -OS=$(uname -s) - if [ $OS = "Darwin" ];then - READLINK=greadlink + echo "cross build ios from MacOS" else - echo "cross build ios only support on macos, abort now!!" + echo "cross build ios only support on MacOS, abort now!!" exit -1 fi -SRC_DIR=$($READLINK -f "`dirname $0`/../../") -source $SRC_DIR/scripts/cmake-build/utils/utils.sh - function cmake_build() { BUILD_DIR=$SRC_DIR/build_dir/apple/$3/$4/$1/$BUILD_TYPE/build INSTALL_DIR=$BUILD_DIR/../install @@ -145,7 +161,7 @@ function cmake_build() { ${EXTRA_CMAKE_ARGS} \ $SRC_DIR " - config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS} bash -c "${NINJA_CMD}" } diff --git a/scripts/cmake-build/cross_build_linux_arm_inference.sh b/scripts/cmake-build/cross_build_linux_arm_inference.sh index 7deca2438e1243316023487a6f8ccf1c5e07d92f..b0832df9fea8b7c5bd54fd8e5575e1a3e71aa9f2 100755 --- a/scripts/cmake-build/cross_build_linux_arm_inference.sh +++ b/scripts/cmake-build/cross_build_linux_arm_inference.sh @@ -13,6 +13,16 @@ NINJA_DRY_RUN=OFF SPECIFIED_TARGET="install/strip" CMAKE_C_FLAGS="-Wno-psabi" CMAKE_CXX_FLAGS="-Wno-psabi" +READLINK=readlink +OS=$(uname -s) + +if [ $OS = "Darwin" ];then + READLINK=greadlink +fi + +SRC_DIR=$($READLINK -f "`dirname $0`/../../") +source $SRC_DIR/scripts/cmake-build/utils/utils.sh +config_ninja_default_max_jobs echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" @@ -27,16 +37,27 @@ function usage() { echo "-r : remove old build dir before make, default off" echo "-v : ninja with verbose and explain, default off" echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" + echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2" echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)" + echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS" echo "-h : show usage" - echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." - echo "example: $0 -d" + echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:" + echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d" exit -1 } -while getopts "nvrkhdcfa:e:" arg +while getopts "lnvrkhdcfa:e:j:" arg do case $arg in + j) + NINJA_MAX_JOBS=$OPTARG + echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}" + ;; + l) + echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS" + show_cmakelist_options + exit 0 + ;; d) echo "Build with Debug mode" BUILD_TYPE=Debug @@ -101,15 +122,11 @@ echo "MGE_WITH_CUDA: $MGE_WITH_CUDA" echo "MGE_ARMV8_2_FEATURE_FP16: $MGE_ARMV8_2_FEATURE_FP16" echo "MGE_DISABLE_FLOAT16: $MGE_DISABLE_FLOAT16" echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}" +echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}" echo "ARCH: $ARCH" echo "----------------------------------------------------" -READLINK=readlink -OS=$(uname -s) - -if [ $OS = "Darwin" ];then - READLINK=greadlink -elif [[ $OS =~ "NT" ]]; then +if [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." fi @@ -129,9 +146,6 @@ if [ $MGE_WITH_CUDA = "OFF" ];then CMAKE_CXX_FLAGS="-Werror=unused-parameter -Wno-psabi" fi -SRC_DIR=$($READLINK -f "`dirname $0`/../../") -source $SRC_DIR/scripts/cmake-build/utils/utils.sh - function cmake_build() { BUILD_DIR=$SRC_DIR/build_dir/gnu-linux/MGE_WITH_CUDA_$3/$1/$BUILD_TYPE/build INSTALL_DIR=$BUILD_DIR/../install @@ -161,7 +175,7 @@ function cmake_build() { ${EXTRA_CMAKE_ARGS} \ $SRC_DIR " - config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS} bash -c "${NINJA_CMD}" } diff --git a/scripts/cmake-build/host_build.sh b/scripts/cmake-build/host_build.sh index f63d8c695001ddf8b22f2bdc8ad91a3a765f0e47..99664631d0a2a481724a92f2b6b86801fd9e7852 100755 --- a/scripts/cmake-build/host_build.sh +++ b/scripts/cmake-build/host_build.sh @@ -1,26 +1,6 @@ #!/usr/bin/env bash set -e -function usage() { - echo "$0 args1 args2 .." - echo "available args detail:" - echo "-d : Build with Debug mode, default Release mode" - echo "-c : Build with CUDA, default without CUDA" - echo "-t : Build with training mode, default inference only" - echo "-m : Build with m32 mode(only for windows build), default m64" - echo "-r : remove old build dir before make, default off" - echo "-v : ninja with verbose and explain, default off" - echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel" - echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" - echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)" - echo "-h : show usage" - echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." - echo "example: $0 -d" - exit -1 -} - -READLINK=readlink -OS=$(uname -s) BUILD_TYPE=Release MGE_WITH_CUDA=OFF MGE_INFERENCE_ONLY=ON @@ -32,16 +12,56 @@ NINJA_VERBOSE=OFF BUILD_DEVELOP=ON NINJA_DRY_RUN=OFF SPECIFIED_TARGET="install/strip" +READLINK=readlink + +OS=$(uname -s) if [[ $OS =~ "NT" ]]; then echo "Windows no need strip, caused by pdb file always split with exe" SPECIFIED_TARGET="install" fi +if [ $OS = "Darwin" ];then + READLINK=greadlink +fi + +SRC_DIR=$($READLINK -f "`dirname $0`/../../") +source $SRC_DIR/scripts/cmake-build/utils/utils.sh +config_ninja_default_max_jobs + echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" -while getopts "nsrhdctmve:" arg +function usage() { + echo "$0 args1 args2 .." + echo "available args detail:" + echo "-d : Build with Debug mode, default Release mode" + echo "-c : Build with CUDA, default without CUDA" + echo "-t : Build with training mode, default inference only" + echo "-m : Build with m32 mode(only for windows build), default m64" + echo "-r : remove old build dir before make, default off" + echo "-v : ninja with verbose and explain, default off" + echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel" + echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" + echo "-j : run N jobs in parallel for ninja, defaut is cpu_number + 2" + echo "-e : build a specified target (always for debug, NOTICE: do not do strip/install target when use -e)" + echo "-l : list CMakeLists.txt all options, can be use to config EXTRA_CMAKE_ARGS" + echo "-h : show usage" + echo "append other cmake config by config EXTRA_CMAKE_ARGS, for example, enable MGE_WITH_TEST and build with Debug mode:" + echo "EXTRA_CMAKE_ARGS=\"-DMGE_WITH_TEST=ON\" $0 -d" + exit -1 +} + +while getopts "lnsrhdctmve:j:" arg do case $arg in + j) + NINJA_MAX_JOBS=$OPTARG + echo "config NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}" + ;; + l) + echo "list CMakeLists.txt all options, can be used to config EXTRA_CMAKE_ARGS" + show_cmakelist_options + exit 0 + ;; d) echo "Build with Debug mode" BUILD_TYPE=Debug @@ -95,10 +115,10 @@ echo "BUILD_TYPE: $BUILD_TYPE" echo "MGE_WITH_CUDA: $MGE_WITH_CUDA" echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY" echo "SPECIFIED_TARGET: ${SPECIFIED_TARGET}" +echo "NINJA_MAX_JOBS: ${NINJA_MAX_JOBS}" echo "------------------------------------" if [ $OS = "Darwin" ];then - READLINK=greadlink if [ $MGE_WITH_CUDA = "ON" ];then echo "MACOS DO NOT SUPPORT TensorRT, ABORT NOW!!" exit -1 @@ -107,9 +127,6 @@ elif [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." fi -SRC_DIR=$($READLINK -f "`dirname $0`/../../") -source $SRC_DIR/scripts/cmake-build/utils/utils.sh - if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON" BUILD_DEVELOP=OFF @@ -141,7 +158,7 @@ function cmake_build() { ${EXTRA_CMAKE_ARGS} \ ${SRC_DIR} " - config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} + config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS} bash -c "${NINJA_CMD}" } @@ -280,7 +297,7 @@ function cmake_build_windows() { -DCMAKE_MAKE_PROGRAM=ninja.exe \ ${EXTRA_CMAKE_ARGS} ../../.. " - config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} + config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "${SPECIFIED_TARGET}" ${NINJA_DRY_RUN} ${NINJA_MAX_JOBS} cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} " } diff --git a/scripts/cmake-build/utils/utils.sh b/scripts/cmake-build/utils/utils.sh index 40d2b0a8c6db594706418eee4cd4a9389319dac5..987d56c6261fea411ddf14bb3c145dafef35bc72 100755 --- a/scripts/cmake-build/utils/utils.sh +++ b/scripts/cmake-build/utils/utils.sh @@ -4,6 +4,8 @@ set -e OS=$(uname -s) NINJA_CMD="" NINJA_BASE="ninja" +cpu_number=`nproc` +NINJA_MAX_JOBS=0 if [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." @@ -15,7 +17,7 @@ if [ $OS = "Darwin" ];then READLINK=greadlink fi -PROJECT_DIR=$(dirname "${BASH_SOURCE[0]}")/../../../ +PROJECT_DIR=$(dirname "${BASH_SOURCE[0]}")/../../.. function cd_real_build_dir() { REAL_DIR=$($READLINK -f $1) echo "may alias dir: $1" @@ -77,21 +79,22 @@ function try_remove_old_build() { } function config_ninja_target_cmd() { - if [ $# -eq 4 ]; then + if [ $# -eq 5 ]; then _NINJA_VERBOSE=$1 _BUILD_DEVELOP=$2 _NINJA_TARGET=$3 _NINJA_DRY_RUN=$4 + _NINJA_MAX_JOBS=$5 else echo "err call config_ninja_target_cmd" exit -1 fi if [ -z "${_NINJA_TARGET}" ]; then - NINJA_CMD="${NINJA_BASE} all" + NINJA_CMD="${NINJA_BASE} all -j ${_NINJA_MAX_JOBS}" elif [[ ${_NINJA_TARGET} =~ "install" ]]; then - NINJA_CMD="${NINJA_BASE} all && ${NINJA_BASE} ${_NINJA_TARGET}" + NINJA_CMD="${NINJA_BASE} all -j ${_NINJA_MAX_JOBS} && ${NINJA_BASE} ${_NINJA_TARGET}" else - NINJA_CMD="${NINJA_BASE} ${_NINJA_TARGET}" + NINJA_CMD="${NINJA_BASE} ${_NINJA_TARGET} -j ${_NINJA_MAX_JOBS}" fi if [ ${_NINJA_DRY_RUN} = "ON" ]; then @@ -108,3 +111,14 @@ function config_ninja_target_cmd() { echo "build ${NINJA_BASE} target command: ${NINJA_CMD}" } + +function show_cmakelist_options() { + cd ${PROJECT_DIR} + grep "option(" lite/CMakeLists.txt CMakeLists.txt --color + cd - > /dev/null +} + +function config_ninja_default_max_jobs() { + ((NINJA_MAX_JOBS = ${cpu_number} + 2)) + echo "config default NINJA_MAX_JOBS to ${NINJA_MAX_JOBS}" +}