diff --git a/CMakeLists.txt b/CMakeLists.txt index 7574e1a091a21ddc932ca4a70abc661ac5dd23ae..061ae831af0e95f44c4169f0df99d2b01a44b689 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,8 @@ cmake_minimum_required(VERSION 3.15.2) +message(STATUS "CMAKE_GENERATOR: ${CMAKE_GENERATOR}" ) +if (NOT ${CMAKE_GENERATOR} STREQUAL "Ninja") + message(WARNING "CMAKE_GENERATOR NOT EQUAL Ninja, which we do not recommend") +endif() include (cmake/FetchMegBrainVersion.cmake) project(MegEngine LANGUAGES C CXX VERSION ${MGB_VER_STRING}) diff --git a/scripts/cmake-build/BUILD_README.md b/scripts/cmake-build/BUILD_README.md index c3e1ddba4c728128a555b07b488cc6ccfe0cf3b6..ba1d83adc754f7eab4ff60a31b8f2e586f6cfe33 100755 --- a/scripts/cmake-build/BUILD_README.md +++ b/scripts/cmake-build/BUILD_README.md @@ -19,7 +19,7 @@ * commands: ``` 1: installl Visual Studio (need support LLVM/clang-cl), eg 2019. Please install LLVM-10, VS LLVM linker have issue, please replace lld-link.exe, which can be download from https://releases.llvm.org/download.html#10.0.0 -2: install extension of VS: Python/Cmake/LLVM +2: install extension of VS: Python/Cmake/LLVM/Ninja 3: now we support cuda10.1+cudnn7.6+TensorRT6.0 on Windows, as Windows can only use DLL in fact with cudnn/TensorRT, so please install the same version; 3a: install cuda10.1 to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1 3b: install cudnn7.6 to C:\Program Files\NVIDIA GPU Computing Toolkit\cudnn-10.1-windows10-x64-v7.6.5.32 @@ -33,9 +33,9 @@ ### Linux host build * commands: ``` -1: install Cmake, which version >= 3.15.2 +1: install Cmake, which version >= 3.15.2, ninja-build 2: install gcc/g++, which version >= 6, (gcc/g++ >= 7, if need build training mode) -3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev swig python3-numpy texinfo +3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev python3-numpy texinfo 4: CUDA env(if enable CUDA), version detail refer to README.md ``` @@ -44,7 +44,7 @@ ``` 1: install Cmake, which version >= 3.15.2 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" -3: brew install python python3 swig coreutils +3: brew install python python3 coreutils ninja 4: install at least xcode command line tool: https://developer.apple.com/xcode/ 5: about cuda: we do not support CUDA on MacOS 6: python3 -m pip install numpy (if you want to build with training mode) diff --git a/scripts/cmake-build/cross_build_android_arm_inference.sh b/scripts/cmake-build/cross_build_android_arm_inference.sh index 04c1763997027d39ad54a1dfa1abcdca6b71d295..d3825a797f8765f7a9e86b9a84ae179f7635c5d5 100755 --- a/scripts/cmake-build/cross_build_android_arm_inference.sh +++ b/scripts/cmake-build/cross_build_android_arm_inference.sh @@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF MGE_DISABLE_FLOAT16=OFF ARCH=arm64-v8a REMOVE_OLD_BUILD=false +NINJA_VERBOSE=OFF +NINJA_DRY_RUN=OFF + echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" function usage() { @@ -17,13 +20,15 @@ function usage() { echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-a : config build arch available: ${ARCHS[@]}" echo "-r : remove old build dir before make, default off" + echo "-v : ninja with verbose and explain, default off" + echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" echo "-h : show usage" echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "example: $0 -d" exit -1 } -while getopts "rkhdfa:" arg +while getopts "nvrkhdfa:" arg do case $arg in d) @@ -62,6 +67,14 @@ do echo "config REMOVE_OLD_BUILD=true" REMOVE_OLD_BUILD=true ;; + v) + echo "config NINJA_VERBOSE=ON" + NINJA_VERBOSE=ON + ;; + n) + echo "config NINJA_DRY_RUN=ON" + NINJA_DRY_RUN=ON + ;; ?) echo "unkonw argument" usage @@ -77,14 +90,12 @@ echo "ARCH: $ARCH" echo "----------------------------------------------------" READLINK=readlink -MAKEFILE_TYPE="Unix" OS=$(uname -s) if [ $OS = "Darwin" ];then READLINK=greadlink elif [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." - MAKEFILE_TYPE="Unix" fi SRC_DIR=$($READLINK -f "`dirname $0`/../../") @@ -105,7 +116,6 @@ function cmake_build() { echo "build type: $BUILD_TYPE" echo "build ABI: $BUILD_ABI" echo "build native level: $BUILD_NATIVE_LEVEL" - echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE" try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR echo "create build dir" @@ -113,7 +123,7 @@ function cmake_build() { mkdir -p $INSTALL_DIR cd_real_build_dir $BUILD_DIR unset IFS - cmake -G "$MAKEFILE_TYPE Makefiles" \ + bash -c "cmake -G Ninja \ -DCMAKE_TOOLCHAIN_FILE="$NDK_ROOT/build/cmake/android.toolchain.cmake" \ -DANDROID_NDK="$NDK_ROOT" \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ @@ -125,10 +135,11 @@ function cmake_build() { -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ ${EXTRA_CMAKE_ARGS} \ - $SRC_DIR + $SRC_DIR " - make -j$(nproc) ${Target} - make install/strip + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "" ${NINJA_DRY_RUN} + bash -c "${NINJA_CMD}" + ${NINJA_BASE} install/strip } build_flatc $SRC_DIR $REMOVE_OLD_BUILD diff --git a/scripts/cmake-build/cross_build_ios_arm_inference.sh b/scripts/cmake-build/cross_build_ios_arm_inference.sh index 290e7673c1d5fb5fc043aa52df0a053bfe265509..55c3e75a07ce2ee2394f93cd2c81892d81384435 100755 --- a/scripts/cmake-build/cross_build_ios_arm_inference.sh +++ b/scripts/cmake-build/cross_build_ios_arm_inference.sh @@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF MGE_DISABLE_FLOAT16=OFF ARCH=arm64 REMOVE_OLD_BUILD=false +NINJA_VERBOSE=OFF +NINJA_DRY_RUN=OFF + echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" function usage() { @@ -17,13 +20,15 @@ function usage() { echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-a : config build arch available: ${ARCHS[@]}" echo "-r : remove old build dir before make, default off" + echo "-v : ninja with verbose and explain, default off" + echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" echo "-h : show usage" echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "example: $0 -d" exit -1 } -while getopts "rkhdfa:" arg +while getopts "nvrkhdfa:" arg do case $arg in d) @@ -62,6 +67,14 @@ do echo "config REMOVE_OLD_BUILD=true" REMOVE_OLD_BUILD=true ;; + v) + echo "config NINJA_VERBOSE=ON" + NINJA_VERBOSE=ON + ;; + n) + echo "config NINJA_DRY_RUN=ON" + NINJA_DRY_RUN=ON + ;; ?) echo "unkonw argument" usage @@ -109,7 +122,8 @@ function cmake_build() { mkdir -p $BUILD_DIR mkdir -p $INSTALL_DIR cd_real_build_dir $BUILD_DIR - cmake -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \ + bash -c "cmake -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DIOS_TOOLCHAIN_ROOT=$TOOLCHAIN \ -DOS_PLATFORM=$OS_PLATFORM \ @@ -121,11 +135,12 @@ function cmake_build() { -DMGE_ARMV8_2_FEATURE_FP16= $MGE_ARMV8_2_FEATURE_FP16 \ -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_MAKE_PROGRAM=ninja \ ${EXTRA_CMAKE_ARGS} \ - $SRC_DIR + $SRC_DIR " - make -j$(nproc) - make install + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install" ${NINJA_DRY_RUN} + bash -c "${NINJA_CMD}" } build_flatc $SRC_DIR $REMOVE_OLD_BUILD diff --git a/scripts/cmake-build/cross_build_linux_arm_inference.sh b/scripts/cmake-build/cross_build_linux_arm_inference.sh index ec1e4390bd0a67f0d6e3943ca84365e6a2486068..a5ce1be0cafe549178636162e77983f2d9c654d1 100755 --- a/scripts/cmake-build/cross_build_linux_arm_inference.sh +++ b/scripts/cmake-build/cross_build_linux_arm_inference.sh @@ -8,8 +8,11 @@ MGE_ARMV8_2_FEATURE_FP16=OFF MGE_DISABLE_FLOAT16=OFF ARCH=arm64-v8a REMOVE_OLD_BUILD=false +NINJA_VERBOSE=OFF +NINJA_DRY_RUN=OFF CMAKE_C_FLAGS="-Wno-psabi" CMAKE_CXX_FLAGS="-Wno-psabi" + echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" function usage() { @@ -21,13 +24,15 @@ function usage() { echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-a : config build arch available: ${ARCHS[@]}" echo "-r : remove old build dir before make, default off" + echo "-v : ninja with verbose and explain, default off" + echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" echo "-h : show usage" echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "example: $0 -d" exit -1 } -while getopts "rkhdcfa:" arg +while getopts "nvrkhdcfa:" arg do case $arg in d) @@ -70,6 +75,14 @@ do echo "config REMOVE_OLD_BUILD=true" REMOVE_OLD_BUILD=true ;; + v) + echo "config NINJA_VERBOSE=ON" + NINJA_VERBOSE=ON + ;; + n) + echo "config NINJA_DRY_RUN=ON" + NINJA_DRY_RUN=ON + ;; ?) echo "unkonw argument" usage @@ -86,14 +99,12 @@ echo "ARCH: $ARCH" echo "----------------------------------------------------" READLINK=readlink -MAKEFILE_TYPE="Unix" OS=$(uname -s) if [ $OS = "Darwin" ];then READLINK=greadlink elif [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." - MAKEFILE_TYPE="Unix" fi if [ ! $OS = "Linux" ] && [ $MGE_WITH_CUDA = "ON" ];then @@ -125,14 +136,13 @@ function cmake_build() { echo "build type: $BUILD_TYPE" echo "build toolchain: $TOOLCHAIN" echo "MGE_WITH_CUDA: $MGE_WITH_CUDA" - echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE" try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR echo "create build dir" mkdir -p $BUILD_DIR mkdir -p $INSTALL_DIR cd_real_build_dir $BUILD_DIR - cmake -G "$MAKEFILE_TYPE Makefiles" \ + bash -c "cmake -G Ninja \ -DCMAKE_C_FLAGS=$CMAKE_C_FLAGS \ -DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \ -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \ @@ -143,10 +153,10 @@ function cmake_build() { -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ ${EXTRA_CMAKE_ARGS} \ - $SRC_DIR + $SRC_DIR " - make -j$(nproc) - make install/strip + config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install/strip" ${NINJA_DRY_RUN} + bash -c "${NINJA_CMD}" } build_flatc $SRC_DIR $REMOVE_OLD_BUILD diff --git a/scripts/cmake-build/host_build.sh b/scripts/cmake-build/host_build.sh index 58f970d6e7df8f515f76d3b9bba852277b4ae1c9..213ba7beccbc74d766c6ce1e18f4789d0b30ef2a 100755 --- a/scripts/cmake-build/host_build.sh +++ b/scripts/cmake-build/host_build.sh @@ -9,6 +9,9 @@ function usage() { echo "-t : Build with training mode, default inference only" echo "-m : Build with m32 mode(only for windows build), default m64" echo "-r : remove old build dir before make, default off" + echo "-v : ninja with verbose and explain, default off" + echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel" + echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)" echo "-h : show usage" echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "example: $0 -d" @@ -22,9 +25,13 @@ MGE_WINDOWS_BUILD_ARCH=x64 MGE_WINDOWS_BUILD_MARCH=m64 MGE_ARCH=x86_64 REMOVE_OLD_BUILD=false +NINJA_VERBOSE=OFF +BUILD_DEVELOP=ON +NINJA_DRY_RUN=OFF + echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" -while getopts "rhdctm" arg +while getopts "nsrhdctmv" arg do case $arg in d) @@ -47,6 +54,18 @@ do echo "config REMOVE_OLD_BUILD=true" REMOVE_OLD_BUILD=true ;; + s) + echo "config BUILD_DEVELOP=OFF" + BUILD_DEVELOP=OFF + ;; + v) + echo "config NINJA_VERBOSE=ON" + NINJA_VERBOSE=ON + ;; + n) + echo "config NINJA_DRY_RUN=ON" + NINJA_DRY_RUN=ON + ;; m) echo "build for m32(only valid use for windows)" MGE_WINDOWS_BUILD_ARCH=x86 @@ -81,6 +100,11 @@ fi SRC_DIR=$($READLINK -f "`dirname $0`/../../") source $SRC_DIR/scripts/cmake-build/utils/utils.sh +if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then + echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON" + BUILD_DEVELOP=OFF +fi + function cmake_build() { BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build INSTALL_DIR=$BUILD_DIR/../install @@ -98,16 +122,17 @@ function cmake_build() { mkdir -p $BUILD_DIR mkdir -p $INSTALL_DIR cd_real_build_dir $BUILD_DIR - cmake \ + # fork a new bash to handle EXTRA_CMAKE_ARGS env with space + bash -c "cmake -G Ninja \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \ -DMGE_WITH_CUDA=$MGE_WITH_CUDA \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ ${EXTRA_CMAKE_ARGS} \ - $SRC_DIR + ${SRC_DIR} " - make -j$(nproc) - make install/strip + config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "install/strip" ${NINJA_DRY_RUN} + bash -c "${NINJA_CMD}" } function windows_env_err() { @@ -199,18 +224,6 @@ function prepare_env_for_windows_build() { export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH } -WINDOWS_BUILD_TARGET="Ninja all" -if [[ -z ${MAKE_DEVELOP} ]] -then - MAKE_DEVELOP="false" -fi -function config_windows_build_target() { - if [ ${MAKE_DEVELOP} = "true" ]; then - echo "build all and develop for pytest test" - WINDOWS_BUILD_TARGET="Ninja all && Ninja develop" - fi -} - function cmake_build_windows() { # windows do not support long path, so we cache the BUILD_DIR ASAP prepare_env_for_windows_build @@ -243,9 +256,10 @@ function cmake_build_windows() { -DCMAKE_C_COMPILER=clang-cl.exe \ -DCMAKE_CXX_COMPILER=clang-cl.exe \ -DCMAKE_MAKE_PROGRAM=ninja.exe \ - ${EXTRA_CMAKE_ARGS} \ - ../../.. && \ - ${WINDOWS_BUILD_TARGET}" + ${EXTRA_CMAKE_ARGS} ../../.. " + + config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "" ${NINJA_DRY_RUN} + cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} " } if [[ $OS =~ "NT" ]]; then @@ -254,7 +268,6 @@ if [[ $OS =~ "NT" ]]; then echo "pls remove -t or remove -m" exit -1 fi - config_windows_build_target cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE else cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE diff --git a/scripts/cmake-build/utils/utils.sh b/scripts/cmake-build/utils/utils.sh index 87704c25614e66db9b7d4572a88e064385acf877..fae31f1f6944e2eb31b23b5391049d17051b8f08 100755 --- a/scripts/cmake-build/utils/utils.sh +++ b/scripts/cmake-build/utils/utils.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash set -e -MAKEFILE_TYPE="Unix" OS=$(uname -s) +NINJA_CMD="" +NINJA_BASE="ninja" if [[ $OS =~ "NT" ]]; then echo "BUILD in NT ..." - MAKEFILE_TYPE="Unix" + NINJA_BASE="Ninja" fi READLINK=readlink @@ -38,7 +39,7 @@ function build_flatc() { mkdir -p $INSTALL_DIR cd_real_build_dir $BUILD_DIR - cmake -G "$MAKEFILE_TYPE Makefiles" \ + cmake -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DFLATBUFFERS_BUILD_TESTS=OFF \ @@ -47,8 +48,8 @@ function build_flatc() { -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF \ $SRC_DIR/third_party/flatbuffers - make -j$(nproc) - make install/strip + ${NINJA_BASE} all + ${NINJA_BASE} install/strip } function try_remove_old_build() { @@ -65,3 +66,33 @@ function try_remove_old_build() { echo "strip remove old build" fi } + +function config_ninja_target_cmd() { + NINJA_CMD="${NINJA_BASE} all" + if [ $# -eq 4 ]; then + _NINJA_VERBOSE=$1 + _BUILD_DEVELOP=$2 + _INSTALL_ALL_TARGET=$3 + _NINJA_DRY_RUN=$4 + else + echo "err call config_ninja_target_cmd" + exit -1 + fi + + if [ ${_NINJA_DRY_RUN} = "ON" ]; then + NINJA_CMD="${NINJA_CMD} -d explain -n" + else + if [ ${_NINJA_VERBOSE} = "ON" ]; then + NINJA_CMD="${NINJA_CMD} -d explain -v" + fi + if [ ${_BUILD_DEVELOP} = "ON" ]; then + echo "add develop target" + NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} develop" + fi + if [ -n "${_INSTALL_ALL_TARGET}" ]; then + NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} ${_INSTALL_ALL_TARGET}" + fi + fi + + echo "build ${NINJA_BASE} target command: ${NINJA_CMD}" +} diff --git a/scripts/whl/macos/macos_build_whl.sh b/scripts/whl/macos/macos_build_whl.sh index 90b8bae53e213db66bd450b7b1a5790d6d82c711..9293201274bd5465488cbd438bd8a9e2d41f6c73 100755 --- a/scripts/whl/macos/macos_build_whl.sh +++ b/scripts/whl/macos/macos_build_whl.sh @@ -34,6 +34,8 @@ function append_path_env_and_check() { append_path_env_and_check SRC_DIR=$($READLINK -f "`dirname $0`/../../../") +source ${SRC_DIR}/scripts/whl/utils/utils.sh + ALL_PYTHON=${ALL_PYTHON} FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3" if [[ -z ${ALL_PYTHON} ]] @@ -91,9 +93,23 @@ function depend_real_copy() { cp "${MEGENGINE_LIB}" ${REAL_DST} } +BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ + +# here we just treat dnn/src/common/conv_bias.cpp should not in the increment build file list +INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty" +IS_IN_FIRST_LOOP=TRUE + function do_build() { for ver in ${ALL_PYTHON} do + # we want run a full clean build at the first loop + if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then + # TODO: may all cmake issue can be resolved after rm CMakeCache? + # if YES, remove this to use old cache and speed up CI + echo "warning: remove old build_dir for the first loop" + rm -rf ${BUILD_DIR} + fi + #config config_python_env ${ver} @@ -108,26 +124,43 @@ function do_build() { fi echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}" echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" - #append cmake args for config python - export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} " #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo " + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo" + #append cmake args for config python + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3" + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_LIBRARY}" + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}" + #we use std::visit in src, so set osx version minimum to 10.14, but 10.14 have objdump #issue, so we now config to 10.15, whl name to 10.14 #TODO: can set to 10.12 after remove use std::visit export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 " - #call build and install - #FIXME: cmake do not triger update python config, after - #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add - #-r to remove build cache after a new ver build, which - #will be more slow build than without -r - echo "build whl with legacy python rt" - ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r + if [ -d "${BUILD_DIR}" ]; then + # insure rm have args + touch ${BUILD_DIR}/empty.so + touch ${BUILD_DIR}/CMakeCache.txt + find ${BUILD_DIR} -name "*.so" | xargs rm + # as we now use increment build mode when switch python + # But I do not known any more issue at CMakeLists.txt or not + # so Force remove CMakeCache.txt + find ${BUILD_DIR} -name CMakeCache.txt | xargs rm + fi + + HOST_BUILD_ARGS="-t -s" + + # call ninja dry run and check increment is invalid or not + if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then + ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}" + fi + + # call real build + echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}" + ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS} - #call setup.py - BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ + # check python api call setup.py cd ${BUILD_DIR} + check_build_ninja_python_api ${ver} rm -rf staging mkdir -p staging @@ -180,6 +213,7 @@ function do_build() { echo "macos whl package location: ${MACOS_WHL_HOME}" ls ${MACOS_WHL_HOME} echo "##############################################################################################" + IS_IN_FIRST_LOOP=FALSE done } diff --git a/scripts/whl/macos/macos_whl_env_prepare.sh b/scripts/whl/macos/macos_whl_env_prepare.sh index 091fb49f8a74089f6f88dfc598f9d2d95b80921a..159e0cf3bbf895f52b480b4aaff5079f67c832f7 100755 --- a/scripts/whl/macos/macos_whl_env_prepare.sh +++ b/scripts/whl/macos/macos_whl_env_prepare.sh @@ -20,7 +20,7 @@ function try_install_brew() { } function install_brew_package() { - BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs" + BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs ninja" for pak in ${BREW_PACKAGE} do echo "###### do command: brew install ${pak}" diff --git a/scripts/whl/manylinux2014/build_wheel_common.sh b/scripts/whl/manylinux2014/build_wheel_common.sh index bdc0605527e875fe41bcbd0ed973d95d36730269..4335e818bc146bde476e7401635dbc65094b5f30 100755 --- a/scripts/whl/manylinux2014/build_wheel_common.sh +++ b/scripts/whl/manylinux2014/build_wheel_common.sh @@ -50,12 +50,13 @@ elif [ $SDK_NAME == "cu111" ];then ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\ ${CUDNN_LIB_DIR}/libcudnn.so.8" EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \ - -gencode arch=compute_61,code=sm_61 \ - arch=compute_70,code=sm_70 \ - arch=compute_75,code=sm_75 \ - arch=compute_80,code=sm_80 \ - arch=compute_86,code=sm_86 \ - arch=compute_86,code=compute_86" + -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \ + -gencode arch=compute_70,code=sm_70 \ + -gencode arch=compute_75,code=sm_75 \ + -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_86,code=compute_86\" " + REQUIR_CUDA_VERSION="11010" REQUIR_CUDNN_VERSION="8.0.4" REQUIR_TENSORRT_VERSION="7.2.2.3" @@ -73,12 +74,13 @@ elif [ $SDK_NAME == "cu112" ];then ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\ ${CUDNN_LIB_DIR}/libcudnn.so.8" EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \ - -gencode arch=compute_61,code=sm_61 \ - arch=compute_70,code=sm_70 \ - arch=compute_75,code=sm_75 \ - arch=compute_80,code=sm_80 \ - arch=compute_86,code=sm_86 \ - arch=compute_86,code=compute_86" + -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \ + -gencode arch=compute_70,code=sm_70 \ + -gencode arch=compute_75,code=sm_75 \ + -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_86,code=compute_86\" " + REQUIR_CUDA_VERSION="11020" REQUIR_CUDNN_VERSION="8.0.4" REQUIR_TENSORRT_VERSION="7.2.2.3" diff --git a/scripts/whl/manylinux2014/do_build_common.sh b/scripts/whl/manylinux2014/do_build_common.sh index eecbfc0a4ec532d28bff1c5c8e09f85df026baab..a8e219ef42146f85981c64e50a14454b76131c8d 100755 --- a/scripts/whl/manylinux2014/do_build_common.sh +++ b/scripts/whl/manylinux2014/do_build_common.sh @@ -75,33 +75,66 @@ then fi SRC_DIR=$(readlink -f "`dirname $0`/../../../") +source ${SRC_DIR}/scripts/whl/utils/utils.sh + BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/ fi +# here we just treat cu file should not in the increment build file list +INCREMENT_KEY_WORDS=".cu.o is dirty" +IS_IN_FIRST_LOOP=TRUE + for ver in ${ALL_PYTHON} do + # we want run a full clean build at the first loop + if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then + # TODO: may all cmake issue can be resolved after rm CMakeCache? + # if YES, remove this to use old cache and speed up CI + echo "warning: remove old build_dir for the first loop" + rm -rf ${BUILD_DIR} + fi + python_ver=${ver:0:2} MAJOR=${python_ver:0:1} MINOR=${ver:1} PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/ export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} ${EXTRA_CMAKE_FLAG}" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo" - export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON" - # TODO: after change to Ninja, only the fisrt loop need add -r to save build time + if [ -d "${BUILD_DIR}" ]; then + # insure rm have args + touch ${BUILD_DIR}/empty.so + touch ${BUILD_DIR}/CMakeCache.txt + find ${BUILD_DIR} -name "*.so" | xargs rm + # as we now use increment build mode when switch python + # But I do not known any more issue at CMakeLists.txt or not + # so Force remove CMakeCache.txt + find ${BUILD_DIR} -name CMakeCache.txt | xargs rm + fi + + HOST_BUILD_ARGS="-t -s" if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then - ${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r - else - ${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r + HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c" + fi + + # call ninja dry run and check increment is invalid or not + if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then + ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}" fi + # call real build + echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}" + ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS} + + # check python api call setup.py cd ${BUILD_DIR} + check_build_ninja_python_api ${ver} rm -rf staging mkdir -p staging cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ @@ -128,4 +161,5 @@ do # compat for root-less docker env to remove output at host side chmod -R 777 . echo "python $ver done" + IS_IN_FIRST_LOOP=FALSE done diff --git a/scripts/whl/manylinux2014/init_image.sh b/scripts/whl/manylinux2014/init_image.sh index f44484d908547bd1633aa561c85a0d820b3d40c2..f0bfa83149d7f7d22b6b3cff0295f4604b9b88ce 100755 --- a/scripts/whl/manylinux2014/init_image.sh +++ b/scripts/whl/manylinux2014/init_image.sh @@ -7,8 +7,11 @@ LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz' CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz' -yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 +yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 ninja-build yum install -y devtoolset-8 devtoolset-8-libatomic-devel.x86_64 +# install a default python3 for cmake PYTHON3_EXECUTABLE_WITHOUT_VERSION +yum install -y python3 +python3 -m pip install numpy for ver in 35m 36m 37m 38 do diff --git a/scripts/whl/utils/utils.sh b/scripts/whl/utils/utils.sh new file mode 100755 index 0000000000000000000000000000000000000000..7148a546bb451dc35f575559d7ff443830e6732f --- /dev/null +++ b/scripts/whl/utils/utils.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -e + +OS=$(uname -s) + +function ninja_dry_run_and_check_increment() { + if [ $# -eq 3 ]; then + _BUILD_SHELL=$1 + _BUILD_FLAGS="$2 -n" + _INCREMENT_KEY_WORDS=$3 + else + echo "err call ninja_dry_run_and_check_increment" + exit -1 + fi + + ${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log + + DIRTY_LOG=`cat dry_run.log` + if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then + echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + echo "python3 switch increment build failed, some MR make a wrong CMakeLists.txt depends" + echo "or build env can not find default python3 in PATH env" + echo "please refs for PYTHON3_EXECUTABLE_WITHOUT_VERSION define at SRC_ROOT/CMakeLists.txt" + echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + exit -1 + fi +} + +PYTHON_API_INCLUDES="" + +function check_build_ninja_python_api() { + INCLUDE_KEYWORD="" + IS_MINOR_HIT=FALSE + if [ $# -eq 1 ]; then + ver=$1 + echo "org args: ${ver}" + if [[ $OS =~ "NT" ]]; then + INCLUDE_KEYWORD="${ver}\\\\include" + PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include" + elif [[ $OS =~ "Linux" ]]; then + INCLUDE_KEYWORD="include/python3.${ver:1:1}" + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8" + elif [[ $OS =~ "Darwin" ]]; then + INCLUDE_KEYWORD="include/python3.${ver:2:1}" + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8" + else + echo "unknown OS: ${OS}" + exit -1 + fi + else + echo "err call check_build_ninja_python_api" + exit -1 + fi + echo "try check python INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} is invalid in ninja.build or not" + + NINJA_BUILD=`cat build.ninja` + for PYTHON_API_INCLUDE in ${PYTHON_API_INCLUDES} + do + echo "check PYTHON_API_INCLUDE vs INCLUDE_KEYWORD : (${PYTHON_API_INCLUDE} : ${INCLUDE_KEYWORD})" + if [ ${PYTHON_API_INCLUDE} = ${INCLUDE_KEYWORD} ]; then + if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then + echo "hit INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja" + IS_MINOR_HIT="TRUE" + else + echo "Err happened can not find INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja" + exit -1 + fi + else + if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then + echo "Err happened: find PYTHON_API_INCLUDE: ${PYTHON_API_INCLUDE} in build.ninja" + echo "But now INCLUDE_KEYWORD: ${INCLUDE_KEYWORD}" + exit -1 + fi + fi + done + + if [ ${IS_MINOR_HIT} = "FALSE" ]; then + echo "Err happened, can not hit any MINOR api in ninja.build" + exit -1 + fi +} diff --git a/scripts/whl/windows/windows_build_whl.sh b/scripts/whl/windows/windows_build_whl.sh index 4a958c5aadbeec8ae6b421dff1527b63d7e52e78..5eac75c88803ec4119bf1c464222693ca7905e4a 100755 --- a/scripts/whl/windows/windows_build_whl.sh +++ b/scripts/whl/windows/windows_build_whl.sh @@ -23,6 +23,8 @@ function append_path_env_and_check() { append_path_env_and_check SRC_DIR=$(READLINK -f "`dirname $0`/../../../") +source ${SRC_DIR}/scripts/whl/utils/utils.sh + ALL_PYTHON=${ALL_PYTHON} FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3" if [[ -z ${ALL_PYTHON} ]] @@ -43,12 +45,12 @@ mkdir -p ${WINDOWS_WHL_HOME} function config_python_env() { PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1 PYTHON_BIN=${PYTHON_DIR} - if [ ! -f "$PYTHON_BIN/python3.exe" ]; then + if [ ! -f "${PYTHON_BIN}/python3.exe" ]; then echo "ERR: can not find $PYTHON_BIN , Invalid python package" echo "now support list: ${FULL_PYTHON_VER}" err_env else - echo "put python3 to env..." + echo "put ${PYTHON_BIN}/python3.exe to env..." export PATH=${PYTHON_BIN}:$PATH which python3 fi @@ -104,11 +106,24 @@ function copy_more_dll() { fi } +BUILD_DIR=${SRC_DIR}/build_dir/host/build/ + +# here we just treat cu file should not in the increment build file list +INCREMENT_KEY_WORDS=".cu.obj is dirty" +IS_IN_FIRST_LOOP=TRUE function do_build() { for ver in ${ALL_PYTHON} do - #config + # we want run a full clean build at the first loop + if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then + # TODO: may all cmake issue can be resolved after rm CMakeCache? + # if YES, remove this to use old cache and speed up CI + echo "warning: remove old build_dir for the first loop" + rm -rf ${BUILD_DIR} + fi + + #config python3 config_python_env ${ver} #check env @@ -122,35 +137,44 @@ function do_build() { fi echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}" echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" - #append cmake args for config python - #FIXME: ninja handle err with cmake 3.17 when assgin PYTHON_LIBRARY - #But after put python3.exe to HEAD of PATH by config_python_env, cmake can also handle the - #right PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, at the same time, clang-cl need swig target - #force LINK a real PYTHON_LIBRARY file, after test we do not find the symbols conflict with python - #export EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} " #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo " #call build and install - #FIXME: cmake do not triger update python config, after - #change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add - #-r to remove build cache after a new ver build, which - #will be more slow build than without -r - BUILD_ARGS=" -t -r" + HOST_BUILD_ARGS=" -t -s" if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then echo "build windows whl with cuda" - BUILD_ARGS="${BUILD_ARGS} -c " + HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c " else echo "build windows whl with cpu only" fi - echo "host_build.sh BUILD_ARGS: ${BUILD_ARGS}" - ${SRC_DIR}/scripts/cmake-build/host_build.sh ${BUILD_ARGS} + if [ -d "${BUILD_DIR}" ]; then + # insure rm have args + touch ${BUILD_DIR}/empty.pyd + touch ${BUILD_DIR}/CMakeCache.txt + /usr/bin/find ${BUILD_DIR} -name "*.pyd" | xargs rm + # ninja/cmake on windows will handle error if just export + # PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE + # But after put python3.exe to HEAD of PATH by config_python_env + # and force remove CMakeCache.txt, ninja/cmake will auto update + # PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE + /usr/bin/find ${BUILD_DIR} -name CMakeCache.txt | xargs rm + fi + echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}" - BUILD_DIR=${SRC_DIR}/build_dir/host/build/ - cd ${BUILD_DIR} + # call ninja dry run and check increment is invalid or not + if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then + ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}" + fi + #call real build + ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS} + + # check python api call setup.py + cd ${BUILD_DIR} + check_build_ninja_python_api ${ver} rm -rf staging mkdir -p staging cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ @@ -178,6 +202,7 @@ function do_build() { echo "windows whl package location: ${WINDOWS_WHL_HOME}" ls ${WINDOWS_WHL_HOME} echo "##############################################################################################" + IS_IN_FIRST_LOOP=FALSE done }