Skip to content

MegEngine
MegEngine

MegEngine 天元 / MegEngine
1 年多 前同步成功

通知 404
Star 4705
Fork 582
MegEngine
MegEngine
提交 baa30400 编写于 作者: M Megvii Engine Team
浏览文件
操作

feat(build): change make to ninja 

GitOrigin-RevId: ccff72e39a4cd79f4cad0fbb474680a300204301
上级 e4af4225
隐藏空白更改
内联 并排
Showing with 365 addition and 102 deletion
CMakeLists.txt
浏览文件 @ baa30400
cmake_minimum_required(VERSION 3.15.2) cmake_minimum_required(VERSION 3.15.2)
message(STATUS "CMAKE_GENERATOR: ${CMAKE_GENERATOR}" )
if (NOT ${CMAKE_GENERATOR} STREQUAL "Ninja")
message(WARNING "CMAKE_GENERATOR NOT EQUAL Ninja, which we do not recommend")
endif()
include (cmake/FetchMegBrainVersion.cmake) include (cmake/FetchMegBrainVersion.cmake)
project(MegEngine LANGUAGES C CXX VERSION ${MGB_VER_STRING}) project(MegEngine LANGUAGES C CXX VERSION ${MGB_VER_STRING})
......
scripts/cmake-build/BUILD_README.md
浏览文件 @ baa30400
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
* commands: * commands:
``` ```
1: installl Visual Studio (need support LLVM/clang-cl), eg 2019. Please install LLVM-10, VS LLVM linker have issue, please replace lld-link.exe, which can be download from https://releases.llvm.org/download.html#10.0.0 1: installl Visual Studio (need support LLVM/clang-cl), eg 2019. Please install LLVM-10, VS LLVM linker have issue, please replace lld-link.exe, which can be download from https://releases.llvm.org/download.html#10.0.0
2: install extension of VS: Python/Cmake/LLVM 2: install extension of VS: Python/Cmake/LLVM/Ninja
3: now we support cuda10.1+cudnn7.6+TensorRT6.0 on Windows, as Windows can only use DLL in fact with cudnn/TensorRT, so please install the same version; 3: now we support cuda10.1+cudnn7.6+TensorRT6.0 on Windows, as Windows can only use DLL in fact with cudnn/TensorRT, so please install the same version;
3a: install cuda10.1 to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1 3a: install cuda10.1 to C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1
3b: install cudnn7.6 to C:\Program Files\NVIDIA GPU Computing Toolkit\cudnn-10.1-windows10-x64-v7.6.5.32 3b: install cudnn7.6 to C:\Program Files\NVIDIA GPU Computing Toolkit\cudnn-10.1-windows10-x64-v7.6.5.32
...@@ -33,9 +33,9 @@ ...@@ -33,9 +33,9 @@
### Linux host build ### Linux host build
* commands: * commands:
``` ```
1: install Cmake, which version >= 3.15.2 1: install Cmake, which version >= 3.15.2, ninja-build
2: install gcc/g++, which version >= 6, (gcc/g++ >= 7, if need build training mode) 2: install gcc/g++, which version >= 6, (gcc/g++ >= 7, if need build training mode)
3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev swig python3-numpy texinfo 3: install build-essential git git-lfs gfortran libgfortran-6-dev autoconf gnupg flex bison gperf curl zlib1g-dev gcc-multilib g++-multilib lib32ncurses5-dev libxml2-utils xsltproc unzip libtool librdmacm-dev rdmacm-utils python3-dev python3-numpy texinfo
4: CUDA env(if enable CUDA), version detail refer to README.md 4: CUDA env(if enable CUDA), version detail refer to README.md
``` ```
...@@ -44,7 +44,7 @@ ...@@ -44,7 +44,7 @@
``` ```
1: install Cmake, which version >= 3.15.2 1: install Cmake, which version >= 3.15.2
2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
3: brew install python python3 swig coreutils 3: brew install python python3 coreutils ninja
4: install at least xcode command line tool: https://developer.apple.com/xcode/ 4: install at least xcode command line tool: https://developer.apple.com/xcode/
5: about cuda: we do not support CUDA on MacOS 5: about cuda: we do not support CUDA on MacOS
6: python3 -m pip install numpy (if you want to build with training mode) 6: python3 -m pip install numpy (if you want to build with training mode)
......
scripts/cmake-build/cross_build_android_arm_inference.sh
浏览文件 @ baa30400
...@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF ...@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a ARCH=arm64-v8a
REMOVE_OLD_BUILD=false REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() { function usage() {
...@@ -17,13 +20,15 @@ function usage() { ...@@ -17,13 +20,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}" echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off" echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage" echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d" echo "example: $0 -d"
exit -1 exit -1
} }
while getopts "rkhdfa:" arg while getopts "nvrkhdfa:" arg
do do
case $arg in case $arg in
d) d)
...@@ -62,6 +67,14 @@ do ...@@ -62,6 +67,14 @@ do
echo "config REMOVE_OLD_BUILD=true" echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true REMOVE_OLD_BUILD=true
;; ;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?) ?)
echo "unkonw argument" echo "unkonw argument"
usage usage
...@@ -77,14 +90,12 @@ echo "ARCH: $ARCH" ...@@ -77,14 +90,12 @@ echo "ARCH: $ARCH"
echo "----------------------------------------------------" echo "----------------------------------------------------"
READLINK=readlink READLINK=readlink
MAKEFILE_TYPE="Unix"
OS=$(uname -s) OS=$(uname -s)
if [ $OS = "Darwin" ];then if [ $OS = "Darwin" ];then
READLINK=greadlink READLINK=greadlink
elif [[ $OS =~ "NT" ]]; then elif [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..." echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix"
fi fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../") SRC_DIR=$($READLINK -f "`dirname $0`/../../")
...@@ -105,7 +116,6 @@ function cmake_build() { ...@@ -105,7 +116,6 @@ function cmake_build() {
echo "build type: $BUILD_TYPE" echo "build type: $BUILD_TYPE"
echo "build ABI: $BUILD_ABI" echo "build ABI: $BUILD_ABI"
echo "build native level: $BUILD_NATIVE_LEVEL" echo "build native level: $BUILD_NATIVE_LEVEL"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir" echo "create build dir"
...@@ -113,7 +123,7 @@ function cmake_build() { ...@@ -113,7 +123,7 @@ function cmake_build() {
mkdir -p $INSTALL_DIR mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR cd_real_build_dir $BUILD_DIR
unset IFS unset IFS
cmake -G "$MAKEFILE_TYPE Makefiles" \ bash -c "cmake -G Ninja \
-DCMAKE_TOOLCHAIN_FILE="$NDK_ROOT/build/cmake/android.toolchain.cmake" \ -DCMAKE_TOOLCHAIN_FILE="$NDK_ROOT/build/cmake/android.toolchain.cmake" \
-DANDROID_NDK="$NDK_ROOT" \ -DANDROID_NDK="$NDK_ROOT" \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
...@@ -125,10 +135,11 @@ function cmake_build() { ...@@ -125,10 +135,11 @@ function cmake_build() {
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} \
$SRC_DIR $SRC_DIR "
make -j$(nproc) ${Target} config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "" ${NINJA_DRY_RUN}
make install/strip bash -c "${NINJA_CMD}"
${NINJA_BASE} install/strip
} }
build_flatc $SRC_DIR $REMOVE_OLD_BUILD build_flatc $SRC_DIR $REMOVE_OLD_BUILD
......
scripts/cmake-build/cross_build_ios_arm_inference.sh
浏览文件 @ baa30400
...@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF ...@@ -7,6 +7,9 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF MGE_DISABLE_FLOAT16=OFF
ARCH=arm64 ARCH=arm64
REMOVE_OLD_BUILD=false REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() { function usage() {
...@@ -17,13 +20,15 @@ function usage() { ...@@ -17,13 +20,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}" echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off" echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage" echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d" echo "example: $0 -d"
exit -1 exit -1
} }
while getopts "rkhdfa:" arg while getopts "nvrkhdfa:" arg
do do
case $arg in case $arg in
d) d)
...@@ -62,6 +67,14 @@ do ...@@ -62,6 +67,14 @@ do
echo "config REMOVE_OLD_BUILD=true" echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true REMOVE_OLD_BUILD=true
;; ;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?) ?)
echo "unkonw argument" echo "unkonw argument"
usage usage
...@@ -109,7 +122,8 @@ function cmake_build() { ...@@ -109,7 +122,8 @@ function cmake_build() {
mkdir -p $BUILD_DIR mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR cd_real_build_dir $BUILD_DIR
cmake -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \ bash -c "cmake -G Ninja \
-DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DIOS_TOOLCHAIN_ROOT=$TOOLCHAIN \ -DIOS_TOOLCHAIN_ROOT=$TOOLCHAIN \
-DOS_PLATFORM=$OS_PLATFORM \ -DOS_PLATFORM=$OS_PLATFORM \
...@@ -121,11 +135,12 @@ function cmake_build() { ...@@ -121,11 +135,12 @@ function cmake_build() {
-DMGE_ARMV8_2_FEATURE_FP16= $MGE_ARMV8_2_FEATURE_FP16 \ -DMGE_ARMV8_2_FEATURE_FP16= $MGE_ARMV8_2_FEATURE_FP16 \
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DCMAKE_MAKE_PROGRAM=ninja \
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} \
$SRC_DIR $SRC_DIR "
make -j$(nproc) config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install" ${NINJA_DRY_RUN}
make install bash -c "${NINJA_CMD}"
} }
build_flatc $SRC_DIR $REMOVE_OLD_BUILD build_flatc $SRC_DIR $REMOVE_OLD_BUILD
......
scripts/cmake-build/cross_build_linux_arm_inference.sh
浏览文件 @ baa30400
...@@ -8,8 +8,11 @@ MGE_ARMV8_2_FEATURE_FP16=OFF ...@@ -8,8 +8,11 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_DISABLE_FLOAT16=OFF MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a ARCH=arm64-v8a
REMOVE_OLD_BUILD=false REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
NINJA_DRY_RUN=OFF
CMAKE_C_FLAGS="-Wno-psabi" CMAKE_C_FLAGS="-Wno-psabi"
CMAKE_CXX_FLAGS="-Wno-psabi" CMAKE_CXX_FLAGS="-Wno-psabi"
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() { function usage() {
...@@ -21,13 +24,15 @@ function usage() { ...@@ -21,13 +24,15 @@ function usage() {
echo "-k : open MGE_DISABLE_FLOAT16 for NEON " echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}" echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off" echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage" echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d" echo "example: $0 -d"
exit -1 exit -1
} }
while getopts "rkhdcfa:" arg while getopts "nvrkhdcfa:" arg
do do
case $arg in case $arg in
d) d)
...@@ -70,6 +75,14 @@ do ...@@ -70,6 +75,14 @@ do
echo "config REMOVE_OLD_BUILD=true" echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true REMOVE_OLD_BUILD=true
;; ;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
?) ?)
echo "unkonw argument" echo "unkonw argument"
usage usage
...@@ -86,14 +99,12 @@ echo "ARCH: $ARCH" ...@@ -86,14 +99,12 @@ echo "ARCH: $ARCH"
echo "----------------------------------------------------" echo "----------------------------------------------------"
READLINK=readlink READLINK=readlink
MAKEFILE_TYPE="Unix"
OS=$(uname -s) OS=$(uname -s)
if [ $OS = "Darwin" ];then if [ $OS = "Darwin" ];then
READLINK=greadlink READLINK=greadlink
elif [[ $OS =~ "NT" ]]; then elif [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..." echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix"
fi fi
if [ ! $OS = "Linux" ] && [ $MGE_WITH_CUDA = "ON" ];then if [ ! $OS = "Linux" ] && [ $MGE_WITH_CUDA = "ON" ];then
...@@ -125,14 +136,13 @@ function cmake_build() { ...@@ -125,14 +136,13 @@ function cmake_build() {
echo "build type: $BUILD_TYPE" echo "build type: $BUILD_TYPE"
echo "build toolchain: $TOOLCHAIN" echo "build toolchain: $TOOLCHAIN"
echo "MGE_WITH_CUDA: $MGE_WITH_CUDA" echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir" echo "create build dir"
mkdir -p $BUILD_DIR mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR cd_real_build_dir $BUILD_DIR
cmake -G "$MAKEFILE_TYPE Makefiles" \ bash -c "cmake -G Ninja \
-DCMAKE_C_FLAGS=$CMAKE_C_FLAGS \ -DCMAKE_C_FLAGS=$CMAKE_C_FLAGS \
-DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \ -DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \
-DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \ -DCMAKE_TOOLCHAIN_FILE=$TOOLCHAIN \
...@@ -143,10 +153,10 @@ function cmake_build() { ...@@ -143,10 +153,10 @@ function cmake_build() {
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \ -DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} \
$SRC_DIR $SRC_DIR "
make -j$(nproc) config_ninja_target_cmd ${NINJA_VERBOSE} "OFF" "install/strip" ${NINJA_DRY_RUN}
make install/strip bash -c "${NINJA_CMD}"
} }
build_flatc $SRC_DIR $REMOVE_OLD_BUILD build_flatc $SRC_DIR $REMOVE_OLD_BUILD
......
scripts/cmake-build/host_build.sh
浏览文件 @ baa30400
...@@ -9,6 +9,9 @@ function usage() { ...@@ -9,6 +9,9 @@ function usage() {
echo "-t : Build with training mode, default inference only" echo "-t : Build with training mode, default inference only"
echo "-m : Build with m32 mode(only for windows build), default m64" echo "-m : Build with m32 mode(only for windows build), default m64"
echo "-r : remove old build dir before make, default off" echo "-r : remove old build dir before make, default off"
echo "-v : ninja with verbose and explain, default off"
echo "-s : Do not build develop even build with training mode, default on when build with training, always for wheel"
echo "-n : ninja with -n dry run (don't run commands but act like they succeeded)"
echo "-h : show usage" echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..." echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d" echo "example: $0 -d"
...@@ -22,9 +25,13 @@ MGE_WINDOWS_BUILD_ARCH=x64 ...@@ -22,9 +25,13 @@ MGE_WINDOWS_BUILD_ARCH=x64
MGE_WINDOWS_BUILD_MARCH=m64 MGE_WINDOWS_BUILD_MARCH=m64
MGE_ARCH=x86_64 MGE_ARCH=x86_64
REMOVE_OLD_BUILD=false REMOVE_OLD_BUILD=false
NINJA_VERBOSE=OFF
BUILD_DEVELOP=ON
NINJA_DRY_RUN=OFF
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}" echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
while getopts "rhdctm" arg while getopts "nsrhdctmv" arg
do do
case $arg in case $arg in
d) d)
...@@ -47,6 +54,18 @@ do ...@@ -47,6 +54,18 @@ do
echo "config REMOVE_OLD_BUILD=true" echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true REMOVE_OLD_BUILD=true
;; ;;
s)
echo "config BUILD_DEVELOP=OFF"
BUILD_DEVELOP=OFF
;;
v)
echo "config NINJA_VERBOSE=ON"
NINJA_VERBOSE=ON
;;
n)
echo "config NINJA_DRY_RUN=ON"
NINJA_DRY_RUN=ON
;;
m) m)
echo "build for m32(only valid use for windows)" echo "build for m32(only valid use for windows)"
MGE_WINDOWS_BUILD_ARCH=x86 MGE_WINDOWS_BUILD_ARCH=x86
...@@ -81,6 +100,11 @@ fi ...@@ -81,6 +100,11 @@ fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../") SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/utils.sh source $SRC_DIR/scripts/cmake-build/utils/utils.sh
if [ ${MGE_INFERENCE_ONLY} = "ON" ]; then
echo "config BUILD_DEVELOP=OFF when MGE_INFERENCE_ONLY=ON"
BUILD_DEVELOP=OFF
fi
function cmake_build() { function cmake_build() {
BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
INSTALL_DIR=$BUILD_DIR/../install INSTALL_DIR=$BUILD_DIR/../install
...@@ -98,16 +122,17 @@ function cmake_build() { ...@@ -98,16 +122,17 @@ function cmake_build() {
mkdir -p $BUILD_DIR mkdir -p $BUILD_DIR
mkdir -p $INSTALL_DIR mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR cd_real_build_dir $BUILD_DIR
cmake \ # fork a new bash to handle EXTRA_CMAKE_ARGS env with space
bash -c "cmake -G Ninja \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \ -DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
-DMGE_WITH_CUDA=$MGE_WITH_CUDA \ -DMGE_WITH_CUDA=$MGE_WITH_CUDA \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} \
$SRC_DIR ${SRC_DIR} "
make -j$(nproc) config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "install/strip" ${NINJA_DRY_RUN}
make install/strip bash -c "${NINJA_CMD}"
} }
function windows_env_err() { function windows_env_err() {
...@@ -199,18 +224,6 @@ function prepare_env_for_windows_build() { ...@@ -199,18 +224,6 @@ function prepare_env_for_windows_build() {
export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH export PATH=/c/Users/${USER}/swigwin-4.0.2::$PATH
} }
WINDOWS_BUILD_TARGET="Ninja all"
if [[ -z ${MAKE_DEVELOP} ]]
then
MAKE_DEVELOP="false"
fi
function config_windows_build_target() {
if [ ${MAKE_DEVELOP} = "true" ]; then
echo "build all and develop for pytest test"
WINDOWS_BUILD_TARGET="Ninja all && Ninja develop"
fi
}
function cmake_build_windows() { function cmake_build_windows() {
# windows do not support long path, so we cache the BUILD_DIR ASAP # windows do not support long path, so we cache the BUILD_DIR ASAP
prepare_env_for_windows_build prepare_env_for_windows_build
...@@ -243,9 +256,10 @@ function cmake_build_windows() { ...@@ -243,9 +256,10 @@ function cmake_build_windows() {
-DCMAKE_C_COMPILER=clang-cl.exe \ -DCMAKE_C_COMPILER=clang-cl.exe \
-DCMAKE_CXX_COMPILER=clang-cl.exe \ -DCMAKE_CXX_COMPILER=clang-cl.exe \
-DCMAKE_MAKE_PROGRAM=ninja.exe \ -DCMAKE_MAKE_PROGRAM=ninja.exe \
${EXTRA_CMAKE_ARGS} \ ${EXTRA_CMAKE_ARGS} ../../.. "
../../.. && \
${WINDOWS_BUILD_TARGET}" config_ninja_target_cmd ${NINJA_VERBOSE} ${BUILD_DEVELOP} "" ${NINJA_DRY_RUN}
cmd.exe /c " vcvarsall.bat $MGE_WINDOWS_BUILD_ARCH && ${NINJA_CMD} "
} }
if [[ $OS =~ "NT" ]]; then if [[ $OS =~ "NT" ]]; then
...@@ -254,7 +268,6 @@ if [[ $OS =~ "NT" ]]; then ...@@ -254,7 +268,6 @@ if [[ $OS =~ "NT" ]]; then
echo "pls remove -t or remove -m" echo "pls remove -t or remove -m"
exit -1 exit -1
fi fi
config_windows_build_target
cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE cmake_build_windows $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
else else
cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE cmake_build $MGE_WITH_CUDA $MGE_INFERENCE_ONLY $BUILD_TYPE
......
scripts/cmake-build/utils/utils.sh
浏览文件 @ baa30400
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
MAKEFILE_TYPE="Unix"
OS=$(uname -s) OS=$(uname -s)
NINJA_CMD=""
NINJA_BASE="ninja"
if [[ $OS =~ "NT" ]]; then if [[ $OS =~ "NT" ]]; then
echo "BUILD in NT ..." echo "BUILD in NT ..."
MAKEFILE_TYPE="Unix" NINJA_BASE="Ninja"
fi fi
READLINK=readlink READLINK=readlink
...@@ -38,7 +39,7 @@ function build_flatc() { ...@@ -38,7 +39,7 @@ function build_flatc() {
mkdir -p $INSTALL_DIR mkdir -p $INSTALL_DIR
cd_real_build_dir $BUILD_DIR cd_real_build_dir $BUILD_DIR
cmake -G "$MAKEFILE_TYPE Makefiles" \ cmake -G Ninja \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
-DFLATBUFFERS_BUILD_TESTS=OFF \ -DFLATBUFFERS_BUILD_TESTS=OFF \
...@@ -47,8 +48,8 @@ function build_flatc() { ...@@ -47,8 +48,8 @@ function build_flatc() {
-DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF \ -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF \
$SRC_DIR/third_party/flatbuffers $SRC_DIR/third_party/flatbuffers
make -j$(nproc) ${NINJA_BASE} all
make install/strip ${NINJA_BASE} install/strip
} }
function try_remove_old_build() { function try_remove_old_build() {
...@@ -65,3 +66,33 @@ function try_remove_old_build() { ...@@ -65,3 +66,33 @@ function try_remove_old_build() {
echo "strip remove old build" echo "strip remove old build"
fi fi
} }
function config_ninja_target_cmd() {
NINJA_CMD="${NINJA_BASE} all"
if [ $# -eq 4 ]; then
_NINJA_VERBOSE=$1
_BUILD_DEVELOP=$2
_INSTALL_ALL_TARGET=$3
_NINJA_DRY_RUN=$4
else
echo "err call config_ninja_target_cmd"
exit -1
fi
if [ ${_NINJA_DRY_RUN} = "ON" ]; then
NINJA_CMD="${NINJA_CMD} -d explain -n"
else
if [ ${_NINJA_VERBOSE} = "ON" ]; then
NINJA_CMD="${NINJA_CMD} -d explain -v"
fi
if [ ${_BUILD_DEVELOP} = "ON" ]; then
echo "add develop target"
NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} develop"
fi
if [ -n "${_INSTALL_ALL_TARGET}" ]; then
NINJA_CMD="${NINJA_CMD} && ${NINJA_BASE} ${_INSTALL_ALL_TARGET}"
fi
fi
echo "build ${NINJA_BASE} target command: ${NINJA_CMD}"
}
scripts/whl/macos/macos_build_whl.sh
浏览文件 @ baa30400
...@@ -34,6 +34,8 @@ function append_path_env_and_check() { ...@@ -34,6 +34,8 @@ function append_path_env_and_check() {
append_path_env_and_check append_path_env_and_check
SRC_DIR=$($READLINK -f "`dirname $0`/../../../") SRC_DIR=$($READLINK -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh
ALL_PYTHON=${ALL_PYTHON} ALL_PYTHON=${ALL_PYTHON}
FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3" FULL_PYTHON_VER="3.5.9 3.6.10 3.7.7 3.8.3"
if [[ -z ${ALL_PYTHON} ]] if [[ -z ${ALL_PYTHON} ]]
...@@ -91,9 +93,23 @@ function depend_real_copy() { ...@@ -91,9 +93,23 @@ function depend_real_copy() {
cp "${MEGENGINE_LIB}" ${REAL_DST} cp "${MEGENGINE_LIB}" ${REAL_DST}
} }
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
# here we just treat dnn/src/common/conv_bias.cpp should not in the increment build file list
INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty"
IS_IN_FIRST_LOOP=TRUE
function do_build() { function do_build() {
for ver in ${ALL_PYTHON} for ver in ${ALL_PYTHON}
do do
# we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi
#config #config
config_python_env ${ver} config_python_env ${ver}
...@@ -108,26 +124,43 @@ function do_build() { ...@@ -108,26 +124,43 @@ function do_build() {
fi fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}" echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo " export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
#append cmake args for config python
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_LIBRARY}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}"
#we use std::visit in src, so set osx version minimum to 10.14, but 10.14 have objdump #we use std::visit in src, so set osx version minimum to 10.14, but 10.14 have objdump
#issue, so we now config to 10.15, whl name to 10.14 #issue, so we now config to 10.15, whl name to 10.14
#TODO: can set to 10.12 after remove use std::visit #TODO: can set to 10.12 after remove use std::visit
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 " export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 "
#call build and install if [ -d "${BUILD_DIR}" ]; then
#FIXME: cmake do not triger update python config, after # insure rm have args
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add touch ${BUILD_DIR}/empty.so
#-r to remove build cache after a new ver build, which touch ${BUILD_DIR}/CMakeCache.txt
#will be more slow build than without -r find ${BUILD_DIR} -name "*.so" | xargs rm
echo "build whl with legacy python rt" # as we now use increment build mode when switch python
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r # But I do not known any more issue at CMakeLists.txt or not
# so Force remove CMakeCache.txt
find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi
HOST_BUILD_ARGS="-t -s"
# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi
# call real build
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
#call setup.py # check python api call setup.py
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
cd ${BUILD_DIR} cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging rm -rf staging
mkdir -p staging mkdir -p staging
...@@ -180,6 +213,7 @@ function do_build() { ...@@ -180,6 +213,7 @@ function do_build() {
echo "macos whl package location: ${MACOS_WHL_HOME}" echo "macos whl package location: ${MACOS_WHL_HOME}"
ls ${MACOS_WHL_HOME} ls ${MACOS_WHL_HOME}
echo "##############################################################################################" echo "##############################################################################################"
IS_IN_FIRST_LOOP=FALSE
done done
} }
......
scripts/whl/macos/macos_whl_env_prepare.sh
浏览文件 @ baa30400
...@@ -20,7 +20,7 @@ function try_install_brew() { ...@@ -20,7 +20,7 @@ function try_install_brew() {
} }
function install_brew_package() { function install_brew_package() {
BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs" BREW_PACKAGE="openssl readline sqlite3 xz gdbm zlib pyenv wget swig coreutils llvm git-lfs ninja"
for pak in ${BREW_PACKAGE} for pak in ${BREW_PACKAGE}
do do
echo "###### do command: brew install ${pak}" echo "###### do command: brew install ${pak}"
......
scripts/whl/manylinux2014/build_wheel_common.sh
浏览文件 @ baa30400
...@@ -50,12 +50,13 @@ elif [ $SDK_NAME == "cu111" ];then ...@@ -50,12 +50,13 @@ elif [ $SDK_NAME == "cu111" ];then
${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\ ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
${CUDNN_LIB_DIR}/libcudnn.so.8" ${CUDNN_LIB_DIR}/libcudnn.so.8"
EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \ EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-gencode arch=compute_61,code=sm_61 \ -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
arch=compute_70,code=sm_70 \ -gencode arch=compute_70,code=sm_70 \
arch=compute_75,code=sm_75 \ -gencode arch=compute_75,code=sm_75 \
arch=compute_80,code=sm_80 \ -gencode arch=compute_80,code=sm_80 \
arch=compute_86,code=sm_86 \ -gencode arch=compute_86,code=sm_86 \
arch=compute_86,code=compute_86" -gencode arch=compute_86,code=compute_86\" "
REQUIR_CUDA_VERSION="11010" REQUIR_CUDA_VERSION="11010"
REQUIR_CUDNN_VERSION="8.0.4" REQUIR_CUDNN_VERSION="8.0.4"
REQUIR_TENSORRT_VERSION="7.2.2.3" REQUIR_TENSORRT_VERSION="7.2.2.3"
...@@ -73,12 +74,13 @@ elif [ $SDK_NAME == "cu112" ];then ...@@ -73,12 +74,13 @@ elif [ $SDK_NAME == "cu112" ];then
${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\ ${CUDNN_LIB_DIR}/libcudnn_ops_train.so.8:\
${CUDNN_LIB_DIR}/libcudnn.so.8" ${CUDNN_LIB_DIR}/libcudnn.so.8"
EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \ EXTRA_CMAKE_FLAG=" -DMGE_WITH_CUDNN_SHARED=ON -DMGE_WITH_CUBLAS_SHARED=ON \
-gencode arch=compute_61,code=sm_61 \ -DMGE_CUDA_GENCODE=\"-gencode arch=compute_61,code=sm_61 \
arch=compute_70,code=sm_70 \ -gencode arch=compute_70,code=sm_70 \
arch=compute_75,code=sm_75 \ -gencode arch=compute_75,code=sm_75 \
arch=compute_80,code=sm_80 \ -gencode arch=compute_80,code=sm_80 \
arch=compute_86,code=sm_86 \ -gencode arch=compute_86,code=sm_86 \
arch=compute_86,code=compute_86" -gencode arch=compute_86,code=compute_86\" "
REQUIR_CUDA_VERSION="11020" REQUIR_CUDA_VERSION="11020"
REQUIR_CUDNN_VERSION="8.0.4" REQUIR_CUDNN_VERSION="8.0.4"
REQUIR_TENSORRT_VERSION="7.2.2.3" REQUIR_TENSORRT_VERSION="7.2.2.3"
......
scripts/whl/manylinux2014/do_build_common.sh
浏览文件 @ baa30400
...@@ -75,33 +75,66 @@ then ...@@ -75,33 +75,66 @@ then
fi fi
SRC_DIR=$(readlink -f "`dirname $0`/../../../") SRC_DIR=$(readlink -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/ BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_ON/MGE_INFERENCE_ONLY_OFF/Release/build/
fi fi
# here we just treat cu file should not in the increment build file list
INCREMENT_KEY_WORDS=".cu.o is dirty"
IS_IN_FIRST_LOOP=TRUE
for ver in ${ALL_PYTHON} for ver in ${ALL_PYTHON}
do do
# we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi
python_ver=${ver:0:2} python_ver=${ver:0:2}
MAJOR=${python_ver:0:1} MAJOR=${python_ver:0:1}
MINOR=${ver:1} MINOR=${ver:1}
PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/ PYTHON_DIR=/opt/python/cp${python_ver}-cp${ver}/
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} ${EXTRA_CMAKE_FLAG}" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} ${EXTRA_CMAKE_FLAG}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_PREFIX_PATH=${PYTHON_DIR}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_EXECUTABLE=${PYTHON_DIR}/bin/python3"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_LIBRARY=${PYTHON_DIR}lib/"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DPYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python${MAJOR}.${MINOR}"
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON" export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_ATLAS=ON"
# TODO: after change to Ninja, only the fisrt loop need add -r to save build time if [ -d "${BUILD_DIR}" ]; then
# insure rm have args
touch ${BUILD_DIR}/empty.so
touch ${BUILD_DIR}/CMakeCache.txt
find ${BUILD_DIR} -name "*.so" | xargs rm
# as we now use increment build mode when switch python
# But I do not known any more issue at CMakeLists.txt or not
# so Force remove CMakeCache.txt
find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi
HOST_BUILD_ARGS="-t -s"
if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
${SRC_DIR}/scripts/cmake-build/host_build.sh -c -t -r HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c"
else fi
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
# call ninja dry run and check increment is invalid or not
if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi fi
# call real build
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
# check python api call setup.py
cd ${BUILD_DIR} cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging rm -rf staging
mkdir -p staging mkdir -p staging
cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
...@@ -128,4 +161,5 @@ do ...@@ -128,4 +161,5 @@ do
# compat for root-less docker env to remove output at host side # compat for root-less docker env to remove output at host side
chmod -R 777 . chmod -R 777 .
echo "python $ver done" echo "python $ver done"
IS_IN_FIRST_LOOP=FALSE
done done
scripts/whl/manylinux2014/init_image.sh
浏览文件 @ baa30400
...@@ -7,8 +7,11 @@ LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz' ...@@ -7,8 +7,11 @@ LLVM_URL='https://github.com/llvm-mirror/llvm/archive/release_60.tar.gz'
CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz' CLANG_URL='https://github.com/llvm-mirror/clang/archive/release_60.tar.gz'
yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 yum install -y pcre-devel devtoolset-9-libatomic-devel.x86_64 ninja-build
yum install -y devtoolset-8 devtoolset-8-libatomic-devel.x86_64 yum install -y devtoolset-8 devtoolset-8-libatomic-devel.x86_64
# install a default python3 for cmake PYTHON3_EXECUTABLE_WITHOUT_VERSION
yum install -y python3
python3 -m pip install numpy
for ver in 35m 36m 37m 38 for ver in 35m 36m 37m 38
do do
......
scripts/whl/utils/utils.sh 0 → 100755
浏览文件 @ baa30400
#!/usr/bin/env bash
set -e
OS=$(uname -s)
function ninja_dry_run_and_check_increment() {
if [ $# -eq 3 ]; then
_BUILD_SHELL=$1
_BUILD_FLAGS="$2 -n"
_INCREMENT_KEY_WORDS=$3
else
echo "err call ninja_dry_run_and_check_increment"
exit -1
fi
${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log
DIRTY_LOG=`cat dry_run.log`
if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
echo "python3 switch increment build failed, some MR make a wrong CMakeLists.txt depends"
echo "or build env can not find default python3 in PATH env"
echo "please refs for PYTHON3_EXECUTABLE_WITHOUT_VERSION define at SRC_ROOT/CMakeLists.txt"
echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
exit -1
fi
}
PYTHON_API_INCLUDES=""
function check_build_ninja_python_api() {
INCLUDE_KEYWORD=""
IS_MINOR_HIT=FALSE
if [ $# -eq 1 ]; then
ver=$1
echo "org args: ${ver}"
if [[ $OS =~ "NT" ]]; then
INCLUDE_KEYWORD="${ver}\\\\include"
PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include"
elif [[ $OS =~ "Linux" ]]; then
INCLUDE_KEYWORD="include/python3.${ver:1:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
elif [[ $OS =~ "Darwin" ]]; then
INCLUDE_KEYWORD="include/python3.${ver:2:1}"
PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8"
else
echo "unknown OS: ${OS}"
exit -1
fi
else
echo "err call check_build_ninja_python_api"
exit -1
fi
echo "try check python INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} is invalid in ninja.build or not"
NINJA_BUILD=`cat build.ninja`
for PYTHON_API_INCLUDE in ${PYTHON_API_INCLUDES}
do
echo "check PYTHON_API_INCLUDE vs INCLUDE_KEYWORD : (${PYTHON_API_INCLUDE} : ${INCLUDE_KEYWORD})"
if [ ${PYTHON_API_INCLUDE} = ${INCLUDE_KEYWORD} ]; then
if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
echo "hit INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
IS_MINOR_HIT="TRUE"
else
echo "Err happened can not find INCLUDE_KEYWORD: ${INCLUDE_KEYWORD} in build.ninja"
exit -1
fi
else
if [[ "${NINJA_BUILD}" =~ ${PYTHON_API_INCLUDE} ]]; then
echo "Err happened: find PYTHON_API_INCLUDE: ${PYTHON_API_INCLUDE} in build.ninja"
echo "But now INCLUDE_KEYWORD: ${INCLUDE_KEYWORD}"
exit -1
fi
fi
done
if [ ${IS_MINOR_HIT} = "FALSE" ]; then
echo "Err happened, can not hit any MINOR api in ninja.build"
exit -1
fi
}
scripts/whl/windows/windows_build_whl.sh
浏览文件 @ baa30400
...@@ -23,6 +23,8 @@ function append_path_env_and_check() { ...@@ -23,6 +23,8 @@ function append_path_env_and_check() {
append_path_env_and_check append_path_env_and_check
SRC_DIR=$(READLINK -f "`dirname $0`/../../../") SRC_DIR=$(READLINK -f "`dirname $0`/../../../")
source ${SRC_DIR}/scripts/whl/utils/utils.sh
ALL_PYTHON=${ALL_PYTHON} ALL_PYTHON=${ALL_PYTHON}
FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3" FULL_PYTHON_VER="3.5.4 3.6.8 3.7.7 3.8.3"
if [[ -z ${ALL_PYTHON} ]] if [[ -z ${ALL_PYTHON} ]]
...@@ -43,12 +45,12 @@ mkdir -p ${WINDOWS_WHL_HOME} ...@@ -43,12 +45,12 @@ mkdir -p ${WINDOWS_WHL_HOME}
function config_python_env() { function config_python_env() {
PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1 PYTHON_DIR=/c/Users/${USER}/mge_whl_python_env/$1
PYTHON_BIN=${PYTHON_DIR} PYTHON_BIN=${PYTHON_DIR}
if [ ! -f "$PYTHON_BIN/python3.exe" ]; then if [ ! -f "${PYTHON_BIN}/python3.exe" ]; then
echo "ERR: can not find $PYTHON_BIN , Invalid python package" echo "ERR: can not find $PYTHON_BIN , Invalid python package"
echo "now support list: ${FULL_PYTHON_VER}" echo "now support list: ${FULL_PYTHON_VER}"
err_env err_env
else else
echo "put python3 to env..." echo "put ${PYTHON_BIN}/python3.exe to env..."
export PATH=${PYTHON_BIN}:$PATH export PATH=${PYTHON_BIN}:$PATH
which python3 which python3
fi fi
...@@ -104,11 +106,24 @@ function copy_more_dll() { ...@@ -104,11 +106,24 @@ function copy_more_dll() {
fi fi
} }
BUILD_DIR=${SRC_DIR}/build_dir/host/build/
# here we just treat cu file should not in the increment build file list
INCREMENT_KEY_WORDS=".cu.obj is dirty"
IS_IN_FIRST_LOOP=TRUE
function do_build() { function do_build() {
for ver in ${ALL_PYTHON} for ver in ${ALL_PYTHON}
do do
#config # we want run a full clean build at the first loop
if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then
# TODO: may all cmake issue can be resolved after rm CMakeCache?
# if YES, remove this to use old cache and speed up CI
echo "warning: remove old build_dir for the first loop"
rm -rf ${BUILD_DIR}
fi
#config python3
config_python_env ${ver} config_python_env ${ver}
#check env #check env
...@@ -122,35 +137,44 @@ function do_build() { ...@@ -122,35 +137,44 @@ function do_build() {
fi fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}" echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}" echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
#FIXME: ninja handle err with cmake 3.17 when assgin PYTHON_LIBRARY
#But after put python3.exe to HEAD of PATH by config_python_env, cmake can also handle the
#right PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, at the same time, clang-cl need swig target
#force LINK a real PYTHON_LIBRARY file, after test we do not find the symbols conflict with python
#export EXTRA_CMAKE_ARGS="-DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc #config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo " export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo "
#call build and install #call build and install
#FIXME: cmake do not triger update python config, after HOST_BUILD_ARGS=" -t -s"
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
#-r to remove build cache after a new ver build, which
#will be more slow build than without -r
BUILD_ARGS=" -t -r"
if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then if [ ${BUILD_WHL_CPU_ONLY} = "OFF" ]; then
echo "build windows whl with cuda" echo "build windows whl with cuda"
BUILD_ARGS="${BUILD_ARGS} -c " HOST_BUILD_ARGS="${HOST_BUILD_ARGS} -c "
else else
echo "build windows whl with cpu only" echo "build windows whl with cpu only"
fi fi
echo "host_build.sh BUILD_ARGS: ${BUILD_ARGS}" if [ -d "${BUILD_DIR}" ]; then
${SRC_DIR}/scripts/cmake-build/host_build.sh ${BUILD_ARGS} # insure rm have args
touch ${BUILD_DIR}/empty.pyd
touch ${BUILD_DIR}/CMakeCache.txt
/usr/bin/find ${BUILD_DIR} -name "*.pyd" | xargs rm
# ninja/cmake on windows will handle error if just export
# PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
# But after put python3.exe to HEAD of PATH by config_python_env
# and force remove CMakeCache.txt, ninja/cmake will auto update
# PYTHON_LIBRARY/PYTHON_INCLUDE_DIR/PYTHON_EXECUTABLE
/usr/bin/find ${BUILD_DIR} -name CMakeCache.txt | xargs rm
fi
echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}"
BUILD_DIR=${SRC_DIR}/build_dir/host/build/ # call ninja dry run and check increment is invalid or not
cd ${BUILD_DIR} if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then
ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}"
fi
#call real build
${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS}
# check python api call setup.py
cd ${BUILD_DIR}
check_build_ninja_python_api ${ver}
rm -rf staging rm -rf staging
mkdir -p staging mkdir -p staging
cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
...@@ -178,6 +202,7 @@ function do_build() { ...@@ -178,6 +202,7 @@ function do_build() {
echo "windows whl package location: ${WINDOWS_WHL_HOME}" echo "windows whl package location: ${WINDOWS_WHL_HOME}"
ls ${WINDOWS_WHL_HOME} ls ${WINDOWS_WHL_HOME}
echo "##############################################################################################" echo "##############################################################################################"
IS_IN_FIRST_LOOP=FALSE
done done
} }
......
MegRay @ eb8365c1
比较 678ecceb...eb8365c1
Subproject commit 678ecceb5b4b40933dc4be512a155030b182351c Subproject commit eb8365c1015624348dbbb0a3ed97eecb40643e9e
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或