Skip to content

MegEngine
MegEngine

MegEngine 天元 / MegEngine
1 年多 前同步成功

通知 403
Star 4705
Fork 582
MegEngine
MegEngine
提交 4d541b72 编写于 作者: M Megvii Engine Team 提交者： Xu Xinran
浏览文件
操作

feat(osx/python_whl): make macos HAPPY for build megbrain python package 

GitOrigin-RevId: 7ffd9b95416d629442c9ab9e0a79fcf2ac2cac42
上级 3bfb704e
隐藏空白更改
内联 并排
Showing with 206 addition and 55 deletion
python_module/src/cpp/megbrain_config.cpp
浏览文件 @ 4d541b72
......@@ -274,7 +274,8 @@ void _config::load_opr_library(const char* self_path, const char* lib_path) {
}
}
std::vector<std::pair<uint64_t, std::string>> _config::dump_registered_oprs() {
std::vector<std::pair<unsigned long int, std::string>>
_config::dump_registered_oprs() {
#if MGB_ENABLE_DEBUG_UTIL
return serialization::OprRegistry::dump_registries();
#else
......
python_module/src/cpp/megbrain_config.h
浏览文件 @ 4d541b72
......@@ -62,7 +62,7 @@ class _config {
static void load_opr_library(
const char* self_path, const char* lib_path);
static std::vector<std::pair<uint64_t, std::string>>
static std::vector<std::pair<unsigned long int, std::string>>
dump_registered_oprs();
static int create_mm_server(const std::string& server_addr, int port);
......
scripts/cmake-build/cross_build_android_arm_inference.sh
浏览文件 @ 4d541b72
......@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_ARMV8_2_FEATURE_DOTPROD=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a
REMOVE_OLD_BUILD=false
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() {
echo "$0 args1 args2 .."
......@@ -16,12 +18,14 @@ function usage() {
echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}
while getopts "khdfpa:" arg
while getopts "rkhdfpa:" arg
do
case $arg in
d)
......@@ -60,6 +64,10 @@ do
echo "show usage"
usage
;;
r)
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
?)
echo "unkonw argument"
usage
......@@ -87,7 +95,7 @@ elif [[ $OS =~ "NT" ]]; then
fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
source $SRC_DIR/scripts/cmake-build/utils/utils.sh
if [ -z $NDK_ROOT ];then
echo "can not find NDK_ROOT env, pls export you NDK root dir to NDK_ROOT"
......@@ -105,14 +113,7 @@ function cmake_build() {
echo "build ABI: $BUILD_ABI"
echo "build native level: $BUILD_NATIVE_LEVEL"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir"
mkdir -p $BUILD_DIR
......@@ -130,6 +131,7 @@ function cmake_build() {
-DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
make -j$(nproc)
......
scripts/cmake-build/cross_build_ios_arm_inference.sh
浏览文件 @ 4d541b72
......@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_ARMV8_2_FEATURE_DOTPROD=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64
REMOVE_OLD_BUILD=false
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() {
echo "$0 args1 args2 .."
......@@ -16,12 +18,14 @@ function usage() {
echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}
while getopts "khdfpa:" arg
while getopts "rkhdfpa:" arg
do
case $arg in
d)
......@@ -60,6 +64,10 @@ do
echo "show usage"
usage
;;
r)
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
?)
echo "unkonw argument"
usage
......@@ -86,7 +94,7 @@ else
fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
source $SRC_DIR/scripts/cmake-build/utils/utils.sh
function cmake_build() {
BUILD_DIR=$SRC_DIR/build_dir/apple/$3/$4/$1/$BUILD_TYPE/build
......@@ -102,14 +110,7 @@ function cmake_build() {
echo "build OS_PLATFORM: $OS_PLATFORM"
echo "build XCODE_IOS_PLATFORM: $XCODE_IOS_PLATFORM"
echo "build IOS_ARCH: $IOS_ARCH"
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir"
mkdir -p $BUILD_DIR
......@@ -128,6 +129,7 @@ function cmake_build() {
-DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
make -j$(nproc)
......
scripts/cmake-build/cross_build_linux_arm_inference.sh
浏览文件 @ 4d541b72
......@@ -7,6 +7,8 @@ MGE_ARMV8_2_FEATURE_FP16=OFF
MGE_ARMV8_2_FEATURE_DOTPROD=OFF
MGE_DISABLE_FLOAT16=OFF
ARCH=arm64-v8a
REMOVE_OLD_BUILD=false
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
function usage() {
echo "$0 args1 args2 .."
......@@ -16,12 +18,14 @@ function usage() {
echo "-p : enable MGE_ARMV8_2_FEATURE_DOTPROD for ARM64, need toolchain and hardware support"
echo "-k : open MGE_DISABLE_FLOAT16 for NEON "
echo "-a : config build arch available: ${ARCHS[@]}"
echo "-r : remove old build dir before make, default off"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}
while getopts "khdfpa:" arg
while getopts "rkhdfpa:" arg
do
case $arg in
d)
......@@ -60,6 +64,10 @@ do
echo "show usage"
usage
;;
r)
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
?)
echo "unkonw argument"
usage
......@@ -87,7 +95,7 @@ elif [[ $OS =~ "NT" ]]; then
fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/host_build_flatc.sh
source $SRC_DIR/scripts/cmake-build/utils/utils.sh
function cmake_build() {
BUILD_DIR=$SRC_DIR/build_dir/gnu-linux/$1/$BUILD_TYPE/build
......@@ -98,14 +106,7 @@ function cmake_build() {
echo "build type: $BUILD_TYPE"
echo "build toolchain: $TOOLCHAIN"
echo "BUILD MAKEFILE_TYPE: $MAKEFILE_TYPE"
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir"
mkdir -p $BUILD_DIR
......@@ -120,6 +121,7 @@ function cmake_build() {
-DMGE_ARMV8_2_FEATURE_DOTPROD=$MGE_ARMV8_2_FEATURE_DOTPROD \
-DMGE_DISABLE_FLOAT16=$MGE_DISABLE_FLOAT16 \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
make -j$(nproc)
......
scripts/cmake-build/host_build.sh
浏览文件 @ 4d541b72
......@@ -8,7 +8,9 @@ function usage() {
echo "-c : Build with CUDA, default without CUDA"
echo "-t : Build with training mode, default inference only"
echo "-m : Build with m32 mode(only for windows build), default m64"
echo "-r : remove old build dir before make, default off"
echo "-h : show usage"
echo "append other cmake config by export EXTRA_CMAKE_ARGS=..."
echo "example: $0 -d"
exit -1
}
......@@ -19,8 +21,10 @@ MGE_INFERENCE_ONLY=ON
MGE_WINDOWS_BUILD_ARCH=x64
MGE_WINDOWS_BUILD_MARCH=m64
MGE_ARCH=x86_64
REMOVE_OLD_BUILD=false
echo "EXTRA_CMAKE_ARGS: ${EXTRA_CMAKE_ARGS}"
while getopts "hdctm" arg
while getopts "rhdctm" arg
do
case $arg in
d)
......@@ -39,6 +43,10 @@ do
echo "show usage"
usage
;;
r)
echo "config REMOVE_OLD_BUILD=true"
REMOVE_OLD_BUILD=true
;;
m)
echo "build for m32(only use for windows)"
MGE_WINDOWS_BUILD_ARCH=x86
......@@ -71,6 +79,7 @@ elif [[ $OS =~ "NT" ]]; then
fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
source $SRC_DIR/scripts/cmake-build/utils/utils.sh
function cmake_build() {
BUILD_DIR=$SRC_DIR/build_dir/host/MGE_WITH_CUDA_$1/MGE_INFERENCE_ONLY_$2/$3/build
......@@ -83,14 +92,7 @@ function cmake_build() {
echo "build type: $BUILD_TYPE"
echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir"
mkdir -p $BUILD_DIR
......@@ -101,6 +103,7 @@ function cmake_build() {
-DMGE_INFERENCE_ONLY=$MGE_INFERENCE_ONLY \
-DMGE_WITH_CUDA=$MGE_WITH_CUDA \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \
${EXTRA_CMAKE_ARGS} \
$SRC_DIR
make -j$(nproc)
......@@ -156,7 +159,7 @@ function prepare_env_for_windows_build() {
echo "Ninja valid ..."
else
echo "Ninja Invalid: ..."
windows_env_er
windows_env_err
fi
export PATH=$VS_PATH/VC/Auxiliary/Build:$PATH
......@@ -176,14 +179,7 @@ function cmake_build_windows() {
echo "build type: $BUILD_TYPE"
echo "MGE_WITH_CUDA: $MGE_WITH_CUDA"
echo "MGE_INFERENCE_ONLY: $MGE_INFERENCE_ONLY"
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
try_remove_old_build $REMOVE_OLD_BUILD $BUILD_DIR $INSTALL_DIR
echo "create build dir"
mkdir -p $BUILD_DIR
......@@ -202,6 +198,7 @@ function cmake_build_windows() {
-DCMAKE_C_COMPILER=clang-cl.exe \
-DCMAKE_CXX_COMPILER=clang-cl.exe \
-DCMAKE_MAKE_PROGRAM=ninja.exe \
${EXTRA_CMAKE_ARGS} \
../../.. && \
echo \"start Ninja build log to build.log, may take serval min...\" && \
Ninja load_and_run > build.log"
......
scripts/cmake-build/macos_build_whl.sh 0 → 100755
浏览文件 @ 4d541b72
#!/bin/bash -e
######################################################################################################################
# macos build whl env prepare #
# 1: install xcodebuild for host-build #
# 2: install brew: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" #
# 3: build/install cmake3-14.4 #
# wget https://cmake.org/files/v3.14/cmake-3.14.4.tar.gz #
# tar -xzvf cmake-3.14.4.tar.gz;cd cmake-3.14.4; #
# ./configure; make -j32; sudo make install #
# #
# 4: brew install wget python swig coreutils llvm #
# echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> ~/.zshrc (if u use other shell, chang this) #
# #
# 5: brew install python@3.8 (sub version may different) #
# /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3 install numpy #
# #
# 6: brew install python@3.7 (sub version may different) #
# /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3 install numpy #
# #
# 7: make link for easy use python3.x (not build necessary) #
# ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/pip3.7 /usr/local/bin/pip3.7 #
# ln -s /usr/local/Cellar/python@3.7/3.7.8_1/bin/python3.7 /usr/local/bin/python3.7 #
# ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/pip3.8 /usr/local/bin/pip3.8 #
# ln -s /usr/local/Cellar/python@3.8/3.8.3_2/bin/python3.8 /usr/local/bin/python3.8 #
######################################################################################################################
READLINK=readlink
OS=$(uname -s)
if [ $OS = "Darwin" ];then
READLINK=greadlink
else
echo "ERR: only run at macos env"
exit -1
fi
SRC_DIR=$($READLINK -f "`dirname $0`/../../")
ALL_PYTHON=${ALL_PYTHON}
if [[ -z ${ALL_PYTHON} ]]
then
#FIXME: on catalina brew only official support 3.7 and 3.8
ALL_PYTHON="37 38"
fi
PYTHON_DIR=
PYTHON_LIBRARY=
PYTHON_INCLUDE_DIR=
function config_python_env() {
if [[ "$1" -eq "38" ]]; then
PYTHON_DIR=/usr/local/Cellar/python@3.8/3.8.3_2/Frameworks/Python.framework/Versions/3.8/
PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.8.dylib
PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.8
elif [[ "$1" -eq "37" ]]; then
PYTHON_DIR=/usr/local/Cellar/python@3.7/3.7.8_1/Frameworks/Python.framework/Versions/3.7/
PYTHON_LIBRARY=${PYTHON_DIR}lib/libpython3.7.dylib
PYTHON_INCLUDE_DIR=${PYTHON_DIR}include/python3.7m
else
echo "ERR: DO NOT SUPPORT PYTHON VERSION"
exit -1
fi
}
MACOS_WHL_HOME=${SRC_DIR}/scripts/cmake-build/macos_whl_home
if [ -e "${MACOS_WHL_HOME}" ]; then
echo "remove old macos whl file"
rm -rf ${MACOS_WHL_HOME}
fi
mkdir -p ${MACOS_WHL_HOME}
for ver in ${ALL_PYTHON}
do
#config
config_python_env ${ver}
#check env
if [ ! -f "$PYTHON_LIBRARY" ]; then
echo "ERR: can not find $PYTHON_LIBRARY , Invalid python package"
exit -1
fi
if [ ! -d "$PYTHON_INCLUDE_DIR" ]; then
echo "ERR: can not find $PYTHON_INCLUDE_DIR , Invalid python package"
exit -1
fi
echo "PYTHON_LIBRARY: ${PYTHON_LIBRARY}"
echo "PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
#append cmake args for config python
export EXTRA_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${PYTHON_DIR} -DPYTHON_LIBRARY=${PYTHON_LIBRARY} -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} "
#config build type to RelWithDebInfo to enable MGB_ENABLE_DEBUG_UTIL etc
export EXTRA_CMAKE_ARGS=${EXTRA_CMAKE_ARGS}" -DCMAKE_BUILD_TYPE=RelWithDebInfo "
#call build and install
#FIXME: cmake do not triger update python config, after
#change PYTHON_LIBRARY and PYTHON_INCLUDE_DIR, so add
#-r to remove build cache after a new ver build, which
#will be more slow build than without -r
${SRC_DIR}/scripts/cmake-build/host_build.sh -t -r
#call setup.py
BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/
cd ${BUILD_DIR}
if [ -d "staging" ]; then
echo "remove old build cache file"
rm -rf staging
fi
mkdir -p staging
cp -a python_module/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/
cd ${BUILD_DIR}/staging/megengine/_internal
#FIXME: set lib suffix to dylib may be better, BUT we find after distutils.file_util.copy_file
#will change to .so at macos even we set suffix to dylib, at the same time, macos also support .so
llvm-strip -s _mgb.so
cd ${BUILD_DIR}/staging
${PYTHON_DIR}/bin/python3 setup.py bdist_wheel
cp ${BUILD_DIR}/staging/dist/Meg*.whl ${MACOS_WHL_HOME}/
echo ""
echo "##############################################################################################"
echo "macos whl package location: ${MACOS_WHL_HOME}"
ls ${MACOS_WHL_HOME}
echo "##############################################################################################"
done
scripts/cmake-build/utils/host_build_flatc.sh → scripts/cmake-build/utils/utils.sh
浏览文件 @ 4d541b72
......@@ -37,3 +37,23 @@ function build_flatc() {
make -j$(nproc)
make install/strip
}
function try_remove_old_build() {
REMOVE_OLD_BUILD=$1
echo $REMOVE_OLD_BUILD
BUILD_DIR=$2
INSTALL_DIR=$3
if [ $REMOVE_OLD_BUILD = "true" ]; then
if [ -e $BUILD_DIR ];then
echo "clean old dir: $BUILD_DIR"
rm -rf $BUILD_DIR
fi
if [ -e $INSTALL_DIR ];then
echo "clean old dir: $INSTALL_DIR"
rm -rf $INSTALL_DIR
fi
else
echo "strip remove old build"
fi
}
sdk/c-opr-loaders/mace/README.md
浏览文件 @ 4d541b72
......@@ -4,7 +4,7 @@
```bash
cd $MEGENGINE_HOME
./scripts/cmake-build/cross_build_android_arm_inference.sh -a arm64-v8a
./scripts/cmake-build/cross_build_android_arm_inference.sh -a arm64-v8a -r
```
After successfully built, load_and_run should be in `$MEGENGINE_HOME/build_dir/android/arm64-v8a/Release/install/bin`
......
src/serialization/impl/opr_registry.cpp
浏览文件 @ 4d541b72
......@@ -159,9 +159,10 @@ void OprRegistry::add_using_dynamic_loader(
}
#if MGB_ENABLE_DEBUG_UTIL
std::vector<std::pair<uint64_t, std::string>> OprRegistry::dump_registries() {
auto &&id2reg = static_data().id2reg;
std::vector<std::pair<uint64_t, std::string>> result;
std::vector<std::pair<unsigned long int, std::string>>
OprRegistry::dump_registries() {
auto&& id2reg = static_data().id2reg;
std::vector<std::pair<unsigned long int, std::string>> result;
for (auto iter = id2reg.begin(); iter != id2reg.end(); ++iter) {
if (iter->second.name.size() == 0)
result.push_back({iter->first, "<special>"});
......
src/serialization/include/megbrain/serialization/opr_registry.h
浏览文件 @ 4d541b72
......@@ -76,7 +76,8 @@ namespace serialization {
#if MGB_ENABLE_DEBUG_UTIL
//! dump registered oprs
static std::vector<std::pair<uint64_t, std::string>> dump_registries();
static std::vector<std::pair<unsigned long int, std::string>>
dump_registries();
#endif
};
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或