From 3ddc32d3e31cc85c5fc9d2c65f3c0ebc42a4595e Mon Sep 17 00:00:00 2001 From: Megvii Engine Team Date: Tue, 11 Jan 2022 17:44:43 +0800 Subject: [PATCH] feat(android/whl): support android whl GitOrigin-RevId: 05df16b49468fff521a4cce336e1bc7a6cdbf9be --- imperative/CMakeLists.txt | 4 + imperative/python/megengine/__init__.py | 11 ++ .../python/megengine/data/dataloader.py | 8 + scripts/whl/BUILD_PYTHON_WHL_README.md | 30 +++- scripts/whl/android/android_build_whl.sh | 164 ++++++++++++++++++ scripts/whl/android/android_opencv_python.sh | 65 +++++++ .../whl/android/android_whl_env_prepare.sh | 155 +++++++++++++++++ .../whl/android/cv_patch/host_android.patch | 75 ++++++++ .../android/patchs/Lib-subprocess.py.patch | 14 ++ .../whl/android/patchs/Lib-tmpfile.py.patch | 13 ++ .../patchs/distutils-command-build.py.patch | 11 ++ .../android/patchs/no-setuid-servers.patch | 36 ++++ .../whl/android/up_3_9_patch/fix-paths.patch | 59 +++++++ scripts/whl/android/utils.sh | 43 +++++ scripts/whl/utils/utils.sh | 19 +- 15 files changed, 699 insertions(+), 8 deletions(-) create mode 100755 scripts/whl/android/android_build_whl.sh create mode 100755 scripts/whl/android/android_opencv_python.sh create mode 100755 scripts/whl/android/android_whl_env_prepare.sh create mode 100644 scripts/whl/android/cv_patch/host_android.patch create mode 100644 scripts/whl/android/patchs/Lib-subprocess.py.patch create mode 100644 scripts/whl/android/patchs/Lib-tmpfile.py.patch create mode 100644 scripts/whl/android/patchs/distutils-command-build.py.patch create mode 100644 scripts/whl/android/patchs/no-setuid-servers.patch create mode 100644 scripts/whl/android/up_3_9_patch/fix-paths.patch create mode 100755 scripts/whl/android/utils.sh diff --git a/imperative/CMakeLists.txt b/imperative/CMakeLists.txt index 3d1226637..76d35e3a9 100644 --- a/imperative/CMakeLists.txt +++ b/imperative/CMakeLists.txt @@ -47,6 +47,10 @@ else() ${MODULE_NAME} PRIVATE megengine_shared -Wl,--version-script=${MGE_VERSION_SCRIPT}) endif() +if(ANDROID) + target_link_libraries(${MODULE_NAME} PRIVATE ${PYTHON_LIBRARIES}) +endif() + add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/range-v3 ${PROJECT_BINARY_DIR}/third_party/range-v3) target_link_libraries(${MODULE_NAME} PRIVATE range-v3) diff --git a/imperative/python/megengine/__init__.py b/imperative/python/megengine/__init__.py index cee58727e..bed7694db 100644 --- a/imperative/python/megengine/__init__.py +++ b/imperative/python/megengine/__init__.py @@ -12,6 +12,17 @@ import os import platform import sys +if os.getenv("TERMUX_VERSION"): + try: + import cv2 + except Exception as exc: + print("Run MegEngine python interface at Android/Termux env") + print("!!!You need build opencv-python manually!!!, by run sh:") + print( + "https://github.com/MegEngine/MegEngine/blob/master/scripts/whl/android/android_opencv_python.sh" + ) + raise exc + if sys.platform == "win32": lib_path = os.path.join(os.path.dirname(__file__), "core/lib") dll_paths = list(filter(os.path.exists, [lib_path,])) diff --git a/imperative/python/megengine/data/dataloader.py b/imperative/python/megengine/data/dataloader.py index e63b36b94..7cfd3f2ca 100644 --- a/imperative/python/megengine/data/dataloader.py +++ b/imperative/python/megengine/data/dataloader.py @@ -10,6 +10,7 @@ import collections import gc import math import multiprocessing +import os import platform import queue import random @@ -150,6 +151,13 @@ class DataLoader: "pyarrow.plasma does not support ParallelDataLoader on windows, changing num_workers to be zero" ) self.num_workers = 0 + if os.getenv("TERMUX_VERSION"): + # FIXME: termux install pyarrow will build error now + # remove this logic after pyarrow fix this issue + print( + "pyarrow do not support on termux env now, changing num_workers to be zero" + ) + self.num_workers = 0 if isinstance(self.dataset, StreamDataset): if not self.num_workers: return _SerialStreamDataLoaderIter(self, self.preload) diff --git a/scripts/whl/BUILD_PYTHON_WHL_README.md b/scripts/whl/BUILD_PYTHON_WHL_README.md index 5eca30db7..f9f02eb02 100755 --- a/scripts/whl/BUILD_PYTHON_WHL_README.md +++ b/scripts/whl/BUILD_PYTHON_WHL_README.md @@ -2,6 +2,7 @@ * Windows build (cpu and gpu) * Linux build (cpu and gpu) * MacOS build (cpu only) +* Android(termux) build (cpu only) # Build env prepare ## Linux @@ -30,6 +31,15 @@ ./scripts/whl/macos/macos_whl_env_prepare.sh ``` +## Android +* install [termux](https://termux.com/) apk on Android Device + * at least 8G DDR + * at least Android 7 +* init wheel build-dependent env by command: +```bash +./scripts/whl/android/android_whl_env_prepare.sh +``` + ## Windows * refer to [BUILD_README.md](../cmake-build/BUILD_README.md) Windows section to init base build environment @@ -43,12 +53,12 @@ commands: ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cu101 ``` -* And you can find all of the outputs in `output` directory.If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg: +* And you can find all of the outputs in `output` directory.If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as: ```bash ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cu101 ``` -* If you just want to build with cpu only version, you can set `-sdk` environment 'cpu'. eg: +* If you just want to build with cpu only version, you can set `-sdk` environment 'cpu'. such as: ```bash ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cpu ``` @@ -58,7 +68,7 @@ ALL_PYTHON="36m" ./scripts/whl/manylinux2014/build_wheel_common.sh -sdk cpu ```bash ./scripts/whl/macos/macos_build_whl.sh ``` -* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg: +* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as: ```bash ALL_PYTHON="3.7.7" ./scripts/whl/macos/macos_build_whl.sh ``` @@ -69,12 +79,22 @@ ALL_PYTHON="3.7.7" ./scripts/whl/macos/macos_build_whl.sh ./scripts/whl/windows/windows_build_whl.sh ``` -* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. eg: +* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as: ```bash ALL_PYTHON="3.8.3" ./scripts/whl/windows/windows_build_whl.sh ``` -* If you just want to build with cpu only version, you can set `BUILD_WHL_CPU_ONLY` environment 'ON'. eg: +* If you just want to build with cpu only version, you can set `BUILD_WHL_CPU_ONLY` environment 'ON'. such as: ``` BUILD_WHL_CPU_ONLY="ON" ALL_PYTHON="3.8.3" ./scripts/whl/windows/windows_build_whl.sh ``` + +## Build for Android +* commands: +```bash +scripts/whl/android/android_build_whl.sh +``` +* If you just want to build for a specific Python verison, you can use `ALL_PYTHON` environment variable. such as: +```bash +ALL_PYTHON="3.10.1" ./scripts/whl/android/android_build_whl.sh +``` diff --git a/scripts/whl/android/android_build_whl.sh b/scripts/whl/android/android_build_whl.sh new file mode 100755 index 000000000..e04381eb5 --- /dev/null +++ b/scripts/whl/android/android_build_whl.sh @@ -0,0 +1,164 @@ +#!/bin/bash -e + +SRC_DIR=$(readlink -f "`dirname $0`/../../../") +cd ${SRC_DIR} +source scripts/whl/android/utils.sh + +ANDROID_WHL_HOME=${SRC_DIR}/scripts/whl/android/ANDROID_WHL_HOME +if [ -e "${ANDROID_WHL_HOME}" ]; then + echo "remove old android whl file" + rm -rf ${ANDROID_WHL_HOME} +fi +mkdir -p ${ANDROID_WHL_HOME} + +BUILD_DIR=${SRC_DIR}/build_dir/host/MGE_WITH_CUDA_OFF/MGE_INFERENCE_ONLY_OFF/Release/build/ + +# We only handle the case where dnn/src/common/conv_bias.cpp is not in the list of incremental build files. +INCREMENT_KEY_WORDS="conv_bias.cpp.o is dirty" +IS_IN_FIRST_LOOP=TRUE + +ORG_EXTRA_CMAKE_FLAG=${EXTRA_CMAKE_FLAG} + +function handle_strip() { + echo "now handle strip $1" + objcopy --only-keep-debug $1 $1.dbg + strip -s $1 + objcopy --add-gnu-debuglink=$1.dbg $1 + rm $1.dbg +} + +function patch_elf_depend_lib_mgb_mge() { + echo "handle common depend lib for mgb or mge" + LIBS_DIR=${BUILD_DIR}/staging/megengine/core/lib + mkdir -p ${LIBS_DIR} + + patchelf --remove-rpath ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so + patchelf --set-rpath '$ORIGIN/lib' ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so + handle_strip ${BUILD_DIR}/staging/megengine/core/_imperative_rt.so + + cp ${BUILD_DIR}/src/libmegengine_shared.so ${LIBS_DIR} + patchelf --remove-rpath ${LIBS_DIR}/libmegengine_shared.so + patchelf --set-rpath '$ORIGIN/.' ${LIBS_DIR}/libmegengine_shared.so + # FXIME: third_party LLVM need c++_static > 5.1 + # but now clang(13) at termux env do not satisfy it + # may use -static-libstdc++ at CMakeLists.txt after + # upgrade third_party LLVM + cp /data/data/com.termux/files/usr/lib/libc++_shared.so ${LIBS_DIR} + handle_strip ${LIBS_DIR}/libmegengine_shared.so +} + +function patch_elf_depend_lib_megenginelite() { + echo "handle common depend lib for megenginelite" + LIBS_DIR=${BUILD_DIR}/staging/megenginelite/libs + mkdir -p ${LIBS_DIR} + + cp ${BUILD_DIR}/lite/liblite_shared_whl.so ${LIBS_DIR}/ + patchelf --remove-rpath ${LIBS_DIR}/liblite_shared_whl.so + patchelf --set-rpath '$ORIGIN/../../megengine/core/lib' ${LIBS_DIR}/liblite_shared_whl.so + handle_strip ${LIBS_DIR}/liblite_shared_whl.so +} +function do_build() { + mge_python_env_root="${HOME}/mge_python_env" + for ver in ${ALL_PYTHON} + do + python_install_dir=${mge_python_env_root}/${ver}/install + # we want to run a full clean build in the first loop + if [ ${IS_IN_FIRST_LOOP} = "TRUE" ]; then + # TODO: can all cmake issues be resolved after removing CMakeCache? + # if YES, remove this logic to use old cache and speed up CI + echo "warning: remove old build_dir for the first loop" + rm -rf ${BUILD_DIR} + fi + + # insert python3_install_dir into head of PATH to enable CMake find it + if [ -e ${python_install_dir}/bin/python3 ];then + echo "will use ${python_install_dir}/bin/python3 to build mge wheel" + export PATH=${python_install_dir}/bin:$PATH + else + echo "ERROR: can not find python3 in: ${python_install_dir}/bin" + echo "please run: %{SRC_DIR}/scripts/whl/android/android_whl_env_prepare.sh to prepare env" + exit -1 + fi + + export EXTRA_CMAKE_ARGS="${ORG_EXTRA_CMAKE_FLAG} -DCMAKE_BUILD_TYPE=RelWithDebInfo" + export EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DMGE_WITH_CUSTOM_OP=ON" + + if [ -d "${BUILD_DIR}" ]; then + # insure rm have args + touch ${BUILD_DIR}/empty.so + touch ${BUILD_DIR}/CMakeCache.txt + find ${BUILD_DIR} -name "*.so" | xargs rm + # Force remove CMakeCache.txt to avoid error owing to unknown issue in CMakeLists.txt + # which comes from using increment build mode when switching python version + find ${BUILD_DIR} -name CMakeCache.txt | xargs rm + fi + + HOST_BUILD_ARGS="-t -s" + + # call ninja dry run and check increment is invalid or not + if [ ${IS_IN_FIRST_LOOP} = "FALSE" ]; then + ninja_dry_run_and_check_increment "${SRC_DIR}/scripts/cmake-build/host_build.sh" "${HOST_BUILD_ARGS}" "${INCREMENT_KEY_WORDS}" + fi + + # call real build + echo "host_build.sh HOST_BUILD_ARGS: ${HOST_BUILD_ARGS}" + bash ${SRC_DIR}/scripts/cmake-build/host_build.sh ${HOST_BUILD_ARGS} + + # check python api call setup.py + cd ${BUILD_DIR} + check_build_ninja_python_api ${ver} + rm -rf staging + mkdir -p staging + cp -a imperative/python/{megengine,setup.py,requires.txt,requires-style.txt,requires-test.txt} staging/ + cp -a ${SRC_DIR}/src/custom/include staging/megengine/core/include/ + + patch_elf_depend_lib_mgb_mge + + # handle megenginelite + cd ${BUILD_DIR} + mkdir -p staging/megenginelite + cp ${SRC_DIR}/lite/pylite/megenginelite/* staging/megenginelite/ + patch_elf_depend_lib_megenginelite + + cd ${BUILD_DIR}/staging + python3 setup.py bdist_wheel + cd ${BUILD_DIR}/staging/dist/ + cp ${BUILD_DIR}/staging/dist/Meg*.whl ${ANDROID_WHL_HOME} + + cd ${SRC_DIR} + echo "" + echo "##############################################################################################" + echo "android whl package location: ${ANDROID_WHL_HOME}" + ls ${ANDROID_WHL_HOME} + echo "##############################################################################################" + IS_IN_FIRST_LOOP=FALSE + done +} + +function third_party_prepare() { + echo "init third_party..." + bash ${SRC_DIR}/third_party/prepare.sh + # fix flatbuffers build at pure LLVM env(not cross link gcc) + # TODO: pr to flatbuffers to fix this issue + sed -i 's/lc++abi/lc/g' ${SRC_DIR}/third_party/flatbuffers/CMakeLists.txt +} + +function remove_requires() { + # do not worry about this, we will provide 'scripts/whl/android/android_opencv_python.sh' + # to build opencv-python from opencv src!! This function may be removed after termux fixes + # this issue + cd ${SRC_DIR} + git checkout imperative/python/requires.txt + sed -i '/opencv-python/d' imperative/python/requires.txt + # FIXME: termux install pyarrow will build error now + # remove this logic after pyarrow fix this issue + # see imperative/python/megengine/data/dataloader.py + # for detail, now will use _SerialStreamDataLoaderIter + sed -i '/pyarrow/d' imperative/python/requires.txt + cd - +} +###################### +check_termux_env +third_party_prepare +remove_requires +do_build diff --git a/scripts/whl/android/android_opencv_python.sh b/scripts/whl/android/android_opencv_python.sh new file mode 100755 index 000000000..6d7e765ac --- /dev/null +++ b/scripts/whl/android/android_opencv_python.sh @@ -0,0 +1,65 @@ +#!/bin/bash -e + +# This script is a workaround of installing opencv-python in termux. + +SRC_DIR=$(readlink -f "`dirname $0`/../../../") +cd ${SRC_DIR} +source scripts/whl/android/utils.sh + +function install_apt_package() { + APT_PACKAGE="build-essential cmake libjpeg-turbo libpng python clang" + echo "try to install: ${APT_PACKAGE}" + apt install ${APT_PACKAGE} +} + +function build_opencv_python() { + python3 -m pip install numpy + mge_python_env_root="${HOME}/mge_python_env" + mkdir -p ${mge_python_env_root} + cd ${mge_python_env_root} + + opencv_repo_dir=${mge_python_env_root}/opencv + if [ -d ${opencv_repo_dir}/.git ];then + echo "already find opencv repo" + cd ${opencv_repo_dir} + git reset --hard + git clean -xdf + git fetch + else + cd ${mge_python_env_root} + rm -rf ${opencv_repo_dir} + git clone https://github.com/opencv/opencv.git + fi + # Build and test latest version by default. You can modify OPENCV_VER to build and test another version!! + python3_site=`python3 -c 'import site; print(site.getsitepackages()[0])'` + OPENCV_VER="3.4.15" + git checkout ${OPENCV_VER} + if [ -e ${python3_site}/cv2/__init__.py ];then + echo "python3 already build cv2, skip build it, if you want to rebuild, you can do: rm -rf ${python3_site}/cv2" + else + cd ${opencv_repo_dir} + git checkout ${OPENCV_VE} + git apply ${SRC_DIR}/scripts/whl/android/cv_patch/*.patch + mkdir -p build + cd build + echo "will install to ${python3_site}" + PYTHON3_EXECUTABLE=`command -v python3` + LDFLAGS=" -llog -lpython3" cmake -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_opencv_python3=on \ + -DBUILD_opencv_python2=off -DWITH_QT=OFF -DWITH_GTK=OFF -DBUILD_ANDROID_PROJECTS=OFF \ + -DBUILD_ANDROID_EXAMPLES=OFF -DBUILD_FAT_JAVA_LIB=OFF -DBUILD_ANDROID_SERVICE=OFF \ + -DHAVE_opencv_python3=ON -D__INSTALL_PATH_PYTHON3=${python3_site} \ + -DPYTHON3_EXECUTABLE=${PYTHON3_EXECUTABLE} \ + -DOPENCV_PYTHON_INSTALL_PATH=${python3_site} -DCMAKE_INSTALL_PREFIX=${python3_site} .. \ + && make -j$(nproc) && make install + + # check if build successfully + cd ~ + python3 -c 'import cv2;print(cv2.__version__)' + fi +} + +############install env now########### +echo "run at root dir: ${SRC_DIR}" +check_termux_env +install_apt_package +build_opencv_python diff --git a/scripts/whl/android/android_whl_env_prepare.sh b/scripts/whl/android/android_whl_env_prepare.sh new file mode 100755 index 000000000..c7b198c79 --- /dev/null +++ b/scripts/whl/android/android_whl_env_prepare.sh @@ -0,0 +1,155 @@ +#!/bin/bash -e + +# Installing package by pkg in termux is unwise because pkg will upgrade +# package to the latest version which is undesired sometimes. We will +# use apt as default package tool. If your env is already broken,e.g. +# clang does not work， you can execute following commands to fix it: +# pkg update +# pkg upgrade + +SRC_DIR=$(readlink -f "`dirname $0`/../../../") +cd ${SRC_DIR} +source scripts/whl/android/utils.sh + +function install_apt_package() { + APT_PACKAGE="proot git wget clang cmake libandroid-spawn binutils build-essential ninja texinfo patchelf python" + echo "try install: ${APT_PACKAGE}" + apt install ${APT_PACKAGE} + echo "check termux status by running: clang --version" + log=`clang --version || true` + if [[ "${log}" =~ "clang version" ]]; then + echo "valid env after installing apt package" + else + echo "Failed to run clang command, please check termux env!!! You can run: pkg update && pkg upgrade to try to solve it" + echo "raw log: ${log}" + exit -1 + fi + +} + +function patch_termux_env() { + # do not try to modify other project build files to adapt to only-llvm environment + echo "many projects can not build without gcc libs, so we create a fake gcc libs linked to librt" + RT_LIB_TARGET="${PREFIX}/lib/librt.so" + if [ -e ${RT_LIB_TARGET} ];then + echo "find librt.so and link it to libgcc.so" + GCC_LIB_TARGET="${PREFIX}/lib/libgcc.so" + if [ -e ${GCC_LIB_TARGET} ];then + echo "already find: ${GCC_LIB_TARGET} skip it" + else + create_libgcc_cmd="ln -s ${RT_LIB_TARGET} ${GCC_LIB_TARGET}" + echo "run cmd: $create_libgcc_cmd" + ${create_libgcc_cmd} + fi + else + echo "broken termux env, can not find librt.so" + exit -1 + fi +} + +function build_python() { + # Up to now many tools (build multi python3) are not supported in termux so that we have to build them from source. + # This function will be changed when some tools are supported in termux, e.g. pyenv. + mge_python_env_root="${HOME}/mge_python_env" + mkdir -p ${mge_python_env_root} + cd ${mge_python_env_root} + + if [ -e ${PREFIX}/local/lib/libffi.a ];then + echo "always find libffi, skip build it" + else + echo "build libffi for python module" + rm -rf libffi + git clone https://github.com/libffi/libffi.git + cd libffi + termux-chroot "./autogen.sh && ./configure && make -j$(nproc) && make install" + # remove dynamic lib to force python to use static lib + rm ${PREFIX}/local/lib/libffi.so* + fi + + if [ -e ${PREFIX}/local/lib/libz.a ];then + echo "always find libzlib, skip build it" + else + echo "build zlib for python module" + rm -rf zlib + git clone https://github.com/madler/zlib.git + cd zlib + termux-chroot "CFLAGS=\"-O3 -fPIC\" ./configure && make -j$(nproc) && make install" + # remove dynamic lib to force python to use static lib + rm ${PREFIX}/local/lib/libz.so* + fi + + cpython_repo_dir=${mge_python_env_root}/cpython + if [ -d ${cpython_repo_dir}/.git ];then + echo "already find cpython repo" + cd ${cpython_repo_dir} + git reset --hard + git clean -xdf + git fetch + else + cd ${mge_python_env_root} + rm -rf ${cpython_repo_dir} + git clone https://github.com/python/cpython.git + fi + for ver in ${ALL_PYTHON} + do + install_dir=${mge_python_env_root}/${ver}/install + if [ -e ${install_dir}/bin/python3 ];then + echo "always find python3, skip build it" + else + mkdir -p ${install_dir} + echo "try build python: ${ver} to ${install_dir}" + cd ${cpython_repo_dir} + git reset --hard + git clean -xdf + git checkout v${ver} + index=`awk -v a="${ver}" -v b="." 'BEGIN{print index(a,b)}'` + sub_str=${ver:${index}} + index_s=`awk -v a="${sub_str}" -v b="." 'BEGIN{print index(a,b)}'` + ((finally_index = ${index} + ${index_s})) + finally_verson=${ver:0:${finally_index}-1} + MINOR_VER=${ver:${index}:${finally_index}-${index}-1} + finally_verson="python${finally_verson}" + echo "finally_verson is: ${finally_verson}" + # apply patchs + git apply ${SRC_DIR}/scripts/whl/android/patchs/*.patch + if [[ ${MINOR_VER} -gt 8 ]] + then + echo "apply more patchs" + git apply ${SRC_DIR}/scripts/whl/android/up_3_9_patch/*.patch + fi + + termux-chroot "FLAGS=\"-D__ANDROID_API__=24 -Wno-unused-value -Wno-empty-body -Qunused-arguments -Wno-error\" \ + ./configure CFLAGS=\"${FLAGS}\" CPPFLAGS=\"${FLAGS}\" CC=clang CXX=clang++ --enable-shared --prefix=${install_dir} \ + && sed -i 's/-Werror=implicit-function-declaration//g' Makefile && \ + sed -i 's/\$(LN) -f \$(INSTSONAME)/cp \$(INSTSONAME)/g' Makefile && make -j$(nproc) && make install" + + # after building successfully, patchelf to make python work out of termux-chroot env + cd ${install_dir}/bin + # Some python versions won't link to python3 automatically, so we create link manually here' + rm -rf python3 + if [[ ${MINOR_VER} -gt 7 ]] + then + echo "python3 remove suffix m after 3.8" + patchelf --add-rpath ${install_dir}/lib ${finally_verson} + cp ${finally_verson} python3 + else + echo "python3 with suffix m before 3.8, add it" + patchelf --add-rpath ${install_dir}/lib "${finally_verson}m" + cp "${finally_verson}m" ${finally_verson} + cp ${finally_verson} python3 + fi + echo "finally try run python3" + ./python3 --version + ./python3 -m pip install --upgrade pip + ./python3 -m pip install numpy wheel + + fi + done +} + +############install env now########### +echo "run at root dir: ${SRC_DIR}" +check_termux_env +install_apt_package +patch_termux_env +build_python diff --git a/scripts/whl/android/cv_patch/host_android.patch b/scripts/whl/android/cv_patch/host_android.patch new file mode 100644 index 000000000..0d89b384c --- /dev/null +++ b/scripts/whl/android/cv_patch/host_android.patch @@ -0,0 +1,75 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f6a2da5310..10354312c9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -643,7 +643,7 @@ if(UNIX) + if(NOT APPLE) + CHECK_INCLUDE_FILE(pthread.h HAVE_PTHREAD) + if(ANDROID) +- set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m log) ++ set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m log z) + elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD|NetBSD|DragonFly|OpenBSD|Haiku") + set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m pthread) + elseif(EMSCRIPTEN) +diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake +index 4ff02a77d3..db1305448f 100644 +--- a/cmake/OpenCVDetectPython.cmake ++++ b/cmake/OpenCVDetectPython.cmake +@@ -123,7 +123,7 @@ if(NOT ${found}) + if(_found) + set(_version_major_minor "${_version_major}.${_version_minor}") + +- if(NOT ANDROID AND NOT APPLE_FRAMEWORK) ++ if(TRUE) + ocv_check_environment_variables(${library_env} ${include_dir_env}) + if(NOT ${${library_env}} STREQUAL "") + set(PYTHON_LIBRARY "${${library_env}}") +@@ -175,7 +175,7 @@ if(NOT ${found}) + endif() + endif() + +- if(NOT ANDROID AND NOT IOS) ++ if(TRUE) + if(CMAKE_HOST_UNIX) + execute_process(COMMAND ${_executable} -c "from distutils.sysconfig import *; print(get_python_lib())" + RESULT_VARIABLE _cvpy_process +@@ -240,7 +240,7 @@ if(NOT ${found}) + OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() + endif() +- endif(NOT ANDROID AND NOT IOS) ++ endif() + endif() + + # Export return values +@@ -285,6 +285,17 @@ find_python("${OPENCV_PYTHON3_VERSION}" "${MIN_VER_PYTHON3}" PYTHON3_LIBRARY PYT + PYTHON3_INCLUDE_DIR PYTHON3_INCLUDE_DIR2 PYTHON3_PACKAGES_PATH + PYTHON3_NUMPY_INCLUDE_DIRS PYTHON3_NUMPY_VERSION) + ++message("DEBUG PYTHON3_LIBRARIES: ${PYTHON3_LIBRARIES}") ++message("DEBUG PYTHON3_INCLUDE_DIR: ${PYTHON3_INCLUDE_DIR}") ++string(COMPARE EQUAL "${PYTHON3_LIBRARIES}" "" result) ++if(result) ++ message(FATAL_ERROR "can not find PYTHON3_LIBRARIES") ++endif() ++ ++string(COMPARE EQUAL "${PYTHON3_INCLUDE_DIR}" "" result) ++if(result) ++ message(FATAL_ERROR "can not find PYTHON3_INCLUDE_DIR") ++endif() + + if(PYTHON_DEFAULT_EXECUTABLE) + set(PYTHON_DEFAULT_AVAILABLE "TRUE") +diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt +index a51acf386e..5605a54a32 100644 +--- a/modules/python/CMakeLists.txt ++++ b/modules/python/CMakeLists.txt +@@ -3,7 +3,7 @@ + # ---------------------------------------------------------------------------- + if(DEFINED OPENCV_INITIAL_PASS) # OpenCV build + +-if(ANDROID OR APPLE_FRAMEWORK OR WINRT) ++ if(False) + ocv_module_disable_(python2) + ocv_module_disable_(python3) + return() diff --git a/scripts/whl/android/patchs/Lib-subprocess.py.patch b/scripts/whl/android/patchs/Lib-subprocess.py.patch new file mode 100644 index 000000000..bea4bbf23 --- /dev/null +++ b/scripts/whl/android/patchs/Lib-subprocess.py.patch @@ -0,0 +1,14 @@ +diff -u -r ../Python-3.7.1/Lib/subprocess.py ./Lib/subprocess.py +--- ../Python-3.7.1/Lib/subprocess.py 2018-10-20 06:04:19.000000000 +0000 ++++ ./Lib/subprocess.py 2018-10-20 20:17:50.157206343 +0000 +@@ -1389,9 +1389,7 @@ + args = list(args) + + if shell: +- # On Android the default shell is at '/system/bin/sh'. +- unix_shell = ('/system/bin/sh' if +- hasattr(sys, 'getandroidapilevel') else '/bin/sh') ++ unix_shell = ('@TERMUX_PREFIX@/bin/sh') + args = [unix_shell, "-c"] + args + if executable: + args[0] = executable diff --git a/scripts/whl/android/patchs/Lib-tmpfile.py.patch b/scripts/whl/android/patchs/Lib-tmpfile.py.patch new file mode 100644 index 000000000..3705c569d --- /dev/null +++ b/scripts/whl/android/patchs/Lib-tmpfile.py.patch @@ -0,0 +1,13 @@ +diff --git a/Lib/tempfile.py b/Lib/tempfile.py +index 531cbf32f1..dae57e22bd 100644 +--- a/Lib/tempfile.py ++++ b/Lib/tempfile.py +@@ -170,7 +170,7 @@ def _candidate_tempdir_list(): + _os.path.expandvars(r'%SYSTEMROOT%\Temp'), + r'c:\temp', r'c:\tmp', r'\temp', r'\tmp' ]) + else: +- dirlist.extend([ '/tmp', '/var/tmp', '/usr/tmp' ]) ++ dirlist.extend([ '@TERMUX_PREFIX@/tmp' ]) + + # As a last resort, the current directory. + try: diff --git a/scripts/whl/android/patchs/distutils-command-build.py.patch b/scripts/whl/android/patchs/distutils-command-build.py.patch new file mode 100644 index 000000000..c9cd42dc1 --- /dev/null +++ b/scripts/whl/android/patchs/distutils-command-build.py.patch @@ -0,0 +1,11 @@ +--- ./Lib/distutils/command/build.py.orig 2019-11-19 23:19:08.878782120 +0000 ++++ ./Lib/distutils/command/build.py 2019-11-19 23:19:18.410915201 +0000 +@@ -118,7 +118,7 @@ + + if self.executable is None and sys.executable: + self.executable = os.path.normpath(sys.executable) +- ++ self.parallel = 1 + if isinstance(self.parallel, str): + try: + self.parallel = int(self.parallel) diff --git a/scripts/whl/android/patchs/no-setuid-servers.patch b/scripts/whl/android/patchs/no-setuid-servers.patch new file mode 100644 index 000000000..e7cc8ce97 --- /dev/null +++ b/scripts/whl/android/patchs/no-setuid-servers.patch @@ -0,0 +1,36 @@ +diff -uNr Python-3.9.2/Lib/http/server.py Python-3.9.2.mod/Lib/http/server.py +--- Python-3.9.2/Lib/http/server.py 2021-02-19 14:31:44.000000000 +0200 ++++ Python-3.9.2.mod/Lib/http/server.py 2021-03-20 16:08:34.173543081 +0200 +@@ -1165,10 +1165,6 @@ + return + # Child + try: +- try: +- os.setuid(nobody) +- except OSError: +- pass + os.dup2(self.rfile.fileno(), 0) + os.dup2(self.wfile.fileno(), 1) + os.execve(scriptfile, args, env) +diff -uNr Python-3.9.2/Lib/smtpd.py Python-3.9.2.mod/Lib/smtpd.py +--- Python-3.9.2/Lib/smtpd.py 2021-02-19 14:31:44.000000000 +0200 ++++ Python-3.9.2.mod/Lib/smtpd.py 2021-03-20 16:11:48.785629393 +0200 +@@ -9,7 +9,8 @@ + -n + This program generally tries to setuid `nobody', unless this flag is + set. The setuid call will fail if this program is not run as root (in +- which case, use this flag). ++ which case, use this flag). Ignored in Termux as no setuid done on this ++ platform. + + --version + -V +@@ -863,7 +864,7 @@ + + + class Options: +- setuid = True ++ setuid = False + classname = 'PureProxy' + size_limit = None + enable_SMTPUTF8 = False diff --git a/scripts/whl/android/up_3_9_patch/fix-paths.patch b/scripts/whl/android/up_3_9_patch/fix-paths.patch new file mode 100644 index 000000000..1135b1b86 --- /dev/null +++ b/scripts/whl/android/up_3_9_patch/fix-paths.patch @@ -0,0 +1,59 @@ +diff -uNr Python-3.6.2/Lib/aifc.py Python-3.6.2.mod/Lib/aifc.py +--- Python-3.6.2/Lib/aifc.py 2017-07-08 06:33:27.000000000 +0300 ++++ Python-3.6.2.mod/Lib/aifc.py 2017-09-15 15:09:08.092797061 +0300 +@@ -920,7 +920,7 @@ + if __name__ == '__main__': + import sys + if not sys.argv[1:]: +- sys.argv.append('/usr/demos/data/audio/bach.aiff') ++ sys.argv.append('@TERMUX_PREFIX@/demos/data/audio/bach.aiff') + fn = sys.argv[1] + with open(fn, 'r') as f: + print("Reading", fn) +diff -uNr Python-3.6.2/Lib/mailcap.py Python-3.6.2.mod/Lib/mailcap.py +--- Python-3.6.2/Lib/mailcap.py 2017-07-08 06:33:27.000000000 +0300 ++++ Python-3.6.2.mod/Lib/mailcap.py 2017-09-15 15:08:41.312797081 +0300 +@@ -55,7 +55,8 @@ + # Don't bother with getpwuid() + home = '.' # Last resort + mailcaps = [home + '/.mailcap', '/etc/mailcap', +- '/usr/etc/mailcap', '/usr/local/etc/mailcap'] ++ '/usr/etc/mailcap', '/usr/local/etc/mailcap', ++ '@TERMUX_PREFIX@/etc/mailcap'] + return mailcaps + + +diff -uNr Python-3.6.2/Lib/mimetypes.py Python-3.6.2.mod/Lib/mimetypes.py +--- Python-3.6.2/Lib/mimetypes.py 2017-07-08 06:33:27.000000000 +0300 ++++ Python-3.6.2.mod/Lib/mimetypes.py 2017-09-15 15:08:05.522797106 +0300 +@@ -49,6 +49,7 @@ + "/usr/local/lib/netscape/mime.types", + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + "/usr/local/etc/mime.types", # Apache 1.3 ++ "@TERMUX_PREFIX@/etc/mime.types", # Termux + ] + + inited = False +diff -uNr Python-3.6.2/Lib/posixpath.py Python-3.6.2.mod/Lib/posixpath.py +--- Python-3.6.2/Lib/posixpath.py 2017-07-08 06:33:27.000000000 +0300 ++++ Python-3.6.2.mod/Lib/posixpath.py 2017-09-15 15:07:20.872797138 +0300 +@@ -32,7 +32,7 @@ + extsep = '.' + sep = '/' + pathsep = ':' +-defpath = '/bin:/usr/bin' ++defpath = '@TERMUX_PREFIX@/bin' + altsep = None + devnull = '/dev/null' + +diff -uNr Python-3.9.0/Lib/uuid.py Python-3.9.0.mod/Lib/uuid.py +--- Python-3.9.0/Lib/uuid.py 2020-10-05 20:37:58.000000000 +0530 ++++ Python-3.9.0.mod/Lib/uuid.py 2020-10-08 18:25:45.565373486 +0530 +@@ -361,7 +361,6 @@ + + try: + path_dirs = os.environ.get('PATH', os.defpath).split(os.pathsep) +- path_dirs.extend(['/sbin', '/usr/sbin']) + executable = shutil.which(command, path=os.pathsep.join(path_dirs)) + if executable is None: + return None diff --git a/scripts/whl/android/utils.sh b/scripts/whl/android/utils.sh new file mode 100755 index 000000000..88b91668f --- /dev/null +++ b/scripts/whl/android/utils.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e + +source ${SRC_DIR}/scripts/whl/utils/utils.sh + +ALL_PYTHON=${ALL_PYTHON} +# FIXME: now imperative py code do not support 3.10 +# but megenginelite and megbrain support it, so we +# config with 3.10.1 now +FULL_PYTHON_VER="3.8.3 3.9.9 3.10.1" +if [[ -z ${ALL_PYTHON} ]] +then + ALL_PYTHON=${FULL_PYTHON_VER} +else + check_python_version_is_valid "${ALL_PYTHON}" "${FULL_PYTHON_VER}" +fi + +# FIXME python3.10+ self build have some issue, need config env +# _PYTHON_SYSCONFIGDATA_NAME remove this env after find the build issue +# do not care about this, apt install python3.10 do not have this issue +export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__linux_aarch64-linux-android" + +function check_termux_env() { + echo "check is in termux env or not" + info=`command -v termux-info || true` + if [[ "${info}" =~ "com.termux" ]]; then + echo "find termux-info at: ${info}" + echo "check env now" + ENVS="PREFIX HOME" + for check_env in ${ENVS} + do + echo "try check env: ${check_env}" + if [[ "${!check_env}" =~ "termux" ]]; then + echo "env ${check_env} is: ${!check_env}" + else + echo "invalid ${check_env} env, may broken termux env" + exit -1 + fi + done + else + echo "invalid env, only support build android whl at termux env, please refs to: scripts/whl/BUILD_PYTHON_WHL_README.md to init env" + exit -1 + fi +} diff --git a/scripts/whl/utils/utils.sh b/scripts/whl/utils/utils.sh index 0eee9a927..81e4ae4ea 100755 --- a/scripts/whl/utils/utils.sh +++ b/scripts/whl/utils/utils.sh @@ -23,7 +23,7 @@ function ninja_dry_run_and_check_increment() { exit -1 fi - ${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log + bash ${_BUILD_SHELL} ${_BUILD_FLAGS} 2>&1 | tee dry_run.log DIRTY_LOG=`cat dry_run.log` if [[ "${DIRTY_LOG}" =~ ${_INCREMENT_KEY_WORDS} ]]; then @@ -79,10 +79,23 @@ function check_build_ninja_python_api() { PYTHON_API_INCLUDES="3.5.4\\\\include 3.6.8\\\\include 3.7.7\\\\include 3.8.3\\\\include" elif [[ $OS =~ "Linux" ]]; then INCLUDE_KEYWORD="include/python3.${ver:1:1}" - PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8" + info=`command -v termux-info || true` + if [[ "${info}" =~ "com.termux" ]]; then + echo "find termux-info at: ${info}" + is_punctuation=${ver:4:1} + INCLUDE_KEYWORD="include/python3.${ver:2:1}" + if [ ${is_punctuation} = "." ]; then + INCLUDE_KEYWORD="include/python3.${ver:2:2}" + fi + fi + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10" elif [[ $OS =~ "Darwin" ]]; then + is_punctuation=${ver:4:1} INCLUDE_KEYWORD="include/python3.${ver:2:1}" - PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8" + if [ ${is_punctuation} = "." ]; then + INCLUDE_KEYWORD="include/python3.${ver:2:2}" + fi + PYTHON_API_INCLUDES="include/python3.5 include/python3.6 include/python3.7 include/python3.8 include/python3.9 include/python3.10" else echo "unknown OS: ${OS}" exit -1 -- GitLab