From 05a9709c1953e9a5e35dcfae5d116bb93e24f66e Mon Sep 17 00:00:00 2001 From: Olli-Pekka Heinisuo Date: Thu, 3 May 2018 19:35:40 +0300 Subject: [PATCH] test headless builds --- .travis.yml | 223 ++++++++++++++++++++++++++++++++++++++++++++++- appveyor.yml | 114 +++++++++++++++++++++--- docker/README.md | 4 +- setup.py | 39 +++++++-- 4 files changed, 357 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e3eb37..dff6602 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,24 +34,58 @@ matrix: env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + + # headless builds for MacOS + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=2.7 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.4 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.5 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.6 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 # Contrib builds for MacOS - os: osx @@ -60,74 +94,179 @@ matrix: env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 - os: osx language: generic osx_image: xcode7.3 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + + # headless contrib builds for MacOS + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=2.7 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.4 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.5 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 + - os: osx + language: generic + osx_image: xcode7.3 + env: + - MB_PYTHON_VERSION=3.6 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 # default builds for Linux - os: linux env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - UNICODE_WIDTH=16 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - PLAT=i686 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - PLAT=i686 + - UNICODE_WIDTH=16 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.4 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.4 + - PLAT=i686 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.5 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.5 + - PLAT=i686 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.6 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - TEST_DEPENDS=numpy==1.11.3 + - os: linux + env: + - MB_PYTHON_VERSION=3.6 + - PLAT=i686 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=0 + - TEST_DEPENDS=numpy==1.11.3 + + # headless builds for Linux + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - UNICODE_WIDTH=16 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - PLAT=i686 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - PLAT=i686 - UNICODE_WIDTH=16 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.4 - PLAT=i686 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.5 - PLAT=i686 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.11.3 - os: linux env: - MB_PYTHON_VERSION=3.6 - PLAT=i686 - ENABLE_CONTRIB=0 + - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.11.3 # contrib builds for Linux @@ -135,51 +274,122 @@ matrix: env: - MB_PYTHON_VERSION=2.7 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - UNICODE_WIDTH=16 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - PLAT=i686 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - PLAT=i686 + - UNICODE_WIDTH=16 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.4 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.4 + - PLAT=i686 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.5 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.5 + - PLAT=i686 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.6 + - TEST_DEPENDS=numpy==1.11.3 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + - os: linux + env: + - MB_PYTHON_VERSION=3.6 + - PLAT=i686 + - TEST_DEPENDS=numpy==1.11.3 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=0 + + # headless contrib builds for Linux + - os: linux + env: + - MB_PYTHON_VERSION=2.7 + - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - UNICODE_WIDTH=16 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - PLAT=i686 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=2.7 - PLAT=i686 - UNICODE_WIDTH=16 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.4 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.4 - PLAT=i686 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.5 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.5 - PLAT=i686 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.6 - TEST_DEPENDS=numpy==1.11.3 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 - os: linux env: - MB_PYTHON_VERSION=3.6 - PLAT=i686 - TEST_DEPENDS=numpy==1.11.3 - ENABLE_CONTRIB=1 + - ENABLE_HEADLESS=1 # The first line is printed in the folding header in Travis output before_install: | @@ -195,6 +405,7 @@ before_install: | # This sets -x source multibuild_customize.sh echo $ENABLE_CONTRIB > contrib.enabled + echo $ENABLE_HEADLESS > headless.enabled before_install # Not interested in travis internal scripts' output set +x @@ -218,9 +429,17 @@ after_success: | pip install twine if [[ $ENABLE_CONTRIB == 0 ]]; then - echo "This is default build. Deployment will be done to to PyPI entry opencv-python." + if [[ $ENABLE_HEADLESS == 0 ]]; then + echo "This is default build. Deployment will be done to to PyPI entry opencv-python." + else + echo "This is headless contrib build. Deployment will be done to to PyPI entry opencv-python-headless." + fi else - echo "This is contrib build. Deployment will be done to to PyPI entry opencv-contrib-python." + if [[ $ENABLE_HEADLESS == 0 ]]; then + echo "This is contrib build. Deployment will be done to to PyPI entry opencv-contrib-python." + else + echo "This is headless contrib build. Deployment will be done to to PyPI entry opencv-contrib-python-headless." + fi fi twine upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv* diff --git a/appveyor.yml b/appveyor.yml index e777c48..c7a8b31 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,51 +8,131 @@ environment: matrix: - PYTHON: "C:/Python27" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python27-x64" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python34" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python34-x64" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python35" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python35-x64" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python36" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python36-x64" ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python27" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python27-x64" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python34" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python34-x64" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python35" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python35-x64" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python36" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 - PYTHON: "C:/Python36-x64" ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 0 + + - PYTHON: "C:/Python27" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python27-x64" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python34" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python34-x64" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python35" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python35-x64" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python36" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python36-x64" + ENABLE_CONTRIB: 0 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python27" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python27-x64" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python34" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python34-x64" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python35" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python35-x64" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python36" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 + + - PYTHON: "C:/Python36-x64" + ENABLE_CONTRIB: 1 + ENABLE_HEADLESS: 1 matrix: fast_finish: true @@ -83,13 +163,27 @@ deploy_script: - ps: | if (${Env:APPVEYOR_REPO_TAG} -eq "true") { - cd ${Env:APPVEYOR_BUILD_FOLDER} - if (${Env:ENABLE_CONTRIB} -eq 0) ` - {echo "This is a default build. Deployment will be done to PyPI entry opencv-python."} ` - else ` - {echo "This is a contrib build. Deployment will be done to PyPI entry opencv-contrib-python."} - - &"${Env:PYTHON}/python.exe" -m pip install twine - &"${Env:PYTHON}/python.exe" -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencv* } ` - else - {echo "Tag not set, deployment skipped."} + cd ${Env:APPVEYOR_BUILD_FOLDER} + if (${Env:ENABLE_CONTRIB} -eq 0) { + if (${Env:ENABLE_HEADLESS} -eq 0) { + echo "This is a default build. Deployment will be done to PyPI entry opencv-python." + } + else { + echo "This is a headless build. Deployment will be done to PyPI entry opencv-python-headless." + } + } + else { + if (${Env:ENABLE_HEADLESS} -eq 0) { + echo "This is a contrib build. Deployment will be done to PyPI entry opencv-contrib-python." + } + else { + echo "This is a headless contrib build. Deployment will be done to PyPI entry opencv-contrib-python-headless." + } + } + + &"${Env:PYTHON}/python.exe" -m pip install twine + &"${Env:PYTHON}/python.exe" -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencv* + } + else { + echo "Tag not set, deployment skipped." + } diff --git a/docker/README.md b/docker/README.md index 9e932c4..2e8602b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,12 +4,12 @@ This folder includes Dockerfiles for extending both ``i686`` and ``x86_64`` many The extended images were created to be able to build OpenCV in reasonable time with Travis. -These images have been built locally because the process takes over an hour with modern i7 processor. The images are hosted at https://quay.io/user/skvark. +These images have been built locally because the process takes over 1,5 hours with modern i7 processor. The images are hosted at https://quay.io/user/skvark. The images have following extra software installed: - Qt 4.8.7 - Cmake 3.9.0 -- FFmpeg with libvpx +- FFmpeg with libvpx (latest snapshots at the build time) - libjpeg-turbo 1.5.3 - Some missing headers included from more recent Linux to be able to enable V4L / V4L2 support in OpenCV \ No newline at end of file diff --git a/setup.py b/setup.py index d0b073f..986f60b 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,14 @@ import sysconfig def main(): + os.chdir(os.path.dirname(os.path.abspath(__file__))) # These are neede for source fetching cmake_source_dir = "opencv" - build_contrib = get_build_contrib() + build_contrib = get_build_env_var_by_name("contrib") + # headless flag to skip GUI deps if needed + build_headless = get_build_env_var_by_name("headless") # Only import 3rd-party modules after having installed all the build dependencies: # any of them, or their dependencies, can be updated during that process, @@ -37,7 +40,17 @@ def main(): # https://stackoverflow.com/questions/1405913/python-32bit-or-64bit-mode x64 = sys.maxsize > 2**32 - package_name = "opencv-contrib-python" if build_contrib else "opencv-python" + package_name = "opencv-python" + + if build_contrib and !build_headless: + package_name = "opencv-contrib-python" + + if build_contrib and build_headless + package_name = "opencv-contrib-python-headless" + + if build_headless and !build_contrib: + package_name = "opencv-python-headless" + long_description = io.open('README_CONTRIB.rst' if build_contrib else 'README.rst', encoding="utf-8").read() package_version = get_opencv_version() @@ -91,9 +104,14 @@ def main(): ] + (["-DOPENCV_EXTRA_MODULES_PATH=" + os.path.abspath("opencv_contrib/modules")] if build_contrib else []) # OS-specific components - if sys.platform == 'darwin' or sys.platform.startswith('linux'): + if (sys.platform == 'darwin' or sys.platform.startswith('linux')) and !build_headless: cmake_args.append("-DWITH_QT=4") + if build_headless: + # it seems that cocoa cannot be disabled so on macOS the package is not truly headless + cmake_args.append("-DWITH_WIN32UI=OFF") + cmake_args.append("-DWITH_QT=OFF") + if sys.platform.startswith('linux'): cmake_args.append("-DWITH_V4L=ON") cmake_args.append("-DENABLE_PRECOMPILED_HEADERS=OFF") @@ -300,19 +318,22 @@ def get_opencv_version(): return opencv_version -def get_build_contrib(): - build_contrib = False +def get_build_env_var_by_name(flag_name): + + flag_set = False + try: - build_contrib = bool(int(os.getenv('ENABLE_CONTRIB', None))) + flag_set = bool(int(os.getenv('ENABLE_' + flag_name.upper() , None))) except Exception: pass - if not build_contrib: + if not flag_set: try: - build_contrib = bool(int(open("contrib.enabled").read(1))) + flag_set = bool(int(open(flag_name + ".enabled").read(1))) except Exception: pass - return build_contrib + + return flag_set def get_or_install(name, version=None): -- GitLab