initial sdist support

.travis.yml

@@ -4,10 +4,10 @@ env:
         # pip dependencies to _test_ your project
         - TEST_DEPENDS="numpy==1.11.1"
         # params to bdist_wheel. used to set osx build target.
-        - BDIST_PARAMS=""
         - CONFIG_PATH="travis_config.sh"
         - USE_CCACHE=1
         - UNICODE_WIDTH=32
+        - SDIST=0
 
 # Save some time, we and setup check them out on demand instead
 # https://docs.travis-ci.com/user/customizing-the-build/#Git-Clone-Depth
@@ -28,6 +28,7 @@ stages:
 
 jobs:
   fast_finish: true
+  # Travis exclude is buggy, this seems to be the only way to disable default build
   exclude:
     - language: ruby
   include:
@@ -57,6 +58,40 @@ jobs:
           # used in OSX custom build script dealing with local bottle caching
           - $HOME/local_bottle_metadata
 
+    # source distributions
+    - os: linux
+      env:
+        - SDIST=1
+        - ENABLE_CONTRIB=0
+        - ENABLE_HEADLESS=0
+      python: "3.8"
+      language: python
+      dist: xenial
+    - os: linux
+      env:
+        - SDIST=1
+        - ENABLE_CONTRIB=0
+        - ENABLE_HEADLESS=1
+      python: "3.8"
+      language: python
+      dist: xenial
+    - os: linux
+      env:
+        - SDIST=1
+        - ENABLE_CONTRIB=1
+        - ENABLE_HEADLESS=0
+      python: "3.8"
+      language: python
+      dist: xenial
+    - os: linux
+      env:
+        - SDIST=1
+        - ENABLE_CONTRIB=1
+        - ENABLE_HEADLESS=1
+      python: "3.8"
+      language: python
+      dist: xenial
+
     # default builds for MacOS
       #further jobs in the list will use the same stage until the next assignment
     - stage: final
@@ -606,56 +641,74 @@ jobs:
 
 # The first line is printed in the folding header in Travis output
 before_install: |
-    # Check out and prepare the source
     set -e
-    # Multibuild doesn't have releases, so --depth would break eventually (see
-    # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised)
-    git submodule update --init multibuild
-    echo "Source multibuild/common_utils.sh"
-    source multibuild/common_utils.sh
-    # https://github.com/matthew-brett/multibuild/issues/116
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi
-    echo "Source multibuild/travis_steps.sh"
-    source multibuild/travis_steps.sh
-    # This sets -x
-    echo "Source travis_multibuild_customize.sh"
-    source travis_multibuild_customize.sh
-    echo $ENABLE_CONTRIB > contrib.enabled
-    echo $ENABLE_HEADLESS > headless.enabled
 
-    if [ -n "$IS_OSX" ]; then
-        echo "homebrew"
-        TAPS="$(brew --repository)/Library/Taps"
-        if [ -e "$TAPS/caskroom/homebrew-cask" -a -e "$TAPS/homebrew/homebrew-cask" ]; then
-            rm -rf "$TAPS/caskroom/homebrew-cask"
-        fi
-        find "$TAPS" -type d -name .git -exec \
-                bash -xec '
-                    cd $(dirname '\''{}'\'') || echo "status: $?"
-                    git clean -fxd || echo "status: $?"
-                    sleep 1 || echo "status: $?"
-                    git status || echo "status: $?"' \; || echo "status: $?"
+    if [ -z "$SDIST" ]; then
+        echo "sdist build"
+    else
+        # Check out and prepare the source
+        # Multibuild doesn't have releases, so --depth would break eventually (see
+        # https://superuser.com/questions/1240216/server-does-not-allow-request-for-unadvertised)
+        git submodule update --init multibuild
 
-        brew_cache_cleanup
+        source multibuild/common_utils.sh
+
+        # https://github.com/matthew-brett/multibuild/issues/116
+        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export ARCH_FLAGS=" "; fi
+
+        source multibuild/travis_steps.sh
+        # This sets -x
+
+        source travis_multibuild_customize.sh
+        echo $ENABLE_CONTRIB > contrib.enabled
+        echo $ENABLE_HEADLESS > headless.enabled
+
+        if [ -n "$IS_OSX" ]; then
+            echo "homebrew"
+            TAPS="$(brew --repository)/Library/Taps"
+            if [ -e "$TAPS/caskroom/homebrew-cask" -a -e "$TAPS/homebrew/homebrew-cask" ]; then
+                rm -rf "$TAPS/caskroom/homebrew-cask"
+            fi
+            find "$TAPS" -type d -name .git -exec \
+                    bash -xec '
+                        cd $(dirname '\''{}'\'') || echo "status: $?"
+                        git clean -fxd || echo "status: $?"
+                        sleep 1 || echo "status: $?"
+                        git status || echo "status: $?"' \; || echo "status: $?"
+            brew_cache_cleanup
+        fi
+        echo "end"
+        # Not interested in travis internal scripts' output
     fi
-    echo "end"
-    # Not interested in travis internal scripts' output
+
     set +x
 
 install: |
     # Build and package
     set -x
-    build_wheel $REPO_DIR $PLAT
+
+    if [ -z "$SDIST" ]; then
+        python setup.py sdist
+    else
+        build_wheel $REPO_DIR $PLAT
+    fi
+
     set +x
 
 script: |
     # Install and run tests
     set -x
-    install_run $PLAT && rc=$? || rc=$?
+
+    if [ -z "$SDIST" ]; then
+      echo "sdist"
+    else
+        install_run $PLAT && rc=$? || rc=$?
+    fi
+
     set +x
 
-    #otherwise, Travis logic terminates prematurely
-    #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817
+    # otherwise, Travis logic terminates prematurely
+    # https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817
     trap ERR
 
     test "$rc" -eq 0
@@ -676,19 +729,11 @@ before_cache: |
     set +x; set +e
 
 after_success: |
-    # Upload wheels to pypi if tag is set, otherwise save to Azure Storage
-    if [ -n "$TRAVIS_TAG" ]; then
-        set -x
+    # Upload wheels to pypi if tag is set
 
-        if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-          pip install --user twine
-          pip install --user --upgrade six
-        fi
+    set -x
 
-        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-          pip install twine
-          pip install --upgrade pyOpenSSL
-        fi
+    if [ -n "$TRAVIS_TAG" ]; then
 
         if [[ $ENABLE_CONTRIB == 0 ]]; then
             if [[ $ENABLE_HEADLESS == 0 ]]; then
@@ -704,19 +749,50 @@ after_success: |
             fi
         fi
 
-        twine upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv*
-        set +x
-    else
-        set -x
-
         if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-        else
-          brew install azure-cli
+          pip install --user twine
+          pip install --user --upgrade six
+
+          if [ -z "$SDIST" ]; then
+            twine upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist
+          else
+            twine upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv*
+          fi
+
         fi
 
-        az storage container create -n ${TRAVIS_COMMIT} --public-access blob
-        az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern *.whl
+        if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+          # macpython 3.5 doesn't support recent TLS protocols which causes twine
+          # upload to fail, so we use the system Python to run twine
+          /usr/bin/python -m ensurepip --user
+          /usr/bin/python -m pip install --user -U pip
+          /usr/bin/python -m pip install --user -U -I twine
+
+          if [ -z "$SDIST" ]; then
+            /usr/bin/python -m upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist
+          else
+            /usr/bin/python -m upload -u ${USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv*
+          fi
+
+        fi
 
-        set -x
     fi
+
+    # Save to Azure storage always
+
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+    else
+      brew install azure-cli
+    fi
+
+    az storage container create -n ${TRAVIS_COMMIT} --public-access blob
+
+    if [ -z "$SDIST" ]; then
+      az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/dist --pattern *.gz
+    else
+      az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern *.whl
+    fi
+
+    set +x
+

MANIFEST.in

+include LICENSE-3RD-PARTY.txt
+include LICENSE.txt
+include README.md
+include find_version.py
+include setup.py
+recursive-include cv2 *
+recursive-include docker *
+recursive-include opencv *
+recursive-include opencv_contrib *
+recursive-include patches *

find_version.py

@@ -2,49 +2,53 @@ import sys
 import os
 import subprocess
 
-opencv_version = ""
-# dig out the version from OpenCV sources
-version_file_path = "opencv/modules/core/include/opencv2/core/version.hpp"
-
-with open(version_file_path, 'r') as f:
-    for line in f:
-        words = line.split()
-
-        if "CV_VERSION_MAJOR" in words:
-            opencv_version += words[2]
-            opencv_version += "."
-
-        if "CV_VERSION_MINOR" in words:
-            opencv_version += words[2]
-            opencv_version += "."
-
-        if "CV_VERSION_REVISION" in words:
-            opencv_version += words[2]
-            break
-
-# used in local dev releases
-git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).splitlines()[0].decode()
-# this outputs the annotated tag if we are exactly on a tag, otherwise <tag>-<n>-g<shortened sha-1>
-try:
-    tag = subprocess.check_output(['git', 'describe', '--tags'], stderr = subprocess.STDOUT).splitlines()[0].decode().split('-')
-except subprocess.CalledProcessError as e:
-    # no tags reachable (e.g. on a topic branch in a fork), see
-    # https://stackoverflow.com/questions/4916492/git-describe-fails-with-fatal-no-names-found-cannot-describe-anything
-    if e.output.rstrip() == b"fatal: No names found, cannot describe anything.":
-        tag=[]
-    else:
-        print(e.output); raise
-
-if len(tag) == 1:
-    # tag identifies the build and should be a sequential revision number
-    version = tag[0]
-    opencv_version += ".{}".format(version)
-else:
-    # local version identifier, not to be published on PyPI
-    version = git_hash
-    opencv_version += "+{}".format(version)
-
-print("Version: ", opencv_version)
-
-with open('cv_version.py', 'w') as f:
-    f.write('opencv_version = "{}"'.format(opencv_version))
+if __name__ == "__main__":
+  contrib = sys.argv[1]
+  headless = sys.argv[2]
+
+  opencv_version = ""
+  # dig out the version from OpenCV sources
+  version_file_path = "opencv/modules/core/include/opencv2/core/version.hpp"
+
+  with open(version_file_path, 'r') as f:
+      for line in f:
+          words = line.split()
+
+          if "CV_VERSION_MAJOR" in words:
+              opencv_version += words[2]
+              opencv_version += "."
+
+          if "CV_VERSION_MINOR" in words:
+              opencv_version += words[2]
+              opencv_version += "."
+
+          if "CV_VERSION_REVISION" in words:
+              opencv_version += words[2]
+              break
+
+  # used in local dev releases
+  git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).splitlines()[0].decode()
+  # this outputs the annotated tag if we are exactly on a tag, otherwise <tag>-<n>-g<shortened sha-1>
+  try:
+      tag = subprocess.check_output(['git', 'describe', '--tags'], stderr = subprocess.STDOUT).splitlines()[0].decode().split('-')
+  except subprocess.CalledProcessError as e:
+      # no tags reachable (e.g. on a topic branch in a fork), see
+      # https://stackoverflow.com/questions/4916492/git-describe-fails-with-fatal-no-names-found-cannot-describe-anything
+      if e.output.rstrip() == b"fatal: No names found, cannot describe anything.":
+          tag=[]
+      else:
+          print(e.output); raise
+
+  if len(tag) == 1:
+      # tag identifies the build and should be a sequential revision number
+      version = tag[0]
+      opencv_version += ".{}".format(version)
+  else:
+      # local version identifier, not to be published on PyPI
+      version = git_hash
+      opencv_version += "+{}".format(version)
+
+  with open('cv2/version.py', 'w') as f:
+      f.write("opencv_version = \"{}\"\n".format(opencv_version))
+      f.write("contrib = {}\n".format(contrib))
+      f.write("headless = {}".format(headless))

pyproject.toml

+[build-system]
+requires = ["setuptools", "wheel", "scikit-build", "cmake"]
\ No newline at end of file

setup.py

-# No 3rd-party modules here, see "3rd-party" note below
 import io
 import os
 import os.path
@@ -7,23 +6,20 @@ import runpy
 import subprocess
 import re
 import sysconfig
+import skbuild
+from skbuild import cmaker
 
 
 def main():
-
     os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
-    # These are neede for source fetching
+    # These are needed for source fetching
     cmake_source_dir = "opencv"
+    minimum_supported_numpy = "1.11.1"
     build_contrib = get_build_env_var_by_name("contrib")
     # headless flag to skip GUI deps if needed
     build_headless = get_build_env_var_by_name("headless")
 
-    # Only import 3rd-party modules after having installed all the build dependencies:
-    # any of them, or their dependencies, can be updated during that process,
-    # leading to version conflicts
-    minimum_supported_numpy = "1.11.1"
-
     if sys.version_info[:2] >= (3, 6):
         minimum_supported_numpy = "1.11.3"
     if sys.version_info[:2] >= (3, 7):
@@ -31,21 +27,14 @@ def main():
     if sys.version_info[:2] >= (3, 8):
         minimum_supported_numpy = "1.17.3"
 
-    numpy_version = get_or_install("numpy", minimum_supported_numpy)
-    get_or_install("scikit-build")
-    get_or_install("cmake")
-
-    import skbuild
-    from skbuild import cmaker
+    numpy_version = "numpy>=%s" % minimum_supported_numpy
 
     python_version = cmaker.CMaker.get_python_version()
     python_lib_path = cmaker.CMaker.get_python_library(python_version).replace('\\', '/')
     python_include_dir = cmaker.CMaker.get_python_include_dir(python_version).replace('\\', '/')
 
     if os.path.exists(".git"):
-
         import pip._internal.vcs.git as git
-
         g = git.Git()  # NOTE: pip API's are internal, this has to be refactored
 
         g.run_command(["submodule", "sync"])
@@ -58,6 +47,8 @@ def main():
                 ["submodule", "update", "--init", "--recursive", "opencv_contrib"]
             )
 
+    package_version, build_contrib, build_headless = get_opencv_version(build_contrib, build_headless)
+
     # https://stackoverflow.com/questions/1405913/python-32bit-or-64bit-mode
     x64 = sys.maxsize > 2 ** 32
 
@@ -73,12 +64,11 @@ def main():
         package_name = "opencv-python-headless"
 
     long_description = io.open("README.md", encoding="utf-8").read()
-    package_version = get_opencv_version()
 
     packages = ["cv2", "cv2.data"]
 
     package_data = {
-        "cv2": ["*%s" % sysconfig.get_config_vars().get("SO")]
+        "cv2": ["*%s" % sysconfig.get_config_vars().get("SO"), "version.py"]
         + (["*.dll"] if os.name == "nt" else [])
         + ["LICENSE.txt", "LICENSE-3RD-PARTY.txt"],
         "cv2.data": ["*.xml"],
@@ -96,7 +86,7 @@ def main():
         # In Windows, in python/X.Y/<arch>/; in Linux, in just python/X.Y/.
         # Naming conventions vary so widely between versions and OSes
         # had to give up on checking them.
-        ["python/cv2[^/]*%(ext)s" % {"ext": re.escape(sysconfig.get_config_var("SO"))}],
+        ["python/cv2[^/]*%(ext)s" % {"ext": re.escape(sysconfig.get_config_var("EXT_SUFFIX"))}],
         "cv2.data": [  # OPENCV_OTHER_INSTALL_PATH
             ("etc" if os.name == "nt" else "share/opencv4") + r"/haarcascades/.*\.xml"
         ],
@@ -108,7 +98,7 @@ def main():
 
     cmake_args = (
         (
-            ["-G", "Visual Studio 14" + (" Win64" if x64 else "")]
+            ["-G", "Visual Studio 15" + (" Win64" if x64 else "")]
             if os.name == "nt"
             else ["-G", "Unix Makefiles"]  # don't make CMake try (and fail) Ninja first
         )
@@ -170,15 +160,6 @@ def main():
     if sys.platform == "darwin":
         subprocess.check_call("patch -p1 < patches/patchQtPlugins", shell=True)
 
-    # Fixes for macOS builds
-    if sys.platform == "darwin":
-        cmake_args.append("-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=10.9")
-
-    if "CMAKE_ARGS" in os.environ:
-        import shlex
-        cmake_args.extend(shlex.split(os.environ["CMAKE_ARGS"]))
-        del shlex
-
     # works via side effect
     RearrangeCMakeOutput(
         rearrange_cmake_output_data, files_outside_package_dir, package_data.keys()
@@ -197,7 +178,7 @@ def main():
         maintainer="Olli-Pekka Heinisuo",
         include_package_data=True,
         ext_modules=EmptyListWithLength(),
-        install_requires="numpy>=%s" % numpy_version,
+        install_requires=numpy_version,
         classifiers=[
             "Development Status :: 5 - Production/Stable",
             "Environment :: Console",
@@ -377,19 +358,22 @@ class RearrangeCMakeOutput(object):
             cmake_install_dir=cmake_install_reldir,
         )
 
+def get_opencv_version(contrib, headless):
+    # cv2/version.py should be generated by running find_version.py
+    version = {}
+    here = os.path.abspath(os.path.dirname(__file__))
+    version_file = os.path.join(here, "cv2", "version.py")
 
-def install_packages(*requirements):
-    # No more convenient way until PEP 518 is implemented; setuptools only handles eggs
-    subprocess.check_call([sys.executable, "-m", "pip", "install"] + list(requirements))
-
+    if not os.path.exists(version_file):
+      old_args = sys.argv.copy()
+      sys.argv = ['', str(contrib), str(headless)]
+      runpy.run_path("find_version.py ", run_name="__main__")
+      sys.argv = old_args
 
-def get_opencv_version():
-    # cv_version.py should be generated by running find_version.py
-    runpy.run_path("find_version.py")
-    from cv_version import opencv_version
-
-    return opencv_version
+    with open(version_file) as fp:
+        exec(fp.read(), version)
 
+    return version['opencv_version'], version['contrib'], version['headless']
 
 def get_build_env_var_by_name(flag_name):
     flag_set = False
@@ -407,26 +391,6 @@ def get_build_env_var_by_name(flag_name):
 
     return flag_set
 
-
-def get_or_install(name, version=None):
-    """ If a package is already installed, build against it. If not, install """
-    # Do not import 3rd-party modules into the current process
-    import json
-
-    js_packages = json.loads(
-        subprocess.check_output(
-            [sys.executable, "-m", "pip", "list", "--format", "json"]
-        ).decode("ascii")
-    )  # valid names & versions are ASCII as per PEP 440
-    try:
-        [package] = (package for package in js_packages if package["name"] == name)
-    except ValueError:
-        install_packages("%s==%s" % (name, version) if version else name)
-        return version
-    else:
-        return package["version"]
-
-
 # This creates a list which is empty but returns a length of 1.
 # Should make the wheel a binary distribution and platlib compliant.
 class EmptyListWithLength(list):