Make library for auto building and caching Homebrew bottles

cache Homebrew

.travis.yml

@@ -25,6 +25,15 @@ dist: trusty
 git:
     submodules: false
 
+# https://docs.travis-ci.com/user/caching
+cache:
+  directories:
+    # https://stackoverflow.com/questions/39930171/cache-brew-builds-with-travis-ci
+    - $HOME/Library/Caches/Homebrew
+    - /usr/local/Homebrew/
+    # used in OSX custom build script dealing with local bottle caching
+    - $HOME/local_bottle_metadata
+    
 matrix:
   fast_finish: true
   include:
@@ -526,7 +535,34 @@ install: |
 script: |
     # Install and run tests
     set -x
-    install_run $PLAT
+    install_run $PLAT && rc=$? || rc=$?
+    set +x
+
+    #otherwise, Travis logic terminates prematurely
+    #https://travis-ci.community/t/shell-session-update-command-not-found-in-build-log-causes-build-to-fail-if-trap-err-is-set/817
+    trap ERR
+    
+    test "$rc" -eq 0
+
+before_cache: |
+    # Cleanup dirs to be cached
+    set -x
+    if [ -n "$IS_OSX" ]; then
+
+        # When Taps is cached, this dir causes "Error: file exists" on `brew update`
+        rm -rf "$(brew --repository)/Library/Taps/homebrew/homebrew-cask/homebrew-cask"
+        find "$(brew --repository)/Library/Taps" -type d -name .git -exec \
+            bash -xec '
+                cd $(dirname '\''{}'\'')
+                git status
+                # https://stackoverflow.com/questions/8296710/how-to-ignore-xargs-commands-if-stdin-input-is-empty/19038748#19038748
+                git ls-files --other -z | xargs -0 -n100 git add
+                git commit -a -m "Travis auto changes"
+                [[ -n $(git stash list) ]] && git stash drop' \;
+
+        brew_cache_cleanup
+        
+    fi
     set +x
     
 after_success: |

config.sh

@@ -24,28 +24,46 @@ else
   echo "    > Linux environment "
 fi
 
+if [ -n "$IS_OSX" ]; then
+
+    source travis_osx_brew_cache.sh
+    
+    BREW_SLOW_BUILIDING_PACKAGES=$(printf '%s\n' \
+        "x265 20"  \
+        "cmake 15" \
+        "ffmpeg 10" \
+    )
+    
+    #Contrib adds significantly to project's build time
+    if [ "$ENABLE_CONTRIB" -eq 1 ]; then
+        BREW_TIME_LIMIT=$((BREW_TIME_LIMIT - 10*60))
+    fi
+        
+fi
+
 function pre_build {
   echo "Starting pre-build"
-  set -e
+  set -e -o pipefail
 
   if [ -n "$IS_OSX" ]; then
     echo "Running for OSX"
     
-    brew update
+    brew update --merge
+    brew_add_local_bottles
+
+    # Don't query analytical info online on `brew info`,
+    #  this takes several seconds and we don't need it
+    # see https://docs.brew.sh/Manpage , "info formula" section
+    export HOMEBREW_NO_GITHUB_API=1
 
     echo 'Installing QT4'
     brew tap | grep -qxF cartr/qt4 || brew tap -v cartr/qt4
     brew tap --list-pinned | grep -qxF cartr/qt4 || brew tap-pin -v cartr/qt4
-    brew list --versions qt@4 || brew install -v qt@4
-    echo '-----------------'
-    echo '-----------------'
+    brew_install_and_cache_within_time_limit qt@4 || { [ $? -gt 1 ] && return 2 || return 0; }
+
     echo 'Installing FFmpeg'
-    # brew install does produce output regularly on a regular MacOS,
-    # but Travis doesn't see it for some reason
-    brew list --versions ffmpeg || \
-    travis_wait brew install -v ffmpeg --without-x264 --without-xvid --without-gpl
-    brew info ffmpeg
-    echo '-----------------'
+
+    brew_install_and_cache_within_time_limit ffmpeg || { [ $? -gt 1 ] && return 2 || return 0; }
 
   else
     echo "Running for linux"

travis_osx_brew_cache.sh

+# Library to cache downloaded and locally-built Homebrew bottles in Travis OSX build.
+
+trap '{ sleep 1;    #if we terminale too abruptly, Travis will lose some log output
+        exit 2;     #The trap isn''t called in the parent function, so can''t use `return` here.
+                    #`exit` will terminate the entire build but it seems we have no choice.
+}' ERR
+set -E
+
+#Should be in Travis' cache
+BREW_LOCAL_BOTTLE_METADATA="$HOME/local_bottle_metadata"
+
+# Starting reference point for elapsed build time; seconds since the epoch.
+#TRAVIS_TIMER_START_TIME is set at the start of a log fold, in nanoseconds since the epoch
+BREW_TIME_START=$(($TRAVIS_TIMER_START_TIME/10**9))
+
+# If after a package is built, elapsed time is more than this many seconds, fail the build but save Travis cache
+# The cutoff moment should leave enough time till Travis' job time limit to process the main project.
+BREW_TIME_LIMIT=$((30*60))
+# If a slow-building package is about to be built and the projected build end moment is beyond this many seconds,
+# skip that build, fail the Travis job and save Travis cache.
+# This cutoff should leave enough time for before_cache and cache save.
+BREW_TIME_HARD_LIMIT=$((40*60))
+
+
+
+#Public functions
+
+function brew_install_and_cache_within_time_limit {
+    # Install the package and its dependencies one by one;
+    # use bottle if available, build and cache bottle if not.
+    # Terminate and exit with status 1 if this takes too long.
+    # Exit with status 2 on any other error.
+    
+    local PACKAGE; PACKAGE="${1:?}"
+    local TIME_LIMIT;TIME_LIMIT=${2:-$BREW_TIME_LIMIT}
+    local TIME_HARD_LIMIT;TIME_HARD_LIMIT=${3:-$BREW_TIME_HARD_LIMIT}
+    local TIME_START;TIME_START=${4:-$BREW_TIME_START}
+
+    local BUILD_FROM_SOURCE INCLUDE_BUILD
+    
+    _brew_is_bottle_available "$PACKAGE" || BUILD_FROM_SOURCE=1
+    [ -n "$BUILD_FROM_SOURCE" ] && INCLUDE_BUILD="--include-build" || true
+
+    # Whitespace is illegal in package names so converting all whitespace into single spaces due to no quotes is okay.
+    DEPS=`brew deps "$PACKAGE" $INCLUDE_BUILD`
+    for dep in $DEPS; do
+        #TIME_LIMIT only has to be met if we'll be actually building the main project this iteration, i.e. after the "root" module installation
+        #While we don't know that yet, we can make better use of Travis-given time with a laxer limit
+        #We still can't overrun TIME_HARD_LIMIT as that would't leave time to save the cache
+        brew_install_and_cache_within_time_limit "$dep" $(((TIME_LIMIT+TIME_HARD_LIMIT)/2)) "$TIME_HARD_LIMIT" "$TIME_START" || return $?
+    done
+
+    _brew_check_slow_building_ahead "$PACKAGE" "$TIME_START" "$TIME_HARD_LIMIT" || return $?
+    _brew_install_and_cache "$PACKAGE" "$([[ -z "$INCLUDE_BUILD" ]] && echo 1 || echo 0)"
+    _brew_check_elapsed_build_time "$TIME_START" "$TIME_LIMIT" || return $?
+}
+
+function brew_add_local_bottles {
+    # Should be called after `brew update` at startup.
+    # Adds metadata for cached locally-built bottles to local formulas
+    #  so that `brew` commands can find them.
+    # If the package was updated, removes the corresponding files
+    #  and the bottle's entry in the formula, if any.
+    
+    # Bottle entry in formula: 
+    #   bottle do
+    #     <...>
+    #     sha256 "<sha256>" => :<os_codename>
+    #     <...>
+    #   end 
+    
+    echo "Cached bottles:"
+    ls "$(brew --cache)/downloads" || true  #may not exist initially since it's "$(brew --cache)" that is in Travis cache
+    echo "Saved .json's and links:"
+    ls "$BREW_LOCAL_BOTTLE_METADATA"
+    
+    for JSON in "$BREW_LOCAL_BOTTLE_METADATA"/*.json; do
+        [ -e "$JSON" ] || break    # OSX 10.11 bash has no nullglob
+        local PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH
+        
+        _brew_parse_bottle_json "$JSON" PACKAGE JSON_VERSION JSON_REBUILD OS_CODENAME BOTTLE_HASH
+
+        echo "Adding local bottle: $PACKAGE ${JSON_VERSION}_${JSON_REBUILD}"
+        
+        local FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH
+        
+        _brew_parse_package_info "$PACKAGE" "$OS_CODENAME" FORMULA_VERSION FORMULA_REBUILD FORMULA_BOTTLE_HASH
+
+        local FORMULA_HAS_BOTTLE; [ -n "$FORMULA_BOTTLE_HASH" ] && FORMULA_HAS_BOTTLE=1 || true
+        
+
+        local BOTTLE_LINK BOTTLE; BOTTLE_LINK="${JSON}.bottle.lnk";
+        local BOTTLE_EXISTS BOTTLE_MISMATCH VERSION_MISMATCH
+
+
+        # Check that the bottle file exists and is still appropriate for the formula
+        if [[ "$FORMULA_VERSION" != "$JSON_VERSION" || "$JSON_REBUILD" != "$FORMULA_REBUILD" ]]; then
+            VERSION_MISMATCH=1;
+            echo "The cached bottle is obsolete: formula ${FORMULA_VERSION}_${FORMULA_REBUILD}"
+        fi
+        if [ -f "$BOTTLE_LINK" ]; then
+            BOTTLE=$(cat "$BOTTLE_LINK");
+            BOTTLE=$(cd "$(dirname "$BOTTLE")"; pwd)/$(basename "$BOTTLE")
+            
+            if [ -e "$BOTTLE" ]; then
+                BOTTLE_EXISTS=1;
+                
+                # The hash in `brew --cache $PACKAGE` entry is generated from download URL,
+                #  which itself is generated from base URL and version
+                # (see Homebrew/Library/Homebrew/download_strategy.rb:cached_location).
+                # So if version changes, hashes will always mismatch anyway
+                #  and we don't need a separate message about this.
+                # XXX: OSX doesn't have `realpath` so can't compare the entire paths
+                if [ -n "$FORMULA_HAS_BOTTLE" -a -z "$VERSION_MISMATCH" -a \
+                    "$(basename "$(brew --cache "$PACKAGE")")" != "$(basename "$BOTTLE")" ]; then
+                        BOTTLE_MISMATCH=1;
+                        echo "Cached bottle file doesn't correspond to formula's cache entry!" \
+                             "This can happen if download URL has changed." >&2
+                fi
+            else
+                echo "Cached bottle file is missing!" >&2
+            fi
+        else
+            echo "Link file is missing or of invalid type!" >&2
+        fi
+                    
+        # Delete cached bottle and all metadata if invalid
+        if [[ -z "$BOTTLE_EXISTS" || -n "$VERSION_MISMATCH" || -n "$BOTTLE_MISMATCH" ]]; then
+            echo "Deleting the cached bottle and all metadata"
+
+            if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then
+                echo "A bottle block for the cached bottle was merged into the updated formula. Removing..."
+                local FORMULA; FORMULA=$(brew formula "$PACKAGE")
+                perl -wpe 'BEGIN { our $IN_BLOCK=0; }
+                    if ( ($IN_BLOCK==0) && /^\s*bottle\s+do\s*$/ ) { $IN_BLOCK=1; next; }
+                    if ( ($IN_BLOCK==1) && /^\s*end\s*$/ )         { $IN_BLOCK=-1; next; }
+                    if ( ($IN_BLOCK==1) && /^\s*sha256\s+"(\w+)"\s+=>\s+:\w+\s*$/ )
+                                                 { if ( $1 eq "'"$BOTTLE_HASH"'" ) {$_="";}; next; }
+                '  <"$FORMULA" >"${FORMULA}.new"
+                # Depending on diff version, 1 may mean differences found
+                # https://stackoverflow.com/questions/6971284/what-are-the-error-exit-values-for-diff
+                diff -u "$FORMULA" "${FORMULA}.new" || test $? -le 1
+                (   cd $(dirname "$FORMULA")
+                    FORMULA=$(basename "$FORMULA")
+                    mv -v "${FORMULA}.new" "$FORMULA"
+                    git commit -m "Removed obsolete local bottle ${JSON_VERSION}_${JSON_REBUILD} :${OS_CODENAME}" "$FORMULA"
+                )
+            fi
+            
+            if [ -n "$BOTTLE" ]; then rm "$BOTTLE"; fi
+            rm -f "$BOTTLE_LINK"
+            rm "$JSON"
+            
+        #(Re)add metadata to the formula otherwise
+        else
+            if [ "$FORMULA_BOTTLE_HASH" == "$BOTTLE_HASH" ]; then
+                echo "The cached bottle is already present in the formula"
+            else
+                brew bottle --merge --write "$JSON"
+            fi
+        fi
+    done
+}
+
+
+function brew_cache_cleanup {
+    #Cleanup caching directories
+    # Is supposed to be called in before_cache
+
+    #Lefovers from some failure probably
+    rm -f "$BREW_LOCAL_BOTTLE_METADATA"/*.tar.gz
+    
+    #`brew cleanup` may delete locally-built bottles that weren't needed this time
+    # so we're saving and restoring them
+    local BOTTLE_LINK BOTTLE
+    for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+        BOTTLE=$(cat "$BOTTLE_LINK")
+        ln "$BOTTLE" "$BREW_LOCAL_BOTTLE_METADATA/"
+    done
+    brew cleanup
+    local BOTTLE_BASENAME
+    for BOTTLE_LINK in "$BREW_LOCAL_BOTTLE_METADATA"/*.lnk; do
+        BOTTLE=$(cat "$BOTTLE_LINK")
+        BOTTLE_BASENAME=$(basename "$BOTTLE")
+        if test ! -e "$BOTTLE"; then
+            echo "Restoring: $BOTTLE_BASENAME"
+            mv "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME" "$BOTTLE"
+        else
+            rm "$BREW_LOCAL_BOTTLE_METADATA/$BOTTLE_BASENAME"
+        fi
+    done
+}
+
+
+function brew_go_bootstrap_mode {
+# Can be overridden
+# Terminate the build but ensure saving the cache
+
+    echo "Going into cache bootstrap mode"
+        
+    #Can't just `exit` because that would terminate the build without saving the cache
+    #Have to replace further actions with no-ops
+    
+    eval '
+    function '"$cmd"' { return 0; }
+    function repair_wheelhouse { return 0; }
+    function install_run {
+        echo -e "\nBuilding dependencies took too long. Restart the build in Travis UI to continue from cache.\n"
+        
+        # Travis runs user scripts via `eval` i.e. in the same shell process.
+        # So have to unset errexit in order to get to cache save stage
+        set +e; return 1
+    }'    
+}
+
+
+
+#Internal functions
+
+function _brew_parse_bottle_json {
+    # Parse JSON info about a package
+    # from `brew info --json=v1` input or a JSON file on stdin
+    # and save it into bash global variables specified in arguments
+
+    local JSON; JSON="${1:?}"; shift
+
+    local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True:
+    import sys,json; j=json.load(open(sys.argv[1],"rb")); [name]=j.keys(); [pdata]=j.values()
+    print name
+    print pdata["formula"]["pkg_version"]
+    print pdata["bottle"]["rebuild"]
+    [(tag_name, tag_dict)]=pdata["bottle"]["tags"].items()
+    print tag_name
+    print tag_dict["sha256"]
+    ' "$JSON")
+    
+    unset JSON
+    
+    { local i v; for i in {1..5}; do
+        read -r v
+        eval "${1:?}=\"$v\""
+        shift
+    done } <<< "$JSON_DATA"
+}
+
+function _brew_parse_package_info {
+    # Get and parse `brew info --json` about a package
+    # and save it into bash variables specified in arguments
+    
+    local PACKAGE; PACKAGE="${1:?}"; shift
+    local OS_CODENAME;OS_CODENAME="${1:?}"; shift
+
+    local JSON_DATA; JSON_DATA=$(python2.7 -c 'if True:
+    import sys, json, subprocess; j=json.loads(subprocess.check_output(("brew","info","--json=v1",sys.argv[1])))
+    data=j[0]
+    print data["versions"]["stable"]
+    bottle_data=data["bottle"]["stable"]
+    print bottle_data["rebuild"]
+    print bottle_data["files"].get(sys.argv[2],{"sha256":""})["sha256"]
+    ' \
+    "$PACKAGE" "$OS_CODENAME")
+    
+    unset PACKAGE OS_CODENAME
+    
+    { local i v; for i in {1..3}; do
+        read -r v
+        eval "${1:?}=\"$v\""
+        shift
+    done } <<< "$JSON_DATA"
+}
+
+
+
+function _brew_is_bottle_available {
+
+    local PACKAGE;PACKAGE="${1:?}"
+    
+    local INFO="$(brew info "$PACKAGE" | head -n 1)"
+    if grep -qwF '(bottled)' <<<"$INFO"; then
+        echo "Bottle available: $INFO"
+        return 0
+    else
+        echo "Bottle not available: $INFO"
+        return 1
+    fi
+}
+
+function _brew_install_and_cache {
+    # Install bottle or make and cache bottle.
+    # assumes that deps were already installed.
+    
+    local PACKAGE;PACKAGE="${1:?}"
+    local USE_BOTTLE;USE_BOTTLE="${2:?}"
+    local VERB
+    
+    if brew list --versions "$PACKAGE"; then
+        if ! (brew outdated | grep -qx "$PACKAGE"); then
+            echo "Already the latest version: $PACKAGE"
+            return 0
+        fi
+        VERB=upgrade
+    else
+        VERB=install
+    fi
+    
+    if [[ "$USE_BOTTLE" -gt 0 ]]; then
+        echo "Installing bottle for: $PACKAGE"
+        brew $VERB "$PACKAGE"
+    else
+        echo "Building bottle for: $PACKAGE"
+        brew $VERB --build-bottle "$PACKAGE"
+        exec 3>&1
+        local OUT=$(brew bottle --json "$PACKAGE" | tee /dev/fd/3)
+        exec 3>&-
+
+        ls "$PACKAGE"*
+        # doesn't seem to be a documented way to get file names
+        local BOTTLE; BOTTLE=$(grep -Ee '^./' <<<"$OUT")
+        #proper procedure as per https://discourse.brew.sh/t/how-are-bottle-and-postinstall-related-is-it-safe-to-run-bottle-after-postinstall/3410/4
+        brew uninstall "$PACKAGE"
+        brew install "$BOTTLE"
+        
+        local JSON; JSON=$(sed -E 's/bottle(.[[:digit:]]+)?\.tar\.gz$/bottle.json/' <<<"$BOTTLE")
+        
+        #`brew bottle --merge` doesn't return nonzero on nonexisting json file
+        test -f "$JSON" -a -f "$BOTTLE"
+        
+        brew bottle --merge --write "$JSON"
+        local CACHED_BOTTLE; CACHED_BOTTLE="$(brew --cache "$PACKAGE")"
+        mv "$BOTTLE" "$CACHED_BOTTLE";
+        local CACHED_JSON; CACHED_JSON="${BREW_LOCAL_BOTTLE_METADATA}/$(basename "$JSON")"
+        mv "$JSON" "$CACHED_JSON"
+        #Symlinks aren't cached by Travis. Will just save paths in files then.
+        local BOTTLE_LINK; BOTTLE_LINK="${CACHED_JSON}.bottle.lnk"
+        echo "$CACHED_BOTTLE" >"$BOTTLE_LINK"
+        
+    fi
+}
+
+
+
+
+function _brew_check_elapsed_build_time {
+    # If time limit has been reached,
+    # arrange for further build to be skipped and return 1
+
+    local TIME_START;TIME_START="${1:?}"
+    local TIME_LIMIT;TIME_LIMIT="${2:?}"
+    
+    local ELAPSED_TIME;ELAPSED_TIME=$(($(date +%s) - $TIME_START))
+    echo "Elapsed time: "$(($ELAPSED_TIME/60))"m (${ELAPSED_TIME}s)"
+    
+    if [[ "$ELAPSED_TIME" -gt $TIME_LIMIT ]]; then 
+        brew_go_bootstrap_mode
+        return 1
+    fi
+    return 0
+}
+
+function _brew_check_slow_building_ahead {
+
+    #If the package's projected build completion is higher than hard limit,
+    # skip it and arrange for further build to be skipped and return 1
+    
+    local PACKAGE="${1:?}"
+    local TIME_START="${2:?}"
+    local TIME_HARD_LIMIT="${3:?}"
+    
+    PROJECTED_BUILD_TIME=$(echo "$BREW_SLOW_BUILIDING_PACKAGES" | awk '$1=="'"$PACKAGE"'"{print $2}')
+    [ -z "$PROJECTED_BUILD_TIME" ] && return 0 || true
+    
+    local PROJECTED_BUILD_END_ELAPSED_TIME
+    PROJECTED_BUILD_END_ELAPSED_TIME=$(( $(date +%s) - TIME_START + PROJECTED_BUILD_TIME * 60))
+    
+    if [[ "$PROJECTED_BUILD_END_ELAPSED_TIME" -ge "$TIME_HARD_LIMIT" ]]; then
+        echo -e "\nProjected build end elapsed time for $PACKAGE: $((PROJECTED_BUILD_END_ELAPSED_TIME/60))m ($PROJECTED_BUILD_END_ELAPSED_TIMEs)"
+        brew_go_bootstrap_mode
+        return 1
+    fi
+    return 0
+}