Retry GitHub status updates

hack/jenkins/common.sh

@@ -289,11 +289,41 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true
 rmdir "${TEST_HOME}"
 echo ">> ${TEST_HOME} completed at $(date)"
 
-if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then
-  readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
-  curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \
-  -H "Content-Type: application/json" \
-  -X POST \
-  -d "{\"state\": \"$status\", \"description\": \"Jenkins\", \"target_url\": \"$target_url\", \"context\": \"${JOB_NAME}\"}"
+if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then
+  exit $result
 fi
+
+# retry_github_status provides reliable github status updates
+function retry_github_status() {
+  local pr=$1
+  local context=$2
+  local state=$4
+  local token=$5
+  local target=$6
+
+   # Retry in case we hit our GitHub API quota or fail other ways.
+  local attempt=0
+  local timeout=2
+  local code=-1
+
+  while [[ "${attempt}" -lt 6 ]]; do
+    code=$(curl -s -i --write-out "%{http_code}" -L \
+      "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \
+      -H "Content-Type: application/json" \
+      -X POST \
+      -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999)
+
+    # 2xx HTTP codes
+    if [[ "${code}" =~ ^2 ]]; then
+      break
+    fi
+
+    echo "HTTP code ${code}! Retrying in ${timeout} .."
+    sleep "${timeout}"
+    attempt=$(( attempt + 1 ))
+    timeout=$(( timeout * 2 ))
+  done
+}
+
+retry_github_status "${MINIKUBE_LOCATION}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
 exit $result

hack/jenkins/minikube_set_pending.sh

@@ -42,11 +42,40 @@ jobs=(
      'none_Linux'
 )
 
+# retry_github_status provides reliable github status updates
+function retry_github_status() {
+  local pr=$1
+  local context=$2
+  local state=$4
+  local token=$5
+  local target=$6
+
+   # Retry in case we hit our GitHub API quota or fail other ways.
+  local attempt=0
+  local timeout=2
+  local code=-1
+
+  while [[ "${attempt}" -lt 6 ]]; do
+    code=$(curl -s -i --write-out "%{http_code}" -L \
+      "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \
+      -H "Content-Type: application/json" \
+      -X POST \
+      -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999)
+
+    # 2xx HTTP codes
+    if [[ "${code}" =~ ^2 ]]; then
+      break
+    fi
+
+    echo "HTTP code ${code}! Retrying in ${timeout} .."
+    sleep "${timeout}"
+    attempt=$(( attempt + 1 ))
+    timeout=$(( timeout * 2 ))
+  done
+}
+
 for j in ${jobs[@]}; do
-  target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt"
-  curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \
-    -H "Content-Type: application/json" \
-    -X POST \
-    -d "{\"state\": \"pending\", \"description\": \"Jenkins\", \"target_url\": \"${target_url}\", \"context\": \"${j}\"}"
+  update_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \
+    "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt"
 done