diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 61d9dfe90caaea95305aad2ecbf447761b2f4da4..4d32792d4bb7d866936fe61880c1b27d27a7bd03 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -289,11 +289,43 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true rmdir "${TEST_HOME}" echo ">> ${TEST_HOME} completed at $(date)" -if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then - readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" - curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "{\"state\": \"$status\", \"description\": \"Jenkins\", \"target_url\": \"$target_url\", \"context\": \"${JOB_NAME}\"}" +if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then + exit $result fi + +# retry_github_status provides reliable github status updates +function retry_github_status() { + local commit=$1 + local context=$2 + local state=$3 + local token=$4 + local target=$5 + + # Retry in case we hit our GitHub API quota or fail other ways. + local attempt=0 + local timeout=2 + local code=-1 + + while [[ "${attempt}" -lt 8 ]]; do + local out=$(mktemp) + code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${commit}?access_token=${token}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) + + # 2xx HTTP codes + if [[ "${code}" =~ ^2 ]]; then + break + fi + + cat "${out}" && rm -f "${out}" + echo "HTTP code ${code}! Retrying in ${timeout} .." + sleep "${timeout}" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done +} + +retry_github_status "${COMMIT}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" exit $result diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index a595773ff8bfdf824e0048c870c842405050f777..75851e3e8fb1d155cf8bc26d98cfd18859a63f35 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -42,11 +42,42 @@ jobs=( 'none_Linux' ) +# retry_github_status provides reliable github status updates +function retry_github_status() { + local commit=$1 + local context=$2 + local state=$3 + local token=$4 + local target=$5 + + # Retry in case we hit our GitHub API quota or fail other ways. + local attempt=0 + local timeout=2 + local code=-1 + + while [[ "${attempt}" -lt 8 ]]; do + local out=$(mktemp) + code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${commit}?access_token=${token}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) + + # 2xx HTTP codes + if [[ "${code}" =~ ^2 ]]; then + break + fi + + cat "${out}" && rm -f "${out}" + echo "HTTP code ${code}! Retrying in ${timeout} .." + sleep "${timeout}" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done +} + for j in ${jobs[@]}; do - target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" - curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "{\"state\": \"pending\", \"description\": \"Jenkins\", \"target_url\": \"${target_url}\", \"context\": \"${j}\"}" + retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ + "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" done