From 02cfbf31c585bc142fd3341c431d4b94848db8eb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 23 Mar 2020 19:55:39 +0800 Subject: [PATCH] #1656 Update ccache script file (#1716) * Increase memory limit in Jenkins CI Signed-off-by: quicksilver * Add pod tolerations to slave pods * Refactor ci/scripts/build.sh * Refactor ci/scripts/build.sh * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Add timeout for whole Jenkins job * cuda yum repository temporarily disabled * Update ccache script file * Update ccache script file * Update ccache script file Signed-off-by: quicksilver * Update ccache script file Signed-off-by: quicksilver * Update ccache script file Signed-off-by: quicksilver * Update ccache script file Signed-off-by: quicksilver --- ci/scripts/check_ccache.sh | 80 ++++++++++++++----------- ci/scripts/update_ccache.sh | 113 +++++++++++++++++++++--------------- 2 files changed, 110 insertions(+), 83 deletions(-) diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index 876ff83d..61055b37 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,48 +1,58 @@ #!/bin/bash -OS_NAME="${OS_NAME}" -BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" -BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') -ARTIFACTORY_URL="" -CCACHE_DIRECTORY="${HOME}/.ccache" +HELP=" +Usage: + $0 [flags] [Arguments] -while getopts "l:d:h" arg -do - case $arg in - l) - ARTIFACTORY_URL=$OPTARG - ;; - d) - CCACHE_DIRECTORY=$OPTARG - ;; - h) # help - echo " + -l [ARTIFACTORY_URL] Artifactory URL + --cache_dir=[CCACHE_DIR] Ccache directory + -f [FILE] or --file=[FILE] Ccache compress package file + -h or --help Print help information -parameter: --l: artifactory url --d: ccache directory --h: help -usage: -./build.sh -l \${ARTIFACTORY_URL} -d \${CCACHE_DIRECTORY} [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; +Use \"$0 --help\" for more information about a given command. +" + +ARGS=$(getopt -o "l:f:h" -l "cache_dir::,file::,help" -n "$0" -- "$@") + +eval set -- "${ARGS}" + +while true ; do + case "$1" in + -l) + # o has an optional argument. As we are in quoted mode, + # an empty parameter will be generated if its optional + # argument is not found. + case "$2" in + "") echo "Option Artifactory URL, no argument"; exit 1 ;; + *) ARTIFACTORY_URL=$2 ; shift 2 ;; + esac ;; + --cache_dir) + case "$2" in + "") echo "Option cache_dir, no argument"; exit 1 ;; + *) CCACHE_DIR=$2 ; shift 2 ;; + esac ;; + -f|--file) + case "$2" in + "") echo "Option file, no argument"; exit 1 ;; + *) PACKAGE_FILE=$2 ; shift 2 ;; + esac ;; + -h|--help) echo -e "${HELP}" ; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; esac done +# Set defaults for vars modified by flags to this script +CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} +PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"} +BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') + if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then - echo "you have not input ARTIFACTORY_URL !" + echo "You have not input ARTIFACTORY_URL !" exit 1 fi - -PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" - function check_ccache() { BRANCH=$1 echo "fetching ${BRANCH}/${PACKAGE_FILE}" @@ -53,8 +63,8 @@ function check_ccache() { function download_file() { BRANCH=$1 wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ - mkdir -p ${CCACHE_DIRECTORY} && \ - tar zxf ${PACKAGE_FILE} -C ${CCACHE_DIRECTORY} && \ + mkdir -p "${CCACHE_DIR}" && \ + tar zxf "${PACKAGE_FILE}" -C "${CCACHE_DIR}" && \ rm ${PACKAGE_FILE} return $? } diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index ec8929ff..6c720805 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,82 +1,97 @@ #!/bin/bash -OS_NAME="${OS_NAME}" -BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" -BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') -ARTIFACTORY_URL="" -ARTIFACTORY_USER="" -ARTIFACTORY_PASSWORD="" -CCACHE_DIRECTORY="${HOME}/.ccache" +HELP=" +Usage: + $0 [flags] [Arguments] -while getopts "l:u:p:d:h" arg -do - case $arg in - l) - ARTIFACTORY_URL=$OPTARG - ;; - u) - ARTIFACTORY_USER=$OPTARG - ;; - p) - ARTIFACTORY_PASSWORD=$OPTARG - ;; - d) - CCACHE_DIRECTORY=$OPTARG - ;; - h) # help - echo " + -l [ARTIFACTORY_URL] Artifactory URL + --cache_dir=[CCACHE_DIR] Ccache directory + -f [FILE] or --file=[FILE] Ccache compress package file + -u [USERNAME] Artifactory Username + -p [PASSWORD] Artifactory Password + -h or --help Print help information -parameter: --l: artifactory url --u: artifactory user --p: artifactory password --d: ccache directory --h: help -usage: -./build.sh -l \${ARTIFACTORY_URL} -u \${ARTIFACTORY_USER} -p \${ARTIFACTORY_PASSWORD} -d \${CCACHE_DIRECTORY} [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; +Use \"$0 --help\" for more information about a given command. +" + +ARGS=$(getopt -o "l:f:u:p:h" -l "cache_dir::,file::,help" -n "$0" -- "$@") + +eval set -- "${ARGS}" + +while true ; do + case "$1" in + -l) + # o has an optional argument. As we are in quoted mode, + # an empty parameter will be generated if its optional + # argument is not found. + case "$2" in + "") echo "Option Artifactory URL, no argument"; exit 1 ;; + *) ARTIFACTORY_URL=$2 ; shift 2 ;; + esac ;; + --cache_dir) + case "$2" in + "") echo "Option cache_dir, no argument"; exit 1 ;; + *) CCACHE_DIR=$2 ; shift 2 ;; + esac ;; + -u) + case "$2" in + "") echo "Option Username, no argument"; exit 1 ;; + *) USERNAME=$2 ; shift 2 ;; + esac ;; + -p) + case "$2" in + "") echo "Option Password, no argument"; exit 1 ;; + *) PASSWORD=$2 ; shift 2 ;; + esac ;; + -f|--file) + case "$2" in + "") echo "Option file, no argument"; exit 1 ;; + *) PACKAGE_FILE=$2 ; shift 2 ;; + esac ;; + -h|--help) echo -e "${HELP}" ; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; esac done +# Set defaults for vars modified by flags to this script +CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} +PACKAGE_FILE=${PACKAGE_FILE:="ccache-${OS_NAME}-${BUILD_ENV_IMAGE_ID}.tar.gz"} +BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') + if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then - echo "you have not input ARTIFACTORY_URL !" + echo "You have not input ARTIFACTORY_URL !" exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +if [[ ! -d "${CCACHE_DIR}" ]]; then + echo "\"${CCACHE_DIR}\" directory does not exist !" + exit 1 +fi function check_ccache() { BRANCH=$1 - echo "fetching ${BRANCH}/${PACKAGE_FILE}" wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" return $? } - if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then - REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" check_ccache ${CHANGE_TARGET} if [[ $? == 0 ]];then echo "Skip Update ccache package ..." && exit 0 fi fi -REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" - +echo -e "===\n=== ccache statistics after build\n===" ccache --show-stats if [[ "${BRANCH_NAME}" != "HEAD" ]];then + REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" echo "Updating ccache package file: ${PACKAGE_FILE}" - tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache . + tar zcf ./"${PACKAGE_FILE}" -C "${CCACHE_DIR}" . echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" - curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH}/${PACKAGE_FILE} + curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}" if [[ $? == 0 ]];then echo "Uploading ccache package file success !" exit 0 @@ -85,3 +100,5 @@ if [[ "${BRANCH_NAME}" != "HEAD" ]];then exit 1 fi fi + +echo "Skip Update ccache package ..." -- GitLab