From 6119eefa9ba08344786c8b4703c00818d72b87d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E9=B8=BF=E7=AB=A0?= Date: Fri, 3 Apr 2020 16:07:06 +0800 Subject: [PATCH] secure the scripts with set -e --- build/build.sh | 62 +++++++++++++--------------- build/scripts/crc32.sh | 94 +++++++++++++++++++----------------------- build/scripts/ui.sh | 29 ++++++------- tests/st/runtest.sh | 23 +++++------ tests/ut/runtest.sh | 19 ++++----- 5 files changed, 101 insertions(+), 126 deletions(-) diff --git a/build/build.sh b/build/build.sh index eaf787b..daa94b2 100755 --- a/build/build.sh +++ b/build/build.sh @@ -13,32 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCRIPT_BASEDIR=$( - cd "$(dirname "$0")" || exit - pwd -) +set -e + +SCRIPT_BASEDIR=$(realpath "$(dirname "$0")") + +PROJECT_BASEDIR=$(dirname "$SCRIPT_BASEDIR") rename_wheel() { + cd "$PROJECT_BASEDIR/output" || exit VERSION="$1" PACKAGE_LIST=$(ls mindinsight-*-any.whl) || exit - for PACKAGE_ORIG in ${PACKAGE_LIST}; do - MINDINSIGHT_VERSION=$(echo "${PACKAGE_ORIG}" | awk -F"-" '{print $2}') - PYTHON_VERSION_NUM=$(echo "${VERSION}" | awk -F"." '{print $1$2}') - PYTHON_VERSION_TAG="cp${PYTHON_VERSION_NUM}" + for PACKAGE_ORIG in $PACKAGE_LIST; do + MINDINSIGHT_VERSION=$(echo "$PACKAGE_ORIG" | awk -F"-" '{print $2}') + PYTHON_VERSION_NUM=$(echo "$VERSION" | awk -F"." '{print $1$2}') + PYTHON_VERSION_TAG="cp$PYTHON_VERSION_NUM" PYTHON_ABI_TAG="cp${PYTHON_VERSION_NUM}m" OS_NAME=$(uname | tr '[:upper:]' '[:lower:]') MACHINE_TAG="${OS_NAME}_$(uname -i)" - PACKAGE_NEW="mindinsight-${MINDINSIGHT_VERSION}-${PYTHON_VERSION_TAG}-${PYTHON_ABI_TAG}-${MACHINE_TAG}.whl" - mv "${PACKAGE_ORIG}" "${PACKAGE_NEW}" + PACKAGE_NEW="mindinsight-$MINDINSIGHT_VERSION-$PYTHON_VERSION_TAG-$PYTHON_ABI_TAG-$MACHINE_TAG.whl" + mv "$PACKAGE_ORIG" "$PACKAGE_NEW" done } build_wheel() { - PROJECT_BASEDIR=$( - cd "$(dirname "$SCRIPT_BASEDIR")" || exit - pwd - ) - cd "${PROJECT_BASEDIR}" || exit + + cd "$PROJECT_BASEDIR" || exit if [ $# -gt 0 ]; then if [ "$1" = "clean" ]; then @@ -54,43 +53,41 @@ build_wheel() { echo "start building mindinsight" clean_files - PYTHON=$(command -v python3 || command -v python) - if [ -z "${PYTHON}" ]; then - echo "Could not find python3 or python command" - exit 1 + if command -v python3; then + PYTHON=python3 + elif command -v python; then + PYTHON=python + else + command python3 fi - PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*') - if [ -z "${PYTHON_VERSION}" ]; then - echo "Could not find Python 3" + + if ! "$PYTHON" -c 'import sys; assert sys.version_info >= (3, 7)' &>/dev/null; then + echo "Python 3.7 or higher is required. You are running $("$PYTHON" -V)" exit 1 fi - rm -f output - mkdir output + rm -rf output - ${PYTHON} setup.py bdist_wheel + "$PYTHON" setup.py bdist_wheel if [ ! -x "dist" ]; then echo "Build failed" exit 1 fi - mv dist/mindinsight-*-any.whl output/ + mv dist output - cd output || exit - rename_wheel "${PYTHON_VERSION}" - cd - >/dev/null 2>&1 || exit + rename_wheel "$("$PYTHON" -c 'import platform; print(platform.python_version())')" clean_files - echo "Build success, output directory is: ${PROJECT_BASEDIR}/output" + echo "Build success, output directory is: $PROJECT_BASEDIR/output" } clean_files() { - rm -rf third_party/build + cd "$PROJECT_BASEDIR" || exit rm -rf build/lib rm -rf build/bdist.* rm -rf mindinsight.egg-info - rm -rf dist } show_usage() { @@ -120,5 +117,4 @@ check_opts() { check_opts "$@" -cd "${SCRIPT_BASEDIR}" || exit build_wheel "$@" diff --git a/build/scripts/crc32.sh b/build/scripts/crc32.sh index 0214c08..69df735 100755 --- a/build/scripts/crc32.sh +++ b/build/scripts/crc32.sh @@ -13,85 +13,75 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCRIPT_BASEDIR=$( - cd "$(dirname "$0")" || exit - pwd -) +set -e -THIRD_PARTY_DIR=$(realpath "${SCRIPT_BASEDIR}/../../third_party") -BUILDDIR=$(dirname "$SCRIPT_BASEDIR") +SCRIPT_BASEDIR=$(realpath "$(dirname "$0")") + +THIRD_PARTY_DIR=$(realpath "$SCRIPT_BASEDIR/../../third_party") +BUILDDIR="$(dirname "$SCRIPT_BASEDIR")/build_securec" build_securec() { - CMAKE=$(command -v cmake) - if [ -z "${CMAKE}" ]; then - echo "Could not find cmake command" - exit 1 - fi + rm -rf "$BUILDDIR" + mkdir "$BUILDDIR" cd "$BUILDDIR" || exit - ${CMAKE} .. + if ! command -v cmake; then + command cmake + fi + cmake ../.. make } clean_securec() { - cd "$BUILDDIR" || exit - for file in *; do - if [ "$file" == build.sh ] || [ "$file" == scripts ] || [ "$file" == lib ]; then - continue - fi - rm -rf "$file" - done + rm -rf "$BUILDDIR" } build_crc32() { - CPP=$(command -v c++) - if [ -z "${CPP}" ]; then - echo "Could not find c++ command" - exit 1 + DATAVISUAL_DIR=$(realpath "$SCRIPT_BASEDIR/../../mindinsight/datavisual") + CRC32_SOURCE_DIR="$DATAVISUAL_DIR/utils/crc32" + CRC32_OUTPUT_DIR="$DATAVISUAL_DIR/utils" + CRC32_SO_FILE="crc32$(python3-config --extension-suffix)" + + cd "$CRC32_SOURCE_DIR" || exit + + if ! command -v c++; then + command c++ fi - PYTHON=$(command -v python3 || command -v python) - if [ -z "${PYTHON}" ]; then - echo "Could not find python3 or python command" - exit 1 + if command -v python3; then + PYTHON=python3 + elif command -v python; then + PYTHON=python + else + command python3 fi - PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*') - if [ -z "${PYTHON_VERSION}" ]; then - echo "Could not find Python 3" + + if ! "$PYTHON" -c 'import sys; assert sys.version_info >= (3, 7)' &>/dev/null; then + echo "Python 3.7 or higher is required. You are running $("$PYTHON" -V)" exit 1 fi - DATAVISUAL_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/datavisual") - CRC32_SOURCE_DIR="${DATAVISUAL_DIR}/utils/crc32" - CRC32_OUTPUT_DIR="${DATAVISUAL_DIR}/utils" - CRC32_SO_FILE="crc32$(python3-config --extension-suffix)" + rm -f "$CRC32_SOURCE_DIR/$CRC32_SO_FILE" + rm -f "$CRC32_OUTPUT_DIR/$CRC32_SO_FILE" - rm -f "${CRC32_SOURCE_DIR}/${CRC32_SO_FILE}" - rm -f "${CRC32_OUTPUT_DIR}/${CRC32_SO_FILE}" - cd "${CRC32_SOURCE_DIR}" || exit - PYBIND11_INCLUDES=$(${PYTHON} -m pybind11 --includes) - if [ -z "${PYBIND11_INCLUDES}" ]; then - echo "Could not find pybind11 module" - exit 1 - fi + PYBIND11_INCLUDES=$($PYTHON -m pybind11 --includes) + PYTHON_INCLUDE=$(echo "$PYBIND11_INCLUDES" | awk '{print $1}' | sed "s/^-I//g") + PYTHON_HEADERS=$(echo "$PYBIND11_INCLUDES" | awk '{print $2}' | sed "s/^-I//g") - PYTHON_INCLUDE=$(echo "${PYBIND11_INCLUDES}" | awk '{print $1}' | sed "s/^-I//g") - PYTHON_HEADERS=$(echo "${PYBIND11_INCLUDES}" | awk '{print $2}' | sed "s/^-I//g") - ${CPP} -O2 -O3 -shared -std=c++11 -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 \ + c++ -O2 -O3 -shared -std=c++11 -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 \ -Wno-maybe-uninitialized -Wno-unused-parameter -Wall -Wl,-z,relro,-z,now,-z,noexecstack \ - -I"${THIRD_PARTY_DIR}" -I"${DATAVISUAL_DIR}/utils" -I"${PYTHON_INCLUDE}" -I"${PYTHON_HEADERS}" \ - -o "${CRC32_SO_FILE}" crc32.cc "$BUILDDIR/libsecurec.a" + -I"$THIRD_PARTY_DIR" -I"$DATAVISUAL_DIR/utils" -I"$PYTHON_INCLUDE" -I"$PYTHON_HEADERS" \ + -o "$CRC32_SO_FILE" crc32.cc "$BUILDDIR/libsecurec.a" - if [ ! -f "${CRC32_SO_FILE}" ]; then - echo "crc so file does not exist, build failed" + if [ ! -f "$CRC32_SO_FILE" ]; then + echo "$CRC32_SO_FILE file does not exist, build failed" exit 1 fi - mv "${CRC32_SO_FILE}" "${CRC32_OUTPUT_DIR}" + + mv "$CRC32_SO_FILE" "$CRC32_OUTPUT_DIR" } -cd "${SCRIPT_BASEDIR}" || exit build_securec -cd "${SCRIPT_BASEDIR}" || exit build_crc32 clean_securec diff --git a/build/scripts/ui.sh b/build/scripts/ui.sh index 6b597e2..f265b4d 100755 --- a/build/scripts/ui.sh +++ b/build/scripts/ui.sh @@ -13,29 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -SCRIPT_BASEDIR=$( - cd "$(dirname "$0")" || exit - pwd -) +set -e + +SCRIPT_BASEDIR=$(realpath "$(dirname "$0")") build_ui() { - NPM=$(command -v npm) - if [ -z "${NPM}" ]; then - echo "Could not find npm command" - exit 1 - fi + cd "$(realpath "$SCRIPT_BASEDIR/../../mindinsight/ui")" || exit - UI_SOURCE_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/ui") + if ! command -v npm; then + command npm + fi - cd "${UI_SOURCE_DIR}" || exit rm -rf dist - ${NPM} config set strict-ssl false - ${NPM} config set unsafe-perm true - ${NPM} config set user 0 + npm config set strict-ssl false + npm config set unsafe-perm true + npm config set user 0 - ${NPM} install - ${NPM} run build + npm install + npm run build if [ ! -f "dist/index.html" ]; then echo "dist does not have file index.html, build failed" @@ -45,5 +41,4 @@ build_ui() { rm -rf node_modules } -cd "${SCRIPT_BASEDIR}" || exit build_ui diff --git a/tests/st/runtest.sh b/tests/st/runtest.sh index 91d774c..2f61d69 100644 --- a/tests/st/runtest.sh +++ b/tests/st/runtest.sh @@ -14,16 +14,13 @@ # limitations under the License. set -e -shopt -s nullglob - -SCRIPT_BASEDIR=$( - cd "$(dirname "$0")" || exit - pwd -) -PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") -CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh" -CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/" -ST_PATH="${PROJECT_DIR}/tests/st" + +SCRIPT_BASEDIR=$(realpath "$(dirname "$0")") + +PROJECT_DIR=$(realpath "$SCRIPT_BASEDIR/../../") +CRC32_SCRIPT_PATH="$PROJECT_DIR/build/scripts/crc32.sh" +CRC32_OUTPUT_DIR="$PROJECT_DIR/mindinsight/datavisual/utils/" +ST_PATH="$PROJECT_DIR/tests/st" IS_BUILD_CRC="" PYTEST_MARK="" @@ -60,7 +57,7 @@ check_opts() { build_crc32() { echo "Start to check crc32." if [ -d "$CRC32_OUTPUT_DIR" ]; then - cd "$CRC32_OUTPUT_DIR" + cd "$CRC32_OUTPUT_DIR" || exit result=$(find . -maxdepth 1 -name "crc32*.so") if [ -z "$result" ]; then echo "Start to build crc32." @@ -74,7 +71,7 @@ build_crc32() { clean_crc32() { echo "Start to clean crc32." if [ -n "$IS_BUILD_CRC" ]; then - rm "$CRC32_OUTPUT_DIR"/crc32*.so -f + rm -f "$CRC32_OUTPUT_DIR"/crc32*.so fi } @@ -93,7 +90,7 @@ after_run_test() { run_test() { echo "Start to run test." - cd "$PROJECT_DIR" + cd "$PROJECT_DIR" || exit for dir in "$ST_PATH"/*; do if [ ! -d "$dir" ] || [ "$dir" = "$ST_PATH/__pycache__" ]; then diff --git a/tests/ut/runtest.sh b/tests/ut/runtest.sh index 1df2943..8708e53 100644 --- a/tests/ut/runtest.sh +++ b/tests/ut/runtest.sh @@ -14,16 +14,13 @@ # limitations under the License. set -e -shopt -s nullglob -SCRIPT_BASEDIR=$( - cd "$(dirname "$0")" || exit - pwd -) -PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") -CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh" -CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/" -UT_PATH="${PROJECT_DIR}/tests/ut" +SCRIPT_BASEDIR=$(realpath "$(dirname "$0")") + +PROJECT_DIR=$(realpath "$SCRIPT_BASEDIR/../../") +CRC32_SCRIPT_PATH="$PROJECT_DIR/build/scripts/crc32.sh" +CRC32_OUTPUT_DIR="$PROJECT_DIR/mindinsight/datavisual/utils/" +UT_PATH="$PROJECT_DIR/tests/ut" IS_BUILD_CRC="" build_crc32() { @@ -43,7 +40,7 @@ build_crc32() { clean_crc32() { echo "Start to clean crc32." if [ -n "$IS_BUILD_CRC" ]; then - rm "$CRC32_OUTPUT_DIR"/crc32*.so -f + rm -f "$CRC32_OUTPUT_DIR"/crc32*.so fi } @@ -62,7 +59,7 @@ after_run_test() { run_test() { echo "Start to run test." - cd "$PROJECT_DIR" + cd "$PROJECT_DIR" || exit pytest "$UT_PATH" -- GitLab