“f94bfbaef02b7767c668192d73411838e87122fe”上不存在“tests/pytest/stream/sys.py”
提交 6119eefa 编写于 作者: 李鸿章

secure the scripts with set -e

上级 432a0acc
...@@ -13,32 +13,31 @@ ...@@ -13,32 +13,31 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
SCRIPT_BASEDIR=$( set -e
cd "$(dirname "$0")" || exit
pwd SCRIPT_BASEDIR=$(realpath "$(dirname "$0")")
)
PROJECT_BASEDIR=$(dirname "$SCRIPT_BASEDIR")
rename_wheel() { rename_wheel() {
cd "$PROJECT_BASEDIR/output" || exit
VERSION="$1" VERSION="$1"
PACKAGE_LIST=$(ls mindinsight-*-any.whl) || exit PACKAGE_LIST=$(ls mindinsight-*-any.whl) || exit
for PACKAGE_ORIG in ${PACKAGE_LIST}; do for PACKAGE_ORIG in $PACKAGE_LIST; do
MINDINSIGHT_VERSION=$(echo "${PACKAGE_ORIG}" | awk -F"-" '{print $2}') MINDINSIGHT_VERSION=$(echo "$PACKAGE_ORIG" | awk -F"-" '{print $2}')
PYTHON_VERSION_NUM=$(echo "${VERSION}" | awk -F"." '{print $1$2}') PYTHON_VERSION_NUM=$(echo "$VERSION" | awk -F"." '{print $1$2}')
PYTHON_VERSION_TAG="cp${PYTHON_VERSION_NUM}" PYTHON_VERSION_TAG="cp$PYTHON_VERSION_NUM"
PYTHON_ABI_TAG="cp${PYTHON_VERSION_NUM}m" PYTHON_ABI_TAG="cp${PYTHON_VERSION_NUM}m"
OS_NAME=$(uname | tr '[:upper:]' '[:lower:]') OS_NAME=$(uname | tr '[:upper:]' '[:lower:]')
MACHINE_TAG="${OS_NAME}_$(uname -i)" MACHINE_TAG="${OS_NAME}_$(uname -i)"
PACKAGE_NEW="mindinsight-${MINDINSIGHT_VERSION}-${PYTHON_VERSION_TAG}-${PYTHON_ABI_TAG}-${MACHINE_TAG}.whl" PACKAGE_NEW="mindinsight-$MINDINSIGHT_VERSION-$PYTHON_VERSION_TAG-$PYTHON_ABI_TAG-$MACHINE_TAG.whl"
mv "${PACKAGE_ORIG}" "${PACKAGE_NEW}" mv "$PACKAGE_ORIG" "$PACKAGE_NEW"
done done
} }
build_wheel() { build_wheel() {
PROJECT_BASEDIR=$(
cd "$(dirname "$SCRIPT_BASEDIR")" || exit cd "$PROJECT_BASEDIR" || exit
pwd
)
cd "${PROJECT_BASEDIR}" || exit
if [ $# -gt 0 ]; then if [ $# -gt 0 ]; then
if [ "$1" = "clean" ]; then if [ "$1" = "clean" ]; then
...@@ -54,43 +53,41 @@ build_wheel() { ...@@ -54,43 +53,41 @@ build_wheel() {
echo "start building mindinsight" echo "start building mindinsight"
clean_files clean_files
PYTHON=$(command -v python3 || command -v python) if command -v python3; then
if [ -z "${PYTHON}" ]; then PYTHON=python3
echo "Could not find python3 or python command" elif command -v python; then
exit 1 PYTHON=python
else
command python3
fi fi
PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*')
if [ -z "${PYTHON_VERSION}" ]; then if ! "$PYTHON" -c 'import sys; assert sys.version_info >= (3, 7)' &>/dev/null; then
echo "Could not find Python 3" echo "Python 3.7 or higher is required. You are running $("$PYTHON" -V)"
exit 1 exit 1
fi fi
rm -f output rm -rf output
mkdir output
${PYTHON} setup.py bdist_wheel "$PYTHON" setup.py bdist_wheel
if [ ! -x "dist" ]; then if [ ! -x "dist" ]; then
echo "Build failed" echo "Build failed"
exit 1 exit 1
fi fi
mv dist/mindinsight-*-any.whl output/ mv dist output
cd output || exit rename_wheel "$("$PYTHON" -c 'import platform; print(platform.python_version())')"
rename_wheel "${PYTHON_VERSION}"
cd - >/dev/null 2>&1 || exit
clean_files clean_files
echo "Build success, output directory is: ${PROJECT_BASEDIR}/output" echo "Build success, output directory is: $PROJECT_BASEDIR/output"
} }
clean_files() { clean_files() {
rm -rf third_party/build cd "$PROJECT_BASEDIR" || exit
rm -rf build/lib rm -rf build/lib
rm -rf build/bdist.* rm -rf build/bdist.*
rm -rf mindinsight.egg-info rm -rf mindinsight.egg-info
rm -rf dist
} }
show_usage() { show_usage() {
...@@ -120,5 +117,4 @@ check_opts() { ...@@ -120,5 +117,4 @@ check_opts() {
check_opts "$@" check_opts "$@"
cd "${SCRIPT_BASEDIR}" || exit
build_wheel "$@" build_wheel "$@"
...@@ -13,85 +13,75 @@ ...@@ -13,85 +13,75 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
SCRIPT_BASEDIR=$( set -e
cd "$(dirname "$0")" || exit
pwd
)
THIRD_PARTY_DIR=$(realpath "${SCRIPT_BASEDIR}/../../third_party") SCRIPT_BASEDIR=$(realpath "$(dirname "$0")")
BUILDDIR=$(dirname "$SCRIPT_BASEDIR")
THIRD_PARTY_DIR=$(realpath "$SCRIPT_BASEDIR/../../third_party")
BUILDDIR="$(dirname "$SCRIPT_BASEDIR")/build_securec"
build_securec() { build_securec() {
CMAKE=$(command -v cmake) rm -rf "$BUILDDIR"
if [ -z "${CMAKE}" ]; then mkdir "$BUILDDIR"
echo "Could not find cmake command"
exit 1
fi
cd "$BUILDDIR" || exit cd "$BUILDDIR" || exit
${CMAKE} .. if ! command -v cmake; then
command cmake
fi
cmake ../..
make make
} }
clean_securec() { clean_securec() {
cd "$BUILDDIR" || exit rm -rf "$BUILDDIR"
for file in *; do
if [ "$file" == build.sh ] || [ "$file" == scripts ] || [ "$file" == lib ]; then
continue
fi
rm -rf "$file"
done
} }
build_crc32() { build_crc32() {
CPP=$(command -v c++) DATAVISUAL_DIR=$(realpath "$SCRIPT_BASEDIR/../../mindinsight/datavisual")
if [ -z "${CPP}" ]; then CRC32_SOURCE_DIR="$DATAVISUAL_DIR/utils/crc32"
echo "Could not find c++ command" CRC32_OUTPUT_DIR="$DATAVISUAL_DIR/utils"
exit 1 CRC32_SO_FILE="crc32$(python3-config --extension-suffix)"
cd "$CRC32_SOURCE_DIR" || exit
if ! command -v c++; then
command c++
fi fi
PYTHON=$(command -v python3 || command -v python) if command -v python3; then
if [ -z "${PYTHON}" ]; then PYTHON=python3
echo "Could not find python3 or python command" elif command -v python; then
exit 1 PYTHON=python
else
command python3
fi fi
PYTHON_VERSION=$(${PYTHON} -c "import platform; print(platform.python_version())" | grep '^3.*')
if [ -z "${PYTHON_VERSION}" ]; then if ! "$PYTHON" -c 'import sys; assert sys.version_info >= (3, 7)' &>/dev/null; then
echo "Could not find Python 3" echo "Python 3.7 or higher is required. You are running $("$PYTHON" -V)"
exit 1 exit 1
fi fi
DATAVISUAL_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/datavisual") rm -f "$CRC32_SOURCE_DIR/$CRC32_SO_FILE"
CRC32_SOURCE_DIR="${DATAVISUAL_DIR}/utils/crc32" rm -f "$CRC32_OUTPUT_DIR/$CRC32_SO_FILE"
CRC32_OUTPUT_DIR="${DATAVISUAL_DIR}/utils"
CRC32_SO_FILE="crc32$(python3-config --extension-suffix)"
rm -f "${CRC32_SOURCE_DIR}/${CRC32_SO_FILE}" PYBIND11_INCLUDES=$($PYTHON -m pybind11 --includes)
rm -f "${CRC32_OUTPUT_DIR}/${CRC32_SO_FILE}" PYTHON_INCLUDE=$(echo "$PYBIND11_INCLUDES" | awk '{print $1}' | sed "s/^-I//g")
cd "${CRC32_SOURCE_DIR}" || exit PYTHON_HEADERS=$(echo "$PYBIND11_INCLUDES" | awk '{print $2}' | sed "s/^-I//g")
PYBIND11_INCLUDES=$(${PYTHON} -m pybind11 --includes)
if [ -z "${PYBIND11_INCLUDES}" ]; then
echo "Could not find pybind11 module"
exit 1
fi
PYTHON_INCLUDE=$(echo "${PYBIND11_INCLUDES}" | awk '{print $1}' | sed "s/^-I//g") c++ -O2 -O3 -shared -std=c++11 -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 \
PYTHON_HEADERS=$(echo "${PYBIND11_INCLUDES}" | awk '{print $2}' | sed "s/^-I//g")
${CPP} -O2 -O3 -shared -std=c++11 -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 \
-Wno-maybe-uninitialized -Wno-unused-parameter -Wall -Wl,-z,relro,-z,now,-z,noexecstack \ -Wno-maybe-uninitialized -Wno-unused-parameter -Wall -Wl,-z,relro,-z,now,-z,noexecstack \
-I"${THIRD_PARTY_DIR}" -I"${DATAVISUAL_DIR}/utils" -I"${PYTHON_INCLUDE}" -I"${PYTHON_HEADERS}" \ -I"$THIRD_PARTY_DIR" -I"$DATAVISUAL_DIR/utils" -I"$PYTHON_INCLUDE" -I"$PYTHON_HEADERS" \
-o "${CRC32_SO_FILE}" crc32.cc "$BUILDDIR/libsecurec.a" -o "$CRC32_SO_FILE" crc32.cc "$BUILDDIR/libsecurec.a"
if [ ! -f "${CRC32_SO_FILE}" ]; then if [ ! -f "$CRC32_SO_FILE" ]; then
echo "crc so file does not exist, build failed" echo "$CRC32_SO_FILE file does not exist, build failed"
exit 1 exit 1
fi fi
mv "${CRC32_SO_FILE}" "${CRC32_OUTPUT_DIR}"
mv "$CRC32_SO_FILE" "$CRC32_OUTPUT_DIR"
} }
cd "${SCRIPT_BASEDIR}" || exit
build_securec build_securec
cd "${SCRIPT_BASEDIR}" || exit
build_crc32 build_crc32
clean_securec clean_securec
...@@ -13,29 +13,25 @@ ...@@ -13,29 +13,25 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
SCRIPT_BASEDIR=$( set -e
cd "$(dirname "$0")" || exit
pwd SCRIPT_BASEDIR=$(realpath "$(dirname "$0")")
)
build_ui() { build_ui() {
NPM=$(command -v npm) cd "$(realpath "$SCRIPT_BASEDIR/../../mindinsight/ui")" || exit
if [ -z "${NPM}" ]; then
echo "Could not find npm command"
exit 1
fi
UI_SOURCE_DIR=$(realpath "${SCRIPT_BASEDIR}/../../mindinsight/ui") if ! command -v npm; then
command npm
fi
cd "${UI_SOURCE_DIR}" || exit
rm -rf dist rm -rf dist
${NPM} config set strict-ssl false npm config set strict-ssl false
${NPM} config set unsafe-perm true npm config set unsafe-perm true
${NPM} config set user 0 npm config set user 0
${NPM} install npm install
${NPM} run build npm run build
if [ ! -f "dist/index.html" ]; then if [ ! -f "dist/index.html" ]; then
echo "dist does not have file index.html, build failed" echo "dist does not have file index.html, build failed"
...@@ -45,5 +41,4 @@ build_ui() { ...@@ -45,5 +41,4 @@ build_ui() {
rm -rf node_modules rm -rf node_modules
} }
cd "${SCRIPT_BASEDIR}" || exit
build_ui build_ui
...@@ -14,16 +14,13 @@ ...@@ -14,16 +14,13 @@
# limitations under the License. # limitations under the License.
set -e set -e
shopt -s nullglob
SCRIPT_BASEDIR=$(realpath "$(dirname "$0")")
SCRIPT_BASEDIR=$(
cd "$(dirname "$0")" || exit PROJECT_DIR=$(realpath "$SCRIPT_BASEDIR/../../")
pwd CRC32_SCRIPT_PATH="$PROJECT_DIR/build/scripts/crc32.sh"
) CRC32_OUTPUT_DIR="$PROJECT_DIR/mindinsight/datavisual/utils/"
PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") ST_PATH="$PROJECT_DIR/tests/st"
CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh"
CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/"
ST_PATH="${PROJECT_DIR}/tests/st"
IS_BUILD_CRC="" IS_BUILD_CRC=""
PYTEST_MARK="" PYTEST_MARK=""
...@@ -60,7 +57,7 @@ check_opts() { ...@@ -60,7 +57,7 @@ check_opts() {
build_crc32() { build_crc32() {
echo "Start to check crc32." echo "Start to check crc32."
if [ -d "$CRC32_OUTPUT_DIR" ]; then if [ -d "$CRC32_OUTPUT_DIR" ]; then
cd "$CRC32_OUTPUT_DIR" cd "$CRC32_OUTPUT_DIR" || exit
result=$(find . -maxdepth 1 -name "crc32*.so") result=$(find . -maxdepth 1 -name "crc32*.so")
if [ -z "$result" ]; then if [ -z "$result" ]; then
echo "Start to build crc32." echo "Start to build crc32."
...@@ -74,7 +71,7 @@ build_crc32() { ...@@ -74,7 +71,7 @@ build_crc32() {
clean_crc32() { clean_crc32() {
echo "Start to clean crc32." echo "Start to clean crc32."
if [ -n "$IS_BUILD_CRC" ]; then if [ -n "$IS_BUILD_CRC" ]; then
rm "$CRC32_OUTPUT_DIR"/crc32*.so -f rm -f "$CRC32_OUTPUT_DIR"/crc32*.so
fi fi
} }
...@@ -93,7 +90,7 @@ after_run_test() { ...@@ -93,7 +90,7 @@ after_run_test() {
run_test() { run_test() {
echo "Start to run test." echo "Start to run test."
cd "$PROJECT_DIR" cd "$PROJECT_DIR" || exit
for dir in "$ST_PATH"/*; do for dir in "$ST_PATH"/*; do
if [ ! -d "$dir" ] || [ "$dir" = "$ST_PATH/__pycache__" ]; then if [ ! -d "$dir" ] || [ "$dir" = "$ST_PATH/__pycache__" ]; then
......
...@@ -14,16 +14,13 @@ ...@@ -14,16 +14,13 @@
# limitations under the License. # limitations under the License.
set -e set -e
shopt -s nullglob
SCRIPT_BASEDIR=$( SCRIPT_BASEDIR=$(realpath "$(dirname "$0")")
cd "$(dirname "$0")" || exit
pwd PROJECT_DIR=$(realpath "$SCRIPT_BASEDIR/../../")
) CRC32_SCRIPT_PATH="$PROJECT_DIR/build/scripts/crc32.sh"
PROJECT_DIR=$(realpath "${SCRIPT_BASEDIR}/../../") CRC32_OUTPUT_DIR="$PROJECT_DIR/mindinsight/datavisual/utils/"
CRC32_SCRIPT_PATH="${PROJECT_DIR}/build/scripts/crc32.sh" UT_PATH="$PROJECT_DIR/tests/ut"
CRC32_OUTPUT_DIR="${PROJECT_DIR}/mindinsight/datavisual/utils/"
UT_PATH="${PROJECT_DIR}/tests/ut"
IS_BUILD_CRC="" IS_BUILD_CRC=""
build_crc32() { build_crc32() {
...@@ -43,7 +40,7 @@ build_crc32() { ...@@ -43,7 +40,7 @@ build_crc32() {
clean_crc32() { clean_crc32() {
echo "Start to clean crc32." echo "Start to clean crc32."
if [ -n "$IS_BUILD_CRC" ]; then if [ -n "$IS_BUILD_CRC" ]; then
rm "$CRC32_OUTPUT_DIR"/crc32*.so -f rm -f "$CRC32_OUTPUT_DIR"/crc32*.so
fi fi
} }
...@@ -62,7 +59,7 @@ after_run_test() { ...@@ -62,7 +59,7 @@ after_run_test() {
run_test() { run_test() {
echo "Start to run test." echo "Start to run test."
cd "$PROJECT_DIR" cd "$PROJECT_DIR" || exit
pytest "$UT_PATH" pytest "$UT_PATH"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册