From 161d27dc1efcc05eb5cc69090c161faf167b5db9 Mon Sep 17 00:00:00 2001 From: Sing_chan <51314274+betterpig@users.noreply.github.com> Date: Fri, 18 Mar 2022 17:10:53 +0800 Subject: [PATCH] set +x to close showing command, update check_change code with linux (#40456) --- paddle/fluid/inference/api/demo_ci/run.sh | 6 +- paddle/scripts/paddle_build.bat | 133 +++++----------------- tools/check_added_ut.sh | 7 +- tools/windows/check_change_of_unittest.sh | 41 +++++++ 4 files changed, 77 insertions(+), 110 deletions(-) create mode 100644 tools/windows/check_change_of_unittest.sh diff --git a/paddle/fluid/inference/api/demo_ci/run.sh b/paddle/fluid/inference/api/demo_ci/run.sh index 79a31555c7f..2c0945cd5b3 100755 --- a/paddle/fluid/inference/api/demo_ci/run.sh +++ b/paddle/fluid/inference/api/demo_ci/run.sh @@ -53,7 +53,11 @@ if [ $7 == ON ]; then if [[ -e "MobileNetV2.inference.model.tar.gz" ]]; then echo "MobileNetV2.inference.model.tar.gz has been downloaded." else - wget -q --no-proxy http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz + if [ $WIN_DETECT != "" ]; then + wget -q -Y off http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz + else + wget -q --no-proxy http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz + fi tar xzf *.tar.gz fi cd .. diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 75afa4ef43f..78a863040ad 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -55,7 +55,6 @@ wmic process where name="python.exe" call terminate 2>NUL rem ------initialize common variable------ if not defined GENERATOR set GENERATOR="Visual Studio 15 2017 Win64" -if not defined BRANCH set BRANCH=develop if not defined WITH_TENSORRT set WITH_TENSORRT=ON if not defined TENSORRT_ROOT set TENSORRT_ROOT=D:/TensorRT if not defined CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto @@ -70,7 +69,6 @@ if not defined WITH_ONNXRUNTIME set WITH_ONNXRUNTIME=OFF if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON if not defined WITH_STATIC_LIB set WITH_STATIC_LIB=ON if not defined WITH_TPCACHE set WITH_TPCACHE=OFF -if not defined WITH_CLCACHE set WITH_CLCACHE=OFF if not defined WITH_CACHE set WITH_CACHE=OFF if not defined WITH_SCCACHE set WITH_SCCACHE=OFF if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF @@ -145,17 +143,6 @@ if %day_now% NEQ %day_before% ( echo %day_now% > %cache_dir%\day.txt type %cache_dir%\day.txt rmdir %BUILD_DIR% /s/q - - : clear third party cache every once in a while - if %day_now% EQU 21 ( - rmdir %cache_dir%\third_party /s/q - ) - if %day_now% EQU 11 ( - rmdir %cache_dir%\third_party /s/q - ) - if %day_now% EQU 01 ( - rmdir %cache_dir%\third_party /s/q - ) goto :mkbuild ) @@ -212,6 +199,7 @@ echo There is not sccache in this PC, will install sccache. echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe')" xcopy sccache.exe %PYTHON_ROOT%\ /Y +del sccache.exe goto:eof rem -------Caching strategy 2: End -------------------------------- @@ -232,13 +220,12 @@ set WITH_AVX=ON set MSVC_STATIC_CRT=OFF set ON_INFER=OFF set WITH_TENSORRT=ON +set WITH_INFERENCE_API_TEST=OFF call :cmake || goto cmake_error call :build || goto build_error call :test_whl_pacakage || goto test_whl_pacakage_error call :test_unit || goto test_unit_error -:: call :test_inference || goto test_inference_error -:: call :check_change_of_unittest || goto check_change_of_unittest_error goto:success rem ------PR CI windows check for OPENBLAS/CPU------ @@ -254,8 +241,6 @@ call :cmake || goto cmake_error call :build || goto build_error call :test_whl_pacakage || goto test_whl_pacakage_error call :test_unit || goto test_unit_error -:: call :test_inference || goto test_inference_error -:: call :check_change_of_unittest || goto check_change_of_unittest_error goto:success rem ------PR CI windows check for unittests and inference in CUDA11-MKL-AVX---------- @@ -265,7 +250,6 @@ set WITH_GPU=ON set WITH_AVX=ON set MSVC_STATIC_CRT=ON set ON_INFER=ON -set WITH_TESTING=ON set WITH_TENSORRT=ON set WITH_INFERENCE_API_TEST=ON @@ -274,7 +258,8 @@ call :build || goto build_error call :test_whl_pacakage || goto test_whl_pacakage_error call :test_unit || goto test_unit_error ::call :test_inference || goto test_inference_error -:: call :check_change_of_unittest || goto check_change_of_unittest_error +::call :test_inference_ut || goto test_inference_ut_error +call :check_change_of_unittest || goto check_change_of_unittest_error goto:success rem ------Build windows avx whl package------ @@ -365,18 +350,6 @@ if "%WITH_GPU%"=="ON" ( nvidia-smi 2>NUL ) -rem ------pre install clcache and init config---------- -rem pip install clcache --user -pip uninstall -y clcache -:: set USE_CLCACHE to enable clcache -rem set USE_CLCACHE=1 -:: In some scenarios, CLCACHE_HARDLINK can save one file copy. -rem set CLCACHE_HARDLINK=1 -:: If it takes more than 1000s to obtain the right to use the cache, an error will be reported -rem set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=1000000 -:: set maximum cache size to 20G -rem clcache.exe -M 21474836480 - rem ------set third_party cache dir------ if "%WITH_TPCACHE%"=="OFF" ( @@ -384,6 +357,25 @@ if "%WITH_TPCACHE%"=="OFF" ( goto :cmake_impl ) +rem clear third party cache every ten days +for /F %%# in ('wmic os get localdatetime^|findstr 20') do set datetime=%%# +set day_now=%datetime:~6,2% +set day_before=-1 +set /p day_before=< %cache_dir%\day_third_party.txt +if %day_now% NEQ %day_before% ( + echo %day_now% > %cache_dir%\day_third_party.txt + type %cache_dir%\day_third_party.txt + if %day_now% EQU 21 ( + rmdir %cache_dir%\third_party /s/q + ) + if %day_now% EQU 11 ( + rmdir %cache_dir%\third_party /s/q + ) + if %day_now% EQU 01 ( + rmdir %cache_dir%\third_party /s/q + ) +) + echo set -ex > cache.sh echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake ^|md5sum ^| awk '{print $1}') >> cache.sh echo echo ${md5_content}^>md5.txt >> cache.sh @@ -535,11 +527,7 @@ echo Build Paddle the %build_times% time: if %GENERATOR% == "Ninja" ( ninja all ) else ( - if "%WITH_CLCACHE%"=="OFF" ( - MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj - ) else ( - MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:CLToolExe=clcache.exe /p:CLToolPath=%PYTHON_ROOT%\Scripts /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj - ) + MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj ) if %ERRORLEVEL% NEQ 0 ( @@ -774,77 +762,8 @@ echo ======================================== echo Step 6. Check whether deleting a unit test ... echo ======================================== -cd /d %work_dir%\%BUILD_DIR% -echo set -e> check_change_of_unittest.sh -echo set +x>> check_change_of_unittest.sh -echo GITHUB_API_TOKEN=%GITHUB_API_TOKEN% >> check_change_of_unittest.sh -echo GIT_PR_ID=%AGILE_PULL_ID% >> check_change_of_unittest.sh -echo BRANCH=%BRANCH%>> check_change_of_unittest.sh -echo if [ "${GITHUB_API_TOKEN}" == "" ] ^|^| [ "${GIT_PR_ID}" == "" ];then>> check_change_of_unittest.sh -echo exit 0 >> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo set -x>> check_change_of_unittest.sh -echo cat ^<^> check_change_of_unittest.sh -echo ============================================ >> check_change_of_unittest.sh -echo Generate unit tests.spec of this PR. >> check_change_of_unittest.sh -echo ============================================ >> check_change_of_unittest.sh -echo EOF>> check_change_of_unittest.sh -echo spec_path=$(pwd)/UNITTEST_PR.spec>> check_change_of_unittest.sh -echo ctest -N ^| awk -F ':' '{print $2}' ^| sed '/^^$/d' ^| sed '$d' ^> ${spec_path}>> check_change_of_unittest.sh -echo num=$(awk 'END{print NR}' ${spec_path})>> check_change_of_unittest.sh -echo echo "Windows 1 card TestCases count is $num">> check_change_of_unittest.sh -echo echo ipipe_log_param_Windows_1_Card_TestCases_Count: $num>> check_change_of_unittest.sh -echo UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle'>> check_change_of_unittest.sh -echo origin_upstream_url=`git remote -v ^| awk '{print $1, $2}' ^| uniq ^| grep upstream ^| awk '{print $2}'`>> check_change_of_unittest.sh -echo if [ "$origin_upstream_url" == "" ]; then>> check_change_of_unittest.sh -echo git remote add upstream $UPSTREAM_URL.git>> check_change_of_unittest.sh -echo elif [ "$origin_upstream_url" ^!= "$UPSTREAM_URL" ] ^\>> check_change_of_unittest.sh -echo ^&^& [ "$origin_upstream_url" ^!= "$UPSTREAM_URL.git" ]; then>> check_change_of_unittest.sh -echo git remote remove upstream>> check_change_of_unittest.sh -echo git remote add upstream $UPSTREAM_URL.git>> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo if [ ! -e "$(pwd)/../.git/refs/remotes/upstream/$BRANCH" ]; then>> check_change_of_unittest.sh -echo git fetch upstream $BRANCH # develop is not fetched>> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo git checkout -b origin_pr >> check_change_of_unittest.sh -echo git checkout -f $BRANCH >> check_change_of_unittest.sh -echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ --DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DON_INFER=%ON_INFER% ^ --DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ --DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^ --DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^ --DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% >> check_change_of_unittest.sh -echo cat ^<^> check_change_of_unittest.sh -echo ============================================ >> check_change_of_unittest.sh -echo Generate unit tests.spec of develop. >> check_change_of_unittest.sh -echo ============================================ >> check_change_of_unittest.sh -echo EOF>> check_change_of_unittest.sh -echo spec_path=$(pwd)/UNITTEST_DEV.spec>> check_change_of_unittest.sh -echo ctest -N ^| awk -F ':' '{print $2}' ^| sed '/^^$/d' ^| sed '$d' ^> ${spec_path}>> check_change_of_unittest.sh -echo unittest_spec_diff=`python $(pwd)/../tools/diff_unittest.py $(pwd)/UNITTEST_DEV.spec $(pwd)/UNITTEST_PR.spec`>> check_change_of_unittest.sh -echo if [ "$unittest_spec_diff" ^!= "" ]; then>> check_change_of_unittest.sh -echo set +x>> check_change_of_unittest.sh -echo approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`>> check_change_of_unittest.sh -echo set -x>> check_change_of_unittest.sh -echo if [ "$approval_line" ^!= "" ]; then>> check_change_of_unittest.sh -echo APPROVALS=`echo ${approval_line} ^|python $(pwd)/../tools/check_pr_approval.py 1 22165420 52485244 6836917`>> check_change_of_unittest.sh -echo echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}">> check_change_of_unittest.sh -echo if [ "${APPROVALS}" == "FALSE" ]; then>> check_change_of_unittest.sh -echo echo "************************************" >> check_change_of_unittest.sh -echo echo -e "It is forbidden to disable or delete the unit-test.\n" >> check_change_of_unittest.sh -echo echo -e "If you must delete it temporarily, please add it to[https://github.com/PaddlePaddle/Paddle/wiki/Temporarily-disabled-Unit-Test]." >> check_change_of_unittest.sh -echo echo -e "Then you must have one RD (kolinwei(recommended) or zhouwei25) approval for the deletion of unit-test. \n" >> check_change_of_unittest.sh -echo echo -e "If you have any problems about deleting unit-test, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/Deleting-unit-test-is-forbidden]. \n" >> check_change_of_unittest.sh -echo echo -e "Following unit-tests are deleted in this PR: \n ${unittest_spec_diff} \n" >> check_change_of_unittest.sh -echo echo "************************************" >> check_change_of_unittest.sh -echo exit 1 >> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo else>> check_change_of_unittest.sh -echo exit 1 >> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo fi>> check_change_of_unittest.sh -echo git checkout -f origin_pr >> check_change_of_unittest.sh -%cache_dir%\tools\busybox64.exe bash check_change_of_unittest.sh +%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\check_change_of_unittest.sh + goto:eof :check_change_of_unittest_error diff --git a/tools/check_added_ut.sh b/tools/check_added_ut.sh index 2a9fb842862..5466a1cdd59 100644 --- a/tools/check_added_ut.sh +++ b/tools/check_added_ut.sh @@ -52,9 +52,10 @@ if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then elif [[ "$SYSTEM" == "Windows_NT" ]];then bash $PADDLE_ROOT/win_cmake.sh >prec_build.log 2>&1 fi -ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/br-ut +# remove line ended with .exe to get correct deleted_ut list +ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | sed '/\.exe$/d' | grep 'test' > $PADDLE_ROOT/br-ut cd $PADDLE_ROOT/build -ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/pr-ut +ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | sed '/\.exe$/d' | grep 'test' > $PADDLE_ROOT/pr-ut cd $PADDLE_ROOT grep -F -x -v -f br-ut pr-ut > $PADDLE_ROOT/added_ut if [[ "$SYSTEM" == 'Linux' ]];then @@ -66,6 +67,8 @@ rm -rf prec_build if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh elif [[ "$SYSTEM" == "Windows_NT" ]];then + # get the deleted ut list in windows, will be used in check_change_of_unittest.sh + grep -F -x -v -f pr-ut br-ut > $PADDLE_ROOT/deleted_ut rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/win_cmake.sh fi git checkout -f $CURBRANCH diff --git a/tools/windows/check_change_of_unittest.sh b/tools/windows/check_change_of_unittest.sh new file mode 100644 index 00000000000..576f0e5d238 --- /dev/null +++ b/tools/windows/check_change_of_unittest.sh @@ -0,0 +1,41 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set +x +export PADDLE_ROOT="$(cd "$PWD/../" && pwd )" +GITHUB_API_TOKEN=$GITHUB_API_TOKEN +GIT_PR_ID=$AGILE_PULL_ID +BRANCH=$BRANCH +if [ "${GITHUB_API_TOKEN}" == "" ] || [ "${GIT_PR_ID}" == "" ];then + exit 0 +fi + +unittest_spec_diff=$(cat $PADDLE_ROOT/deleted_ut | sed 's/^/ - /g') +if [ "$unittest_spec_diff" != "" ]; then + approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000` + APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 22165420 52485244 32428676 45041955` + echo "current pr ${GIT_PR_ID} got approvals: ${APPROVALS}" + if [ "${APPROVALS}" == "FALSE" ]; then + echo "************************************" + echo -e "It is forbidden to disable or delete the unit-test.\n" + echo -e "If you must delete it temporarily, please add it to[https://github.com/PaddlePaddle/Paddle/wiki/Temporarily-disabled-Unit-Test]." + echo -e "Then you must have one RD (kolinwei(recommended), chalsliu, XieYunshen or zhouwei25) approval for the deletion of unit-test. \n" + echo -e "If you have any problems about deleting unit-test, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/Deleting-unit-test-is-forbidden]. \n" + echo -e "Following unit-tests are deleted in this PR: \n${unittest_spec_diff} \n" + echo "************************************" + exit 6 + fi +fi +set -x -- GitLab