未验证 提交 5017aa76 编写于 作者: Z Zhou Wei 提交者: GitHub

set default python3,fix incompatible,cache dir for third party,unify error...

set default python3,fix incompatible,cache dir for third party,unify error code,for windows (#26178)

* set default python3 for paddle windows,test=win

* set default python3,cache dir for third party,error code,test=win

* fix some incompatible

* fix some error

* set virtual environment,test=win
上级 6d22f5c7
@ECHO OFF @ECHO ON
SETLOCAL SETLOCAL
rem Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. rem Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
...@@ -19,22 +19,66 @@ rem ================================================= ...@@ -19,22 +19,66 @@ rem =================================================
rem Paddle CI Task On Windows Platform rem Paddle CI Task On Windows Platform
rem ================================================= rem =================================================
set work_dir=%cd% set work_dir=%cd%
if exist build rmdir build /s/q
mkdir build
cd /d build
rem ------initialize the virtual environment------
if not defined PYTHON_ROOT set PYTHON_ROOT=c:\Python37
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
%PYTHON_EXECUTABLE% -m pip install virtualenv
if not exist paddle_winci (%PYTHON_EXECUTABLE% -m virtualenv paddle_winci)
call paddle_winci\Scripts\activate.bat
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
rem ------pre install requirement----------
where python
where pip
pip install --upgrade pip
pip install wheel
pip install gym
pip install -r %work_dir%\python\requirements.txt
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
rem ------initialize common variable------
if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0"
if not defined BRANCH set BRANCH=develop if not defined BRANCH set BRANCH=develop
if not defined PYTHON_ROOT set PYTHON_ROOT=c:\Python27
if not defined WITH_MKL set WITH_MKL=ON
if not defined WITH_AVX set WITH_AVX=ON
if not defined WITH_AVX set WITH_AVX=ON if not defined WITH_AVX set WITH_AVX=ON
if not defined WITH_GPU set WITH_GPU=OFF
if not defined WITH_TESTING set WITH_TESTING=ON if not defined WITH_TESTING set WITH_TESTING=ON
if not defined WITH_PYTHON set WITH_PYTHON=ON if not defined WITH_PYTHON set WITH_PYTHON=ON
if not defined ON_INFER set ON_INFER=ON if not defined ON_INFER set ON_INFER=ON
if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=OFF if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=OFF
if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=d:/.cache/inference_demo if not defined WITH_TPCACHE set WITH_TPCACHE=ON
if not defined THIRD_PARTY_PATH set THIRD_PARTY_PATH=%work_dir:\=/%/build/third_party
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe rem ------set cache third_party------
dir d:\.cache set cache_dir=%work_dir%\..\cache
dir %cache_dir%
set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo
if not exist %cache_dir%\tools (
git clone https://github.com/zhouwei25/tools %cache_dir%\tools
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
)
if "%WITH_TPCACHE%"=="OFF" (
set THIRD_PARTY_PATH=%work_dir:\=/%/build/third_party
goto :CASE_%1
)
echo set -ex > cache.sh
echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake ^|md5sum ^| awk '{print $1}') >> cache.sh
echo echo ${md5_content}^>md5.txt >> cache.sh
%cache_dir%\tools\busybox64.exe cat cache.sh
%cache_dir%\tools\busybox64.exe bash cache.sh
set /p md5=< md5.txt
if "%WITH_GPU%"=="ON" (
set THIRD_PARTY_PATH=%cache_dir:\=/%/third_party_GPU/%md5%
) else (
set THIRD_PARTY_PATH=%cache_dir:\=/%/third_party/%md5%
)
goto :CASE_%1 goto :CASE_%1
...@@ -45,6 +89,8 @@ echo "wincheck_openbals: run Windows OPENBLAS/CPU CI tasks on Windows" ...@@ -45,6 +89,8 @@ echo "wincheck_openbals: run Windows OPENBLAS/CPU CI tasks on Windows"
exit /b 1 exit /b 1
:CASE_wincheck_mkl :CASE_wincheck_mkl
set WITH_MKL=ON
set WITH_GPU=OFF
call :cmake || goto cmake_error call :cmake || goto cmake_error
call :build || goto build_error call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error call :test_whl_pacakage || goto test_whl_pacakage_error
...@@ -54,6 +100,8 @@ call :check_change_of_unittest || goto check_change_of_unittest_error ...@@ -54,6 +100,8 @@ call :check_change_of_unittest || goto check_change_of_unittest_error
goto:success goto:success
:CASE_wincheck_openblas :CASE_wincheck_openblas
set WITH_MKL=OFF
set WITH_GPU=ON
call :cmake || goto cmake_error call :cmake || goto cmake_error
call :build || goto build_error call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error call :test_whl_pacakage || goto test_whl_pacakage_error
...@@ -70,13 +118,14 @@ echo ======================================== ...@@ -70,13 +118,14 @@ echo ========================================
echo Step 1. Cmake ... echo Step 1. Cmake ...
echo ======================================== echo ========================================
mkdir build echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
cd /d build cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0" -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH%
goto:eof goto:eof
:cmake_error :cmake_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bat
echo cmake failed!
exit /b 1
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
:build :build
...@@ -115,7 +164,9 @@ if %ERRORLEVEL% NEQ 0 ( ...@@ -115,7 +164,9 @@ if %ERRORLEVEL% NEQ 0 (
goto:eof goto:eof
:build_error :build_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bat
echo build paddle failed!
exit /b 7
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
:test_whl_pacakage :test_whl_pacakage
...@@ -124,20 +175,23 @@ echo Step 3. Test pip install whl package ... ...@@ -124,20 +175,23 @@ echo Step 3. Test pip install whl package ...
echo ======================================== echo ========================================
dir /s /b python\dist\*.whl > whl_file.txt dir /s /b python\dist\*.whl > whl_file.txt
set /p PADDLE_WHL_FILE_WIN=< whl_file.txt set /p PADDLE_WHL_FILE_WIN=< whl_file.txt
%PYTHON_EXECUTABLE% -m pip install -U %PADDLE_WHL_FILE_WIN%
pip install -U %PADDLE_WHL_FILE_WIN%
if %ERRORLEVEL% NEQ 0 exit /b %ERRORLEVEL%
echo import paddle.fluid;print(paddle.__version__) > test_whl.py echo import paddle.fluid;print(paddle.__version__) > test_whl.py
%PYTHON_EXECUTABLE% test_whl.py python test_whl.py
goto:eof goto:eof
:test_whl_pacakage_error :test_whl_pacakage_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bat
exit /b 3
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
:unit_test :unit_test
echo ======================================== echo ========================================
echo Step 4. Running unit tests ... echo Step 4. Running unit tests ...
echo ======================================== echo ========================================
%PYTHON_EXECUTABLE% -m pip install --upgrade pip
dir %THIRD_PARTY_PATH:/=\%\install\openblas\lib dir %THIRD_PARTY_PATH:/=\%\install\openblas\lib
dir %THIRD_PARTY_PATH:/=\%\install\openblas\bin dir %THIRD_PARTY_PATH:/=\%\install\openblas\bin
...@@ -147,27 +201,28 @@ dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin ...@@ -147,27 +201,28 @@ dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin
dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin
set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;%THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;%THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH% set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;%THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;%THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH%
ctest.exe --output-on-failure -C Release -j 10 ctest.exe --output-on-failure -C Release -j 7
goto:eof goto:eof
:unit_test_error :unit_test_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bat
exit /b 8
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
:test_inference :test_inference
echo ======================================== echo ========================================
echo Step 5. Testing fluid library for inference ... echo Step 5. Testing fluid library for inference ...
echo ======================================== echo ========================================
if NOT EXIST "d:\.cache\tools" (
git clone https://github.com/zhouwei25/tools.git d:\.cache\tools
)
cd %work_dir%\paddle\fluid\inference\api\demo_ci cd %work_dir%\paddle\fluid\inference\api\demo_ci
d:\.cache\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% d:/.cache/inference_demo %cache_dir%\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% %cache_dir:\=/%/inference_demo
goto:eof goto:eof
:test_inference_error :test_inference_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bats
echo Testing fluid library for inference failed!
exit /b 5
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
:check_change_of_unittest :check_change_of_unittest
...@@ -205,7 +260,7 @@ echo git fetch upstream $BRANCH # develop is not fetched>> check_change_of_ ...@@ -205,7 +260,7 @@ echo git fetch upstream $BRANCH # develop is not fetched>> check_change_of_
echo fi>> check_change_of_unittest.sh echo fi>> check_change_of_unittest.sh
echo git checkout -b origin_pr >> check_change_of_unittest.sh echo git checkout -b origin_pr >> check_change_of_unittest.sh
echo git checkout -f $BRANCH >> check_change_of_unittest.sh echo git checkout -f $BRANCH >> check_change_of_unittest.sh
echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE:\=\\% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0" -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% >> check_change_of_unittest.sh echo cmake .. -G "Visual Studio 14 2015 Win64" -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR% -DON_INFER=%ON_INFER% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% >> check_change_of_unittest.sh
echo cat ^<^<EOF>> check_change_of_unittest.sh echo cat ^<^<EOF>> check_change_of_unittest.sh
echo ============================================ >> check_change_of_unittest.sh echo ============================================ >> check_change_of_unittest.sh
echo Generate unit tests.spec of develop. >> check_change_of_unittest.sh echo Generate unit tests.spec of develop. >> check_change_of_unittest.sh
...@@ -236,11 +291,12 @@ echo exit 1 >> check_change_of_unittest.sh ...@@ -236,11 +291,12 @@ echo exit 1 >> check_change_of_unittest.sh
echo fi>> check_change_of_unittest.sh echo fi>> check_change_of_unittest.sh
echo fi>> check_change_of_unittest.sh echo fi>> check_change_of_unittest.sh
echo git checkout -f origin_pr >> check_change_of_unittest.sh echo git checkout -f origin_pr >> check_change_of_unittest.sh
d:\.cache\tools\busybox64.exe bash check_change_of_unittest.sh %cache_dir%\tools\busybox64.exe bash check_change_of_unittest.sh
goto:eof goto:eof
:check_change_of_unittest_error :check_change_of_unittest_error
exit /b %ERRORLEVEL% call paddle_winci\Scripts\deactivate.bat
exit /b 6
rem --------------------------------------------------------------------------------------------- rem ---------------------------------------------------------------------------------------------
...@@ -258,6 +314,7 @@ taskkill /f /im git-remote-https.exe 2>NUL ...@@ -258,6 +314,7 @@ taskkill /f /im git-remote-https.exe 2>NUL
taskkill /f /im vctip.exe 2>NUL taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL taskkill /f /im rc.exe 2>NUL
call paddle_winci\Scripts\deactivate.bat
echo Windows CI run successfully! echo Windows CI run successfully!
exit /b 0 exit /b 0
......
...@@ -36,8 +36,9 @@ class TestDistributedReader(unittest.TestCase): ...@@ -36,8 +36,9 @@ class TestDistributedReader(unittest.TestCase):
data = next(reader()) data = next(reader())
assert data == 1 assert data == 1
os.unsetenv('PADDLE_TRAINER_ID') #Note: windows python3 don't have unsetenv
os.unsetenv('PADDLE_TRAINERS_NUM') del os.environ['PADDLE_TRAINER_ID']
del os.environ['PADDLE_TRAINERS_NUM']
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -191,6 +191,7 @@ function(py_test_modules TARGET_NAME) ...@@ -191,6 +191,7 @@ function(py_test_modules TARGET_NAME)
endif() endif()
endfunction() endfunction()
function(bash_test_modules TARGET_NAME) function(bash_test_modules TARGET_NAME)
if(NOT WITH_TESTING) if(NOT WITH_TESTING)
return() return()
......
...@@ -10,4 +10,15 @@ ...@@ -10,4 +10,15 @@
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.p
# Note: On Windows, import form subdirectories such as dirA()->dirB(), current directory
# will still be dirA(), But is should be dirB(). So it will ModulNotFoundError
# please refer to https://stackoverflow.com/questions/8953844/import-module-from-subfolder
import os
if os.name == 'nt':
import sys
dirname, filename = os.path.split(os.path.abspath(__file__))
sys.path.insert(0, dirname)
print(sys.path)
...@@ -6,7 +6,7 @@ gast==0.3.3 ...@@ -6,7 +6,7 @@ gast==0.3.3
matplotlib<=2.2.4 ; python_version<"3.6" matplotlib<=2.2.4 ; python_version<"3.6"
scipy>=0.19.0, <=1.2.1 ; python_version<"3.5" scipy>=0.19.0, <=1.2.1 ; python_version<"3.5"
nltk>=3.2.2, <=3.4 ; python_version<"3.5" nltk>=3.2.2, <=3.4 ; python_version<"3.5"
matplotlib ; python_version>="3.6" matplotlib<=3.2.1 ; python_version>="3.6"
scipy<=1.3.1 ; python_version=="3.5" scipy<=1.3.1 ; python_version=="3.5"
scipy ; python_version>"3.5" scipy ; python_version>"3.5"
nltk ; python_version>="3.5" nltk ; python_version>="3.5"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册