diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index a50b764c1cf8b73a6b3a721f5ac8edfb4aa6acb2..ff89af62ded62d0842672a1d386ed2645d3f1e71 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -123,6 +123,7 @@ where pip pip install wheel --user pip install -r %work_dir%\python\unittest_py\requirements.txt --user pip install -r %work_dir%\python\requirements.txt --user + if %ERRORLEVEL% NEQ 0 ( echo pip install requirements.txt failed! exit /b 7 @@ -234,7 +235,6 @@ set start=%start:~4,10% @ECHO ON if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0 set PATH=%TENSORRT_ROOT:/=\%\lib;%CUDA_TOOLKIT_ROOT_DIR%\bin;%CUDA_TOOLKIT_ROOT_DIR%\libnvvp;%PATH% -set CUDA_PATH=%CUDA_TOOLKIT_ROOT_DIR% rem ------set third_party cache dir------ @@ -246,7 +246,11 @@ set /p day_before=< %cache_dir%\day.txt if %day_now% NEQ %day_before% ( echo %day_now% > %cache_dir%\day.txt type %cache_dir%\day.txt - if %day_now% EQU 20 ( + if %day_now% EQU 25 ( + rmdir %cache_dir%\third_party_GPU/ /s/q + rmdir %cache_dir%\third_party/ /s/q + ) + if %day_now% EQU 10 ( rmdir %cache_dir%\third_party_GPU/ /s/q rmdir %cache_dir%\third_party/ /s/q ) @@ -452,7 +456,7 @@ if "%WITH_GPU%"=="ON" ( :parallel_test_base_gpu echo ======================================== -echo Running GPU unit tests... +echo Running GPU unit tests in parallel way ... echo ======================================== setlocal enabledelayedexpansion @@ -471,6 +475,7 @@ goto:eof echo ======================================== echo Running CPU unit tests in parallel way ... echo ======================================== + ctest.exe -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 goto:eof @@ -676,6 +681,7 @@ taskkill /f /im vctip.exe 2>NUL taskkill /f /im cvtres.exe 2>NUL taskkill /f /im rc.exe 2>NUL wmic process where name="op_function_generator.exe" call terminate 2>NUL +wmic process where name="python.exe" call terminate 2>NUL taskkill /f /im python.exe 2>NUL echo Windows CI run successfully! exit /b 0 diff --git a/python/paddle/fluid/tests/CMakeLists.txt b/python/paddle/fluid/tests/CMakeLists.txt index 4b6fb6de0d06f41581262a0be01a082b6abc58a5..899d6ae7f0e314ee05ae67ef5639d9f79410a38f 100644 --- a/python/paddle/fluid/tests/CMakeLists.txt +++ b/python/paddle/fluid/tests/CMakeLists.txt @@ -10,13 +10,8 @@ add_subdirectory(unittests) add_subdirectory(book) # TODO: support New Custom OP on Mac -if(LINUX) +if(NOT APPLE) add_subdirectory(custom_op) endif() -# Windows CPU machine doesn't have CUDA, can't compile .cu file -# if(WIN32 AND WITH_GPU) -# add_subdirectory(custom_op) -# endif() - set_tests_properties(test_beam_search_decoder PROPERTIES TIMEOUT 120) diff --git a/python/paddle/fluid/tests/custom_op/CMakeLists.txt b/python/paddle/fluid/tests/custom_op/CMakeLists.txt index 1307df1fc1a4a715cf3dc4d9ca104b1dcadc4b74..10d8b898c7589b3bc7c8a7e07ca094618de1405e 100644 --- a/python/paddle/fluid/tests/custom_op/CMakeLists.txt +++ b/python/paddle/fluid/tests/custom_op/CMakeLists.txt @@ -1,11 +1,13 @@ # New custom OP can support Windows/Linux now -# 'test_custom_relu_op_setup/jit' compile .cc and .cu file -py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py) -py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py) - -# Compiling shared library will cost some time, but running process is very fast. -set_tests_properties(test_custom_relu_op_setup PROPERTIES TIMEOUT 250) -set_tests_properties(test_custom_relu_op_jit PROPERTIES TIMEOUT 180) +if(WITH_GPU) + # 'test_custom_relu_op_setup/jit' compile .cc and .cu file + py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py) + py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py) + + # Compiling shared library will cost some time, but running process is very fast. + set_tests_properties(test_custom_relu_op_setup PROPERTIES TIMEOUT 250) + set_tests_properties(test_custom_relu_op_jit PROPERTIES TIMEOUT 180) +endif() py_test(test_sysconfig SRCS test_sysconfig.py) diff --git a/python/paddle/utils/cpp_extension/cpp_extension.py b/python/paddle/utils/cpp_extension/cpp_extension.py index 2789a89978b6b407407c4e3e30db5218a06d4efc..be22f3b66d6117c66300f9290f2e10f9fbcb5ee8 100644 --- a/python/paddle/utils/cpp_extension/cpp_extension.py +++ b/python/paddle/utils/cpp_extension/cpp_extension.py @@ -24,7 +24,7 @@ from setuptools.command.easy_install import easy_install from setuptools.command.build_ext import build_ext from .extension_utils import find_cuda_home, normalize_extension_kwargs, add_compile_flag, bootstrap_context -from .extension_utils import is_cuda_file, prepare_unix_cflags, prepare_win_cflags, add_std_without_repeat, get_build_directory +from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags, add_std_without_repeat, get_build_directory from .extension_utils import _import_module_from_library, CustomOpInfo, _write_setup_file, _jit_compile, parse_op_name_from from .extension_utils import check_abi_compatibility, log_v, IS_WINDOWS, OS_NAME from .extension_utils import use_new_custom_op_load_method, MSVC_COMPILE_FLAGS @@ -267,7 +267,7 @@ class BuildExtension(build_ext, object): if isinstance(cflags, dict): cflags = cflags['nvcc'] else: - cflags = prepare_unix_cflags(cflags) + cflags = prepare_unix_cudaflags(cflags) # cxx compile Cpp source elif isinstance(cflags, dict): cflags = cflags['cxx'] @@ -332,7 +332,7 @@ class BuildExtension(build_ext, object): else: cflags = [] - cflags = prepare_win_cflags(cflags) + ['--use-local-env'] + cflags = prepare_win_cudaflags(cflags) + ['--use-local-env'] for flag in MSVC_COMPILE_FLAGS: cflags = ['-Xcompiler', flag] + cflags cmd = [nvcc_cmd, '-c', src, '-o', obj diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index e53df3f083d8c94331b913fe36081ee1f9fe6f1d..ccdc3eb49c438f2b4896f5511ce09e8fb77ad74f 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -41,17 +41,13 @@ IS_WINDOWS = OS_NAME.startswith('win') MSVC_COMPILE_FLAGS = [ '/MT', '/wd4819', '/wd4251', '/wd4244', '/wd4267', '/wd4275', '/wd4018', - '/wd4190', '/EHsc', '/w', '/DPADDLE_WITH_CUDA', '/DEIGEN_USE_GPU', - '/DNDEBUG' + '/wd4190', '/EHsc', '/w', '/DGOOGLE_GLOG_DLL_DECL', + '/DBOOST_HAS_STATIC_ASSERT', '/DNDEBUG', '/DPADDLE_USE_DSO' ] -MSVC_LINK_FLAGS = [ - '/MACHINE:X64', 'paddle_framework.lib', 'cudadevrt.lib', 'cudart_static.lib' -] +MSVC_LINK_FLAGS = ['/MACHINE:X64', 'paddle_framework.lib'] -COMMON_NVCC_FLAGS = [ - '-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU', '-DPADDLE_USE_DSO', '-O3' -] +COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU', '-O3'] GCC_MINI_VERSION = (5, 4, 0) # Give warning if using wrong compiler @@ -216,7 +212,7 @@ class CustomOpInfo: return next(reversed(self.op_info_map.items())) -def prepare_unix_cflags(cflags): +def prepare_unix_cudaflags(cflags): """ Prepare all necessary compiled flags for nvcc compiling CUDA files. """ @@ -228,13 +224,11 @@ def prepare_unix_cflags(cflags): return cflags -def prepare_win_cflags(cflags): +def prepare_win_cudaflags(cflags): """ Prepare all necessary compiled flags for nvcc compiling CUDA files. """ - cflags = COMMON_NVCC_FLAGS + [ - '-DGOOGLE_GLOG_DLL_DECL', '-DBOOST_HAS_STATIC_ASSERT', '-w' - ] + cflags + get_cuda_arch_flags(cflags) + cflags = COMMON_NVCC_FLAGS + ['-w'] + cflags + get_cuda_arch_flags(cflags) return cflags @@ -269,6 +263,7 @@ def normalize_extension_kwargs(kwargs, use_cuda=False): # append necessary include dir path of paddle include_dirs = kwargs.get('include_dirs', []) include_dirs.extend(find_paddle_includes(use_cuda)) + kwargs['include_dirs'] = include_dirs # append necessary lib path of paddle @@ -282,6 +277,8 @@ def normalize_extension_kwargs(kwargs, use_cuda=False): # append link flags extra_link_args = kwargs.get('extra_link_args', []) extra_link_args.extend(MSVC_LINK_FLAGS) + if use_cuda: + extra_link_args.extend(['cudadevrt.lib', 'cudart_static.lib']) kwargs['extra_link_args'] = extra_link_args else: # append compile flags @@ -323,6 +320,7 @@ def find_cuda_home(): if six.PY3: nvcc_path = nvcc_path.decode() nvcc_path = nvcc_path.rstrip('\r\n') + # for example: /usr/local/cuda/bin/nvcc cuda_home = os.path.dirname(os.path.dirname(nvcc_path)) except: @@ -368,10 +366,9 @@ def find_paddle_includes(use_cuda=False): third_party_dir = os.path.join(paddle_include_dir, 'third_party') include_dirs = [paddle_include_dir, third_party_dir] - #TODO(zhouwei): because eigen need cuda_runtime.h - #So, extend cuda_include_dir always - cuda_include_dir = find_cuda_includes() - include_dirs.extend(cuda_include_dir) + if use_cuda: + cuda_include_dir = find_cuda_includes() + include_dirs.extend(cuda_include_dir) return include_dirs @@ -400,10 +397,9 @@ def find_paddle_libraries(use_cuda=False): # pythonXX/site-packages/paddle/libs paddle_lib_dirs = [get_lib()] - #TODO(zhouwei): because eigen need cuda_runtime.h - #So, extend cuda_lib_dir always - cuda_lib_dir = find_cuda_libraries() - paddle_lib_dirs.extend(cuda_lib_dir) + if use_cuda: + cuda_lib_dir = find_cuda_libraries() + paddle_lib_dirs.extend(cuda_lib_dir) return paddle_lib_dirs