From cc89120a2c210cd5d7b8ac586be2b0995f4c2f53 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Sun, 28 Feb 2021 09:05:46 +0800 Subject: [PATCH] [Custom OP]add MSVC compile check on Windows (#31265) --- .../fluid/tests/custom_op/CMakeLists.txt | 5 +- .../fluid/tests/custom_op/test_check_abi.py | 79 +++++++------------ .../utils/cpp_extension/extension_utils.py | 47 +++++++---- 3 files changed, 62 insertions(+), 69 deletions(-) diff --git a/python/paddle/fluid/tests/custom_op/CMakeLists.txt b/python/paddle/fluid/tests/custom_op/CMakeLists.txt index 857ee543b4a..1be1623d4a1 100644 --- a/python/paddle/fluid/tests/custom_op/CMakeLists.txt +++ b/python/paddle/fluid/tests/custom_op/CMakeLists.txt @@ -23,15 +23,14 @@ set_tests_properties(test_multi_out_jit PROPERTIES TIMEOUT 120) py_test(test_custom_attrs_jit SRCS test_custom_attrs_jit.py) set_tests_properties(test_custom_attrs_jit PROPERTIES TIMEOUT 120) +py_test(test_check_abi SRCS test_check_abi.py) + cc_test(test_check_error SRCS test_check_error.cc DEPS gtest) if(NOT LINUX) return() endif() -# TODO(zhouwei): support test_check_abi and abi check on Windows -py_test(test_check_abi SRCS test_check_abi.py) - # Old custom OP only support Linux, only run on Linux py_test(test_custom_op SRCS test_custom_op.py) py_test(test_jit_load SRCS test_jit_load.py) diff --git a/python/paddle/fluid/tests/custom_op/test_check_abi.py b/python/paddle/fluid/tests/custom_op/test_check_abi.py index b171fca2076..3ea7c35ee0d 100644 --- a/python/paddle/fluid/tests/custom_op/test_check_abi.py +++ b/python/paddle/fluid/tests/custom_op/test_check_abi.py @@ -22,10 +22,11 @@ import paddle.utils.cpp_extension.extension_utils as utils class TestABIBase(unittest.TestCase): def test_environ(self): - compiler = 'gcc' - for flag in ['1', 'True', 'true']: - os.environ['PADDLE_SKIP_CHECK_ABI'] = flag - self.assertTrue(utils.check_abi_compatibility(compiler)) + compiler_list = ['gcc', 'cl'] + for compiler in compiler_list: + for flag in ['1', 'True', 'true']: + os.environ['PADDLE_SKIP_CHECK_ABI'] = flag + self.assertTrue(utils.check_abi_compatibility(compiler)) def del_environ(self): key = 'PADDLE_SKIP_CHECK_ABI' @@ -33,43 +34,49 @@ class TestABIBase(unittest.TestCase): del os.environ[key] -class TestCheckLinux(TestABIBase): +class TestCheckCompiler(TestABIBase): def test_expected_compiler(self): if utils.OS_NAME.startswith('linux'): gt = ['gcc', 'g++', 'gnu-c++', 'gnu-cc'] - self.assertListEqual(utils._expected_compiler_current_platform(), - gt) + elif utils.IS_WINDOWS: + gt = ['cl'] + elif utils.OS_NAME.startswith('darwin'): + gt = ['clang', 'clang++'] + + self.assertListEqual(utils._expected_compiler_current_platform(), gt) - def test_gcc_version(self): + def test_compiler_version(self): # clear environ self.del_environ() - compiler = 'g++' if utils.OS_NAME.startswith('linux'): - # all CI gcc version > 5.4.0 - self.assertTrue( - utils.check_abi_compatibility( - compiler, verbose=True)) + compiler = 'g++' + elif utils.IS_WINDOWS: + compiler = 'cl' + + # Linux: all CI gcc version > 5.4.0 + # Windows: all CI MSVC version > 19.00.24215 + # Mac: clang has no version limitation, always return true + self.assertTrue(utils.check_abi_compatibility(compiler, verbose=True)) def test_wrong_compiler_warning(self): # clear environ self.del_environ() compiler = 'nvcc' # fake wrong compiler - if utils.OS_NAME.startswith('linux'): - with warnings.catch_warnings(record=True) as error: - flag = utils.check_abi_compatibility(compiler, verbose=True) - # check return False - self.assertFalse(flag) - # check Compiler Compatibility WARNING - self.assertTrue(len(error) == 1) - self.assertTrue( - "Compiler Compatibility WARNING" in str(error[0].message)) + with warnings.catch_warnings(record=True) as error: + flag = utils.check_abi_compatibility(compiler, verbose=True) + # check return False + self.assertFalse(flag) + # check Compiler Compatibility WARNING + self.assertTrue(len(error) == 1) + self.assertTrue( + "Compiler Compatibility WARNING" in str(error[0].message)) def test_exception(self): # clear environ self.del_environ() compiler = 'python' # fake command if utils.OS_NAME.startswith('linux'): - # to skip _expected_compiler_current_platform + def fake(): return [compiler] @@ -89,32 +96,6 @@ class TestCheckLinux(TestABIBase): utils._expected_compiler_current_platform = raw_func -class TestCheckMacOs(TestABIBase): - def test_expected_compiler(self): - if utils.OS_NAME.startswith('darwin'): - gt = ['clang', 'clang++'] - self.assertListEqual(utils._expected_compiler_current_platform(), - gt) - - def test_gcc_version(self): - # clear environ - self.del_environ() - - if utils.OS_NAME.startswith('darwin'): - # clang has no version limitation. - self.assertTrue(utils.check_abi_compatibility()) - - -class TestCheckWindows(TestABIBase): - def test_gcc_version(self): - # clear environ - self.del_environ() - - if utils.IS_WINDOWS: - # we skip windows now - self.assertTrue(utils.check_abi_compatibility()) - - class TestJITCompilerException(unittest.TestCase): def test_exception(self): with self.assertRaisesRegexp(RuntimeError, diff --git a/python/paddle/utils/cpp_extension/extension_utils.py b/python/paddle/utils/cpp_extension/extension_utils.py index 712342b41e5..475928f2437 100644 --- a/python/paddle/utils/cpp_extension/extension_utils.py +++ b/python/paddle/utils/cpp_extension/extension_utils.py @@ -51,6 +51,7 @@ MSVC_LINK_FLAGS = ['/MACHINE:X64', 'paddle_custom_op.lib'] COMMON_NVCC_FLAGS = ['-DPADDLE_WITH_CUDA', '-DEIGEN_USE_GPU', '-O3'] GCC_MINI_VERSION = (5, 4, 0) +MSVC_MINI_VERSION = (19, 0, 24215) # Give warning if using wrong compiler WRONG_COMPILER_WARNING = ''' ************************************* @@ -64,7 +65,7 @@ built Paddle for this platform, which is {paddle_compiler} on {platform}. Please use {paddle_compiler} to compile your custom op. Or you may compile Paddle from source using {user_compiler}, and then also use it compile your custom op. -See https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/2.0/install/compile/linux-compile.html +See https://www.paddlepaddle.org.cn/documentation/docs/zh/install/compile/fromsource.html for help with compiling Paddle from source. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -877,13 +878,12 @@ def check_abi_compatibility(compiler, verbose=False): Check whether GCC version on user local machine is compatible with Paddle in site-packages. """ - # TODO(Aurelius84): After we support windows, remove IS_WINDOWS in following code. - if os.environ.get('PADDLE_SKIP_CHECK_ABI') in ['True', 'true', '1' - ] or IS_WINDOWS: + if os.environ.get('PADDLE_SKIP_CHECK_ABI') in ['True', 'true', '1']: return True + which = 'where' if IS_WINDOWS else 'which' cmd_out = subprocess.check_output( - ['which', compiler], stderr=subprocess.STDOUT) + [which, compiler], stderr=subprocess.STDOUT) compiler_path = os.path.realpath(cmd_out.decode() if six.PY3 else cmd_out).strip() # step 1. if not found any suitable compiler, raise error @@ -896,32 +896,41 @@ def check_abi_compatibility(compiler, verbose=False): platform=OS_NAME)) return False + version = (0, 0, 0) # clang++ have no ABI compatibility problem if OS_NAME.startswith('darwin'): return True try: if OS_NAME.startswith('linux'): + mini_required_version = GCC_MINI_VERSION version_info = subprocess.check_output( [compiler, '-dumpfullversion', '-dumpversion']) if six.PY3: version_info = version_info.decode() version = version_info.strip().split('.') - assert len(version) == 3 - # check version compatibility - if tuple(map(int, version)) >= GCC_MINI_VERSION: - return True - else: - warnings.warn( - ABI_INCOMPATIBILITY_WARNING.format( - user_compiler=compiler, version=version_info.strip())) elif IS_WINDOWS: - # TODO(zhouwei): support check abi compatibility on windows - warnings.warn("We don't support Windows now.") + mini_required_version = MSVC_MINI_VERSION + compiler_info = subprocess.check_output( + compiler, stderr=subprocess.STDOUT) + if six.PY3: + compiler_info = compiler_info.decode() + match = re.search(r'(\d+)\.(\d+)\.(\d+)', compiler_info.strip()) + if match is not None: + version = match.groups() except Exception: + # check compiler version failed _, error, _ = sys.exc_info() warnings.warn('Failed to check compiler version for {}: {}'.format( compiler, error)) + return False + # check version compatibility + assert len(version) == 3 + if tuple(map(int, version)) >= mini_required_version: + return True + warnings.warn( + ABI_INCOMPATIBILITY_WARNING.format( + user_compiler=compiler, version=version.strip())) return False @@ -929,8 +938,12 @@ def _expected_compiler_current_platform(): """ Returns supported compiler string on current platform """ - expect_compilers = ['clang', 'clang++'] if OS_NAME.startswith( - 'darwin') else ['gcc', 'g++', 'gnu-c++', 'gnu-cc'] + if OS_NAME.startswith('darwin'): + expect_compilers = ['clang', 'clang++'] + elif OS_NAME.startswith('linux'): + expect_compilers = ['gcc', 'g++', 'gnu-c++', 'gnu-cc'] + elif IS_WINDOWS: + expect_compilers = ['cl'] return expect_compilers -- GitLab