From 1f5841c2a0556f62c797fffcfb99d87c7c836ae2 Mon Sep 17 00:00:00 2001 From: Qi Li Date: Thu, 21 Jan 2021 19:24:14 +0800 Subject: [PATCH] [ROCM] update cmake and dockerfile, test=develop (#30598) --- CMakeLists.txt | 33 ++- cmake/configure.cmake | 8 +- cmake/flags.cmake | 16 +- cmake/generic.cmake | 125 +++------ cmake/hip.cmake | 178 ++++++------ cmake/operators.cmake | 129 +++++---- cmake/rccl.cmake | 28 ++ python/CMakeLists.txt | 2 + .../fluid/tests/custom_op/CMakeLists.txt | 4 +- .../fluid/tests/unittests/CMakeLists.txt | 27 +- .../fluid/tests/unittests/ir/CMakeLists.txt | 2 +- .../tests/unittests/test_fleet_launch_ps.sh | 2 +- python/paddle/tests/CMakeLists.txt | 2 +- python/setup.py.in | 4 +- tools/dockerfile/Dockerfile.rocm | 264 ++++++++---------- tools/dockerfile/build_scripts/build.sh | 3 +- tools/dockerfile/rocm_dev.sh | 45 --- 17 files changed, 391 insertions(+), 481 deletions(-) create mode 100644 cmake/rccl.cmake delete mode 100755 tools/dockerfile/rocm_dev.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c2848d0b1..06518c9def 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,8 +148,9 @@ option(WITH_DISTRIBUTE "Compile with distributed support" OFF) option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF) option(ON_INFER "Turn on inference optimization and inference-lib generation" OFF) ################################ Internal Configurations ####################################### -option(WITH_ROCM_PLATFORM "Compile PaddlePaddle with ROCM platform" OFF) -option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF) +option(WITH_ROCM "Compile PaddlePaddle with ROCM platform" OFF) +option(WITH_RCCL "Compile PaddlePaddle with RCCL support" OFF) +option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF) option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools" OFF) option(WITH_COVERAGE "Compile PaddlePaddle with code coverage" OFF) option(WITH_INCREMENTAL_COVERAGE "Generate coverage reports only for incremental code" OFF) @@ -278,19 +279,25 @@ include(configure) # add paddle env configuration include_directories("${PADDLE_SOURCE_DIR}") -if(NOT DEFINED ENV{ROCM_PATH}) - set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed") - set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed") -else() - set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed") - set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed") +if(WITH_ROCM) + include(hip) +endif(WITH_ROCM) + +if (NOT WITH_ROCM AND WITH_RCCL) + MESSAGE(WARNING + "Disable RCCL when compiling without GPU. Force WITH_RCCL=OFF.") + set(WITH_NCCL OFF CACHE STRING + "Disable RCCL when compiling without GPU" FORCE) endif() -set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) -if(WITH_ROCM_PLATFORM) - find_package(HIP) - include(hip) -endif(WITH_ROCM_PLATFORM) +if(WITH_RCCL) + add_definitions("-DPADDLE_WITH_RCCL") + include(rccl) +else() + if(WITH_ROCM) + MESSAGE(WARNING "If the environment is multi-card, the WITH_RCCL option needs to be turned on, otherwise only a single card can be used.") + endif() +endif() if(WITH_NV_JETSON) set(WITH_ARM ON CACHE STRING "Set WITH_ARM=ON when compiling WITH_NV_JETSON=ON." FORCE) diff --git a/cmake/configure.cmake b/cmake/configure.cmake index fc1e72ba3f..9c1bd52e7f 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -130,14 +130,10 @@ if(WITH_GPU) endif() include_directories(${TENSORRT_INCLUDE_DIR}) endif() -elseif(WITH_ROCM_PLATFORM) +elseif(WITH_ROCM) add_definitions(-DPADDLE_WITH_HIP) + add_definitions(-DEIGEN_USE_GPU) add_definitions(-DEIGEN_USE_HIP) - add_definitions(-D__HIP_PLATFORM_HCC__) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__HIP_PLATFORM_HCC__") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__HIP_PLATFORM_HCC__") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP") - set(THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_HIP) else() add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 4e3dcac532..e110524dd1 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -155,7 +155,7 @@ set(COMMON_FLAGS ) if(NOT APPLE) - if((${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.0) OR (WITH_ROCM_PLATFORM AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 7.3)) + if((${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 8.0) OR (WITH_ROCM)) set(COMMON_FLAGS ${COMMON_FLAGS} -Wno-format-truncation # Warning in boost gcc 8.2 @@ -213,5 +213,17 @@ foreach(flag ${GPU_COMMON_FLAGS}) safe_set_nvflag(${flag}) endforeach() -set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${SAFE_GPU_COMMON_FLAGS}") +if(WITH_GPU) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${SAFE_GPU_COMMON_FLAGS}") +endif() + +if(WITH_ROCM) + set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} ${SAFE_GPU_COMMON_FLAGS}") +endif() + + # Disable -Werror, otherwise the compile will fail for rocblas_gemm_ex +if(WITH_ROCM) + string (REPLACE "-Werror" "-Wno-error" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + string (REPLACE "-Werror" "-Wno-error" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) +endif() diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 363803bb6b..1e9fc878da 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -382,6 +382,9 @@ function(cc_binary TARGET_NAME) endif() get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) target_link_libraries(${TARGET_NAME} ${os_dependency_modules}) + if(WITH_ROCM) + target_link_libraries(${TARGET_NAME} ${ROCM_HIPRTC_LIB}) + endif() check_coverage_opt(${TARGET_NAME} ${cc_binary_SRCS}) @@ -403,6 +406,9 @@ function(cc_test_build TARGET_NAME) target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} ${os_dependency_modules} paddle_gtest_main lod_tensor memory gtest gflags glog) add_dependencies(${TARGET_NAME} ${cc_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog) common_link(${TARGET_NAME}) + if(WITH_ROCM) + target_link_libraries(${TARGET_NAME} ${ROCM_HIPRTC_LIB}) + endif() endif() check_coverage_opt(${TARGET_NAME} ${cc_test_SRCS}) @@ -538,33 +544,24 @@ function(nv_test TARGET_NAME) endfunction(nv_test) function(hip_library TARGET_NAME) - if (WITH_ROCM_PLATFORM) + if (WITH_ROCM) set(options STATIC static SHARED shared) set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(hip_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(_sources ${hip_library_SRCS}) - set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) - HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options}) - if(_source_files) - list(REMOVE_ITEM _sources ${_source_files}) - endif() if(hip_library_SRCS) + # FindHIP.cmake defined hip_add_library, HIP_SOURCE_PROPERTY_FORMAT is requried if no .cu files found + if(NOT ${CMAKE_CURRENT_SOURCE_DIR} MATCHES ".*/operators") + set_source_files_properties(${hip_library_SRCS} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) + endif() if (hip_library_SHARED OR hip_library_shared) # build *.so - add_library(${TARGET_NAME} SHARED ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP) + hip_add_library(${TARGET_NAME} SHARED ${hip_library_SRCS}) else() - add_library(${TARGET_NAME} STATIC ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${TARGET_NAME} ${ROCM_PATH}/hip/lib/libhip_hcc.so) + hip_add_library(${TARGET_NAME} STATIC ${hip_library_SRCS}) find_fluid_modules(${TARGET_NAME}) endif() - if("${hip_library_DEPS}" MATCHES "ARCHIVE_START") - # Support linking flags: --whole-archive (Linux) / -force_load (MacOS). - # WARNING: Please don't use ARCHIVE_START&ARCHIVE_END if TARGET_NAME will be linked by other libraries. - target_circle_link_libraries(${TARGET_NAME} ${hip_library_DEPS}) - list(REMOVE_ITEM hip_library_DEPS ARCHIVE_START ARCHIVE_END) - else() + if (hip_library_DEPS) + add_dependencies(${TARGET_NAME} ${hip_library_DEPS}) target_link_libraries(${TARGET_NAME} ${hip_library_DEPS}) endif() # cpplint code style @@ -574,72 +571,27 @@ function(hip_library TARGET_NAME) list(APPEND hip_library_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h) endif() endforeach() - - check_coverage_opt(${TARGET_NAME} ${hip_library_SRCS}) - else(hip_library_SRCS) if (hip_library_DEPS) - merge_static_libs(${TARGET_NAME} ${hip_library_DEPS}) + list(REMOVE_DUPLICATES hip_library_DEPS) + generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:hip_library") + + target_link_libraries(${TARGET_NAME} ${hip_library_DEPS}) + add_dependencies(${TARGET_NAME} ${hip_library_DEPS}) else() - message(FATAL "Please specify source file or library in nv_library.") + message(FATAL "Please specify source file or library in hip_library.") endif() endif(hip_library_SRCS) endif() endfunction(hip_library) -function(hip_library_ops TARGET_NAME) - if (WITH_ROCM_PLATFORM) - set(options STATIC static SHARED shared) - set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) - cmake_parse_arguments(hip_library_ops "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(_sources ${hip_library_ops_SRCS}) - HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options}) - if(_source_files) - list(REMOVE_ITEM _sources ${_source_files}) - endif() - if(hip_library_ops_SRCS) - if (hip_library_ops_SHARED OR hip_library_ops_shared) # build *.so - add_library(${TARGET_NAME} SHARED ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP) - else() - add_library(${TARGET_NAME} STATIC ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX) - target_link_libraries(${TARGET_NAME} ${ROCM_PATH}/hip/lib/libhip_hcc.so) - find_fluid_modules(${TARGET_NAME}) - endif() - if("${hip_library_ops_DEPS}" MATCHES "ARCHIVE_START") - # Support linking flags: --whole-archive (Linux) / -force_load (MacOS). - # WARNING: Please don't use ARCHIVE_START&ARCHIVE_END if TARGET_NAME will be linked by other libraries. - target_circle_link_libraries(${TARGET_NAME} ${hip_library_ops_DEPS}) - list(REMOVE_ITEM hip_library_ops_DEPS ARCHIVE_START ARCHIVE_END) - else() - target_link_libraries(${TARGET_NAME} ${hip_library_ops_DEPS}) - endif() - # cpplint code style - foreach(source_file ${hip_library_ops_SRCS}) - string(REGEX REPLACE "\\.[^.]*$" "" source ${source_file}) - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h) - list(APPEND hip_library_ops_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h) - endif() - endforeach() - else(hip_library_ops_SRCS) - if (hip_library_ops_DEPS) - merge_static_libs(${TARGET_NAME} ${hip_library_ops_DEPS}) - else() - message(FATAL "Please specify source file or library in nv_library.") - endif() - endif(hip_library_ops_SRCS) - endif() -endfunction(hip_library_ops) - function(hip_binary TARGET_NAME) - if (WITH_ROCM_PLATFORM) + if (WITH_ROCM) set(options "") set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(hip_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) + # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files hip_add_executable(${TARGET_NAME} ${hip_binary_SRCS}) if(hip_binary_DEPS) target_link_libraries(${TARGET_NAME} ${hip_binary_DEPS}) @@ -647,34 +599,29 @@ function(hip_binary TARGET_NAME) common_link(${TARGET_NAME}) endif() endif() - - check_coverage_opt(${TARGET_NAME} ${hip_binary_SRCS}) - endfunction(hip_binary) function(hip_test TARGET_NAME) - if (WITH_ROCM_PLATFORM AND WITH_TESTING) - set(options "") + # The environment variable `CI_SKIP_CPP_TEST` is used to skip the compilation + # and execution of test in CI. `CI_SKIP_CPP_TEST` is set to ON when no files + # other than *.py are modified. + if (WITH_ROCM AND WITH_TESTING AND NOT "$ENV{CI_SKIP_CPP_TEST}" STREQUAL "ON") set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(hip_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(_sources ${hip_test_SRCS}) - set_source_files_properties(${_sources} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) - HIP_PREPARE_TARGET_COMMANDS(${TARGET_NAME} OBJ _generated_files _source_files ${_sources} HIPCC_OPTIONS ${_hipcc_options} HCC_OPTIONS ${_hcc_options} NVCC_OPTIONS ${_nvcc_options}) - if(_source_files) - list(REMOVE_ITEM _sources ${_source_files}) - endif() - add_executable(${TARGET_NAME} ${_cmake_options} ${_generated_files} ${_sources}) - set_target_properties(${TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP) + # FindHIP.cmake defined hip_add_executable, HIP_SOURCE_PROPERTY_FORMAT is requried for .cc files + hip_add_executable(${TARGET_NAME} ${hip_test_SRCS}) + # "-pthread -ldl -lrt" is defined in CMAKE_CXX_LINK_EXECUTABLE + target_link_options(${TARGET_NAME} PRIVATE -pthread -ldl -lrt) get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) - target_link_libraries(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main memory gtest gflags ${os_dependency_modules}) - add_dependencies(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main memory gtest gflags) + target_link_libraries(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog ${os_dependency_modules}) + add_dependencies(${TARGET_NAME} ${hip_test_DEPS} paddle_gtest_main lod_tensor memory gtest gflags glog) common_link(${TARGET_NAME}) add_test(${TARGET_NAME} ${TARGET_NAME}) + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true) + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true) + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true) endif() - - check_coverage_opt(${TARGET_NAME} ${hip_test_SRCS}) - endfunction(hip_test) function(go_library TARGET_NAME) diff --git a/cmake/hip.cmake b/cmake/hip.cmake index ac666ec686..523540c979 100644 --- a/cmake/hip.cmake +++ b/cmake/hip.cmake @@ -1,104 +1,86 @@ -if(NOT WITH_ROCM_PLATFORM) +if(NOT WITH_ROCM) return() endif() -include_directories("${ROCM_PATH}/include") -include_directories("${ROCM_PATH}/hip/include") -include_directories("${ROCM_PATH}/miopen/include") -include_directories("${ROCM_PATH}/hipblas/include") -include_directories("${ROCM_PATH}/rocblas/include") -include_directories("${ROCM_PATH}/hiprand/include") -include_directories("${ROCM_PATH}/rocrand/include") -include_directories("${ROCM_PATH}/rccl/include") - -include_directories("${ROCM_PATH}/rocthrust/include/") -include_directories("${ROCM_PATH}/hipcub/include/") -include_directories("${ROCM_PATH}/rocprim/include/") -include_directories("${ROCM_PATH}/hipsparse/include/") -include_directories("${ROCM_PATH}/rocsparse/include/") -include_directories("${ROCM_PATH}/rocfft/include/") - -set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "") -set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "") -# now default is clang -set(HIP_COMPILER "clang") - -list(APPEND EXTERNAL_LIBS "-L${ROCM_PATH}/lib/ -lhip_hcc") -set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -fPIC -DPADDLE_WITH_HIP -DEIGEN_USE_HIP -DEIGEN_USE_GPU -D__HIP_NO_HALF_CONVERSIONS__ -std=c++11 --amdgpu-target=gfx906" ) - -if(WITH_RCCL) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_RCCL") +if(NOT DEFINED ENV{ROCM_PATH}) + set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCm has been installed") + set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed") + set(HIP_CLANG_PATH ${ROCM_PATH}/llvm/bin CACHE PATH "Path to which clang has been installed") +else() + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCm has been installed") + set(HIP_PATH ${ROCM_PATH}/hip CACHE PATH "Path to which HIP has been installed") + set(HIP_CLANG_PATH ${ROCM_PATH}/llvm/bin CACHE PATH "Path to which clang has been installed") endif() - -if(NOT WITH_PYTHON) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_NO_PYTHON") -endif(NOT WITH_PYTHON) - -if(WITH_DSO) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_USE_DSO") -endif(WITH_DSO) - -if(WITH_TESTING) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_TESTING") -endif(WITH_TESTING) - -if(WITH_DISTRIBUTE) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_DISTRIBUTE") -endif(WITH_DISTRIBUTE) - -if(WITH_GRPC) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_GRPC") -endif(WITH_GRPC) - -if(WITH_MKLDNN) - set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DPADDLE_WITH_MKLDNN") -endif(WITH_MKLDNN) - -set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} -DANY_IMPL_ANY_CAST_MOVEABLE") - -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - list(APPEND HIP_HIPCC_FLAGS ${CMAKE_CXX_FLAGS_DEBUG}) -elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - list(APPEND HIP_HIPCC_FLAGS ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) -elseif(CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") - list(APPEND HIP_HIPCC_FLAGS ${CMAKE_CXX_FLAGS_MINSIZEREL}) +set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH}) + +find_package(HIP REQUIRED) +include_directories(${ROCM_PATH}/include) +message(STATUS "HIP version: ${HIP_VERSION}") +message(STATUS "HIP_CLANG_PATH: ${HIP_CLANG_PATH}") + +macro(find_package_and_include PACKAGE_NAME) + find_package("${PACKAGE_NAME}" REQUIRED) + include_directories("${ROCM_PATH}/${PACKAGE_NAME}/include") + message(STATUS "${PACKAGE_NAME} version: ${${PACKAGE_NAME}_VERSION}") +endmacro() + +find_package_and_include(miopen) +find_package_and_include(rocblas) +find_package_and_include(hiprand) +find_package_and_include(rocrand) +find_package_and_include(rccl) +find_package_and_include(rocthrust) +find_package_and_include(hipcub) +find_package_and_include(rocprim) +find_package_and_include(hipsparse) +find_package_and_include(rocsparse) +find_package_and_include(rocfft) + +# set CXX flags for HIP +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__HIP_PLATFORM_HCC__") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__HIP_PLATFORM_HCC__") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP") +set(THRUST_DEVICE_SYSTEM THRUST_DEVICE_SYSTEM_HIP) + +# define HIP_CXX_FLAGS +list(APPEND HIP_CXX_FLAGS -fPIC) +list(APPEND HIP_CXX_FLAGS -D__HIP_PLATFORM_HCC__=1) +list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_CONVERSIONS__=1) +list(APPEND HIP_CXX_FLAGS -Wno-macro-redefined) +list(APPEND HIP_CXX_FLAGS -Wno-inconsistent-missing-override) +list(APPEND HIP_CXX_FLAGS -Wno-exceptions) +list(APPEND HIP_CXX_FLAGS -Wno-shift-count-negative) +list(APPEND HIP_CXX_FLAGS -Wno-shift-count-overflow) +list(APPEND HIP_CXX_FLAGS -Wno-unused-command-line-argument) +list(APPEND HIP_CXX_FLAGS -Wno-duplicate-decl-specifier) +list(APPEND HIP_CXX_FLAGS -Wno-implicit-int-float-conversion) +list(APPEND HIP_CXX_FLAGS -Wno-pass-failed) +list(APPEND HIP_CXX_FLAGS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_HIP) +list(APPEND HIP_CXX_FLAGS -std=c++14) + +if(CMAKE_BUILD_TYPE MATCHES Debug) + list(APPEND HIP_CXX_FLAGS -g2) + list(APPEND HIP_CXX_FLAGS -O0) + list(APPEND HIP_HIPCC_FLAGS -fdebug-info-for-profiling) +endif(CMAKE_BUILD_TYPE MATCHES Debug) + +set(HIP_HCC_FLAGS ${HIP_CXX_FLAGS}) +set(HIP_CLANG_FLAGS ${HIP_CXX_FLAGS}) +# Ask hcc to generate device code during compilation so we can use +# host linker to link. +list(APPEND HIP_HCC_FLAGS -fno-gpu-rdc) +list(APPEND HIP_HCC_FLAGS --amdgpu-target=gfx906) +list(APPEND HIP_CLANG_FLAGS -fno-gpu-rdc) +list(APPEND HIP_CLANG_FLAGS --amdgpu-target=gfx906) + + +if(HIP_COMPILER STREQUAL clang) + set(hip_library_name amdhip64) +else() + set(hip_library_name hip_hcc) endif() +message(STATUS "HIP library name: ${hip_library_name}") -if("${HIP_COMPILER}" STREQUAL "hcc") - if("x${HCC_HOME}" STREQUAL "x") - set(HCC_HOME "${ROCM_PATH}/hcc") - endif() - - set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} -o -ldl --amdgpu-target=gfx906 ") - set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} -o -shared --amdgpu-target=gfx906") - set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} -o -shared --amdgpu-target=gfx906") - -elseif("${HIP_COMPILER}" STREQUAL "clang") - - if("x${HIP_CLANG_PATH}" STREQUAL "x") - set(HIP_CLANG_PATH "${ROCM_PATH}/llvm/bin") - endif() - - #Number of parallel jobs by default is 1 - if(NOT DEFINED HIP_CLANG_NUM_PARALLEL_JOBS) - set(HIP_CLANG_NUM_PARALLEL_JOBS 1) - endif() - #Add support for parallel build and link - if(${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") - check_cxx_compiler_flag("-parallel-jobs=1" HIP_CLANG_SUPPORTS_PARALLEL_JOBS) - endif() - if(HIP_CLANG_NUM_PARALLEL_JOBS GREATER 1) - if(${HIP_CLANG_SUPPORTS_PARALLEL_JOBS}) - set(HIP_CLANG_PARALLEL_BUILD_COMPILE_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS} -Wno-format-nonliteral") - set(HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS "-parallel-jobs=${HIP_CLANG_NUM_PARALLEL_JOBS}") - else() - message("clang compiler doesn't support parallel jobs") - endif() - endif() - - - # Set the CMake Flags to use the HIP-Clang Compiler. - set(CMAKE_HIP_CREATE_SHARED_LIBRARY "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} -o --amdgpu-target=gfx906") - set(CMAKE_HIP_CREATE_SHARED_MODULE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} -o -shared --amdgpu-target=gfx906" ) - set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HIP_CLANG_PATH} ${HIP_CLANG_PARALLEL_BUILD_LINK_OPTIONS} -o -ldl --amdgpu-target=gfx906") -endif() +# set HIP link libs +find_library(ROCM_HIPRTC_LIB ${hip_library_name} HINTS ${HIP_PATH}/lib) +message(STATUS "ROCM_HIPRTC_LIB: ${ROCM_HIPRTC_LIB}") diff --git a/cmake/operators.cmake b/cmake/operators.cmake index 824daf7751..757da1c829 100644 --- a/cmake/operators.cmake +++ b/cmake/operators.cmake @@ -7,13 +7,16 @@ function(op_library TARGET) # for ops. set(cc_srcs) set(cu_srcs) - set(hip_cu_srcs) - set(miopen_hip_cc_srcs) + set(hip_srcs) set(cu_cc_srcs) + set(hip_cc_srcs) set(xpu_cc_srcs) set(cudnn_cu_cc_srcs) + set(miopen_cu_cc_srcs) set(cudnn_cu_srcs) + set(miopen_cu_srcs) set(CUDNN_FILE) + set(MIOPEN_FILE) set(mkldnn_cc_srcs) set(MKLDNN_FILE) set(op_common_deps operator op_registry math_function layer common_infer_shape_functions) @@ -30,46 +33,44 @@ function(op_library TARGET) if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cc) list(APPEND cc_srcs ${TARGET}.cc) endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc) - list(APPEND cu_cc_srcs ${TARGET}.cu.cc) - endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu) - list(APPEND cu_srcs ${TARGET}.cu) - endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) - set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu - ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE) - list(APPEND cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) - endif() - - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu) - list(APPEND hip_cu_srcs ${TARGET}.hip.cu) - endif() - string(REPLACE "_op" "_cudnn_op" CUDNN_FILE "${TARGET}") - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu.cc) - list(APPEND cudnn_cu_cc_srcs ${CUDNN_FILE}.cu.cc) - endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu) - list(APPEND cudnn_cu_srcs ${CUDNN_FILE}.cu) + if(WITH_GPU) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc) + list(APPEND cu_cc_srcs ${TARGET}.cu.cc) + endif() + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu) + list(APPEND cu_srcs ${TARGET}.cu) + endif() + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) + set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu + ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE) + list(APPEND cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) + endif() + string(REPLACE "_op" "_cudnn_op" CUDNN_FILE "${TARGET}") + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu.cc) + list(APPEND cudnn_cu_cc_srcs ${CUDNN_FILE}.cu.cc) + endif() + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu) + list(APPEND cudnn_cu_srcs ${CUDNN_FILE}.cu) + endif() endif() - if(WITH_ROCM_PLATFORM) - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu.cc) - list(APPEND hip_cu_cc_srcs ${TARGET}.hip.cu.cc) + if(WITH_ROCM) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc) + list(APPEND hip_cc_srcs ${TARGET}.cu.cc) endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.hip.cu) - list(APPEND hip_cu_srcs ${TARGET}.hip.cu) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu) + list(APPEND hip_srcs ${TARGET}.cu) endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu) - set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) + set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE) - list(APPEND hip_cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.hip.cu) + list(APPEND hip_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu) endif() - string(REPLACE "_op" "_miopen_op" MIOPEN_FILE "${TARGET}") - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.hip.cu.cc) - list(APPEND miopen_hip_cu_cc_srcs ${MIOPEN_FILE}.hip.cu.cc) + string(REPLACE "_op" "_cudnn_op" MIOPEN_FILE "${TARGET}") + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu.cc) + list(APPEND miopen_cu_cc_srcs ${MIOPEN_FILE}.cu.cc) endif() - if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.hip.cu) - list(APPEND miopen_hip_cu_srcs ${MIOPEN_FILE}.hip.cu) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu) + list(APPEND miopen_cu_srcs ${MIOPEN_FILE}.cu) endif() endif() if(WITH_MKLDNN) @@ -86,20 +87,20 @@ function(op_library TARGET) endif() else() foreach(src ${op_library_SRCS}) - if (WITH_ROCM_PLATFORM AND ${src} MATCHES ".*\\.hip.cu$") - list(APPEND hip_cu_srcs ${src}) - elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*\\.hip.cu.cc$") - list(APPEND hip_cu_cc_srcs ${src}) + if(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu$") + list(APPEND miopen_cu_srcs ${src}) + elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu$") + list(APPEND hip_srcs ${src}) + elseif(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu.cc$") + list(APPEND miopen_cu_cc_srcs ${src}) + elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu.cc$") + list(APPEND hip_cc_srcs ${src}) elseif(${src} MATCHES ".*_cudnn_op.cu$") list(APPEND cudnn_cu_srcs ${src}) elseif (${src} MATCHES ".*\\.cu$") list(APPEND cu_srcs ${src}) elseif(${src} MATCHES ".*_cudnn_op.cu.cc$") list(APPEND cudnn_cu_cc_srcs ${src}) - elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*_miopen_op.hip.cc$") - list(APPEND miopen_hip_cc_srcs ${src}) - elseif(WITH_ROCM_PLATFORM AND ${src} MATCHES ".*_miopen_op.hip.cu$") - list(APPEND miopen_hip_cu_srcs ${src}) elseif(WITH_MKLDNN AND ${src} MATCHES ".*_mkldnn_op.cc$") list(APPEND mkldnn_cc_srcs ${src}) elseif(${src} MATCHES ".*\\.cu.cc$") @@ -163,8 +164,13 @@ function(op_library TARGET) nv_library(${TARGET} SRCS ${cc_srcs} ${cu_cc_srcs} ${cudnn_cu_cc_srcs} ${cudnn_cu_srcs} ${mkldnn_cc_srcs} ${cu_srcs} DEPS ${op_library_DEPS} ${op_common_deps}) endif() - elseif (WITH_ROCM_PLATFORM) - hip_library_ops(${TARGET} SRCS ${cc_srcs} ${hip_cu_cc_srcs} ${hip_cu_srcs} ${miopen_hip_cu_cc_srcs} ${miopen_hip_cu_srcs} ${mkldnn_cc_srcs} DEPS ${op_library_DEPS} + elseif (WITH_ROCM) + list(REMOVE_ITEM miopen_cu_cc_srcs "affine_grid_cudnn_op.cu.cc") + list(REMOVE_ITEM miopen_cu_cc_srcs "grid_sampler_cudnn_op.cu.cc") + list(REMOVE_ITEM hip_srcs "cholesky_op.cu") + list(REMOVE_ITEM hip_srcs "correlation_op.cu") + list(REMOVE_ITEM hip_srcs "multinomial_op.cu") + hip_library(${TARGET} SRCS ${cc_srcs} ${hip_cc_srcs} ${miopen_cu_cc_srcs} ${miopen_cu_srcs} ${mkldnn_cc_srcs} ${hip_srcs} DEPS ${op_library_DEPS} ${op_common_deps}) else() # Unity Build relies on global option `WITH_UNITY_BUILD` and local option `UNITY`. @@ -227,13 +233,14 @@ function(op_library TARGET) # pybind USE_CPU_ONLY_OP list(LENGTH cu_srcs cu_srcs_len) + list(LENGTH hip_srcs hip_srcs_len) list(LENGTH cu_cc_srcs cu_cc_srcs_len) + list(LENGTH hip_cc_srcs hip_cc_srcs_len) list(LENGTH mkldnn_cc_srcs mkldnn_cc_srcs_len) list(LENGTH xpu_cc_srcs xpu_cc_srcs_len) - list(LENGTH hip_cu_srcs hip_cu_srcs_len) - list(LENGTH miopen_hip_cc_srcs miopen_hip_cc_srcs_len) + list(LENGTH miopen_cu_cc_srcs miopen_cu_cc_srcs_len) if (${pybind_flag} EQUAL 0 AND ${mkldnn_cc_srcs_len} EQUAL 0 AND ${cu_srcs_len} EQUAL 0 AND ${cu_cc_srcs_len} EQUAL 0 AND - ${hip_cu_srcs_len} EQUAL 0 AND ${miopen_hip_cc_srcs_len} EQUAL 0 AND ${xpu_cc_srcs_len} EQUAL 0) + ${hip_srcs_len} EQUAL 0 AND ${hip_cc_srcs_len} EQUAL 0 AND ${miopen_cu_cc_srcs_len} EQUAL 0 AND ${xpu_cc_srcs_len} EQUAL 0) file(APPEND ${pybind_file} "USE_CPU_ONLY_OP(${TARGET});\n") set(pybind_flag 1) endif() @@ -248,26 +255,26 @@ function(op_library TARGET) endif() endif() - # pybind USE_OP_DEVICE_KERNEL for CUDNN - list(LENGTH cudnn_cu_srcs cudnn_cu_srcs_len) - if (WITH_GPU AND ${cudnn_cu_srcs_len} GREATER 0) - file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") - endif() - # pybind USE_OP_DEVICE_KERNEL for MIOPEN - list(LENGTH miopen_hip_cu_cc_srcs miopen_hip_cu_cc_srcs_len) - if (WITH_ROCM_PLATFORM AND ${miopen_hip_cu_cc_srcs_len} GREATER 0) + list(LENGTH miopen_cu_cc_srcs miopen_cu_cc_srcs_len) + if (WITH_ROCM AND ${miopen_cu_cc_srcs_len} GREATER 0) if(${TARGET} STREQUAL "activation") file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(relu, CUDNN);\n") else() - file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") + file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") endif() endif() + # pybind USE_OP_DEVICE_KERNEL for CUDNN + list(LENGTH cudnn_cu_srcs cudnn_cu_srcs_len) + if (WITH_GPU AND ${cudnn_cu_srcs_len} GREATER 0) + file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") + endif() + # pybind USE_OP_DEVICE_KERNEL for MIOPEN - list(LENGTH miopen_hip_cu_srcs miopen_hip_cu_srcs_len) - if (WITH_ROCM_PLATFORM AND ${miopen_hip_cu_srcs_len} GREATER 0) - file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") + list(LENGTH miopen_cu_srcs miopen_cu_srcs_len) + if (WITH_ROCM AND ${miopen_cu_srcs_len} GREATER 0) + file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${TARGET}, CUDNN);\n") endif() if (WITH_XPU AND ${xpu_cc_srcs_len} GREATER 0) diff --git a/cmake/rccl.cmake b/cmake/rccl.cmake new file mode 100644 index 0000000000..f3a472ac93 --- /dev/null +++ b/cmake/rccl.cmake @@ -0,0 +1,28 @@ +if(NOT WITH_ROCM) + return() +endif() + +# Now we don't support RCCL on windows +if(WIN32) + return() +endif() + +if(WITH_RCCL) + set(RCCL_ROOT ${ROCM_PATH}/rccl CACHE PATH "RCCL ROOT") + find_path(RCCL_INCLUDE_DIR rccl.h + PATHS ${RCCL_ROOT} ${RCCL_ROOT}/include ${RCCL_ROOT}/local/include + $ENV{RCCL_ROOT} $ENV{RCCL_ROOT}/include $ENV{RCCL_ROOT}/local/include + NO_DEFAULT_PATH + ) + + file(READ ${RCCL_INCLUDE_DIR}/rccl.h RCCL_VERSION_FILE_CONTENTS) + + string(REGEX MATCH "define NCCL_VERSION_CODE +([0-9]+)" + RCCL_VERSION "${RCCL_VERSION_FILE_CONTENTS}") + string(REGEX REPLACE "define NCCL_VERSION_CODE +([0-9]+)" "\\1" + RCCL_VERSION "${RCCL_VERSION}") + + # 2604 for ROCM3.5 and 2708 for ROCM 3.9 + message(STATUS "Current RCCL header is ${RCCL_INCLUDE_DIR}/rccl.h. " + "Current RCCL version is v${RCCL_VERSION}. ") +endif() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0be09c1ec6..e0e845601c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -6,6 +6,8 @@ set(PY_FILES paddle/__init__.py if(WITH_GPU) SET(PACKAGE_NAME "paddlepaddle-gpu") +elseif(WITH_ROCM) + SET(PACKAGE_NAME "paddlepaddle-rocm") else() SET(PACKAGE_NAME "paddlepaddle") endif() diff --git a/python/paddle/fluid/tests/custom_op/CMakeLists.txt b/python/paddle/fluid/tests/custom_op/CMakeLists.txt index bb74c37c04..ef3b39ef5c 100644 --- a/python/paddle/fluid/tests/custom_op/CMakeLists.txt +++ b/python/paddle/fluid/tests/custom_op/CMakeLists.txt @@ -1,4 +1,6 @@ -if (WITH_GPU) +if(WITH_ROCM) + hip_library(relu_op_shared SHARED SRCS relu_op.cc relu_op.cu DEPS paddle_framework_shared) +elseif(WITH_GPU) nv_library(relu_op_shared SHARED SRCS relu_op.cc relu_op.cu DEPS paddle_framework_shared) else() cc_library(relu_op_shared SHARED SRCS relu_op.cc DEPS paddle_framework_shared) diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 7d3194d44e..6f66d0f044 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -5,7 +5,7 @@ set(dist_ENVS http_proxy="" https_proxy="") file(GLOB DIST_TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_dist_*.py") list(REMOVE_ITEM DIST_TEST_OPS "test_dist_op") -if(NOT WITH_NCCL) +if ((NOT WITH_NCCL) AND (NOT WITH_RCCL)) list(REMOVE_ITEM DIST_TEST_OPS "test_dist_mnist_dgc_nccl") endif() string(REPLACE ".py" "" DIST_TEST_OPS "${DIST_TEST_OPS}") @@ -63,7 +63,7 @@ foreach(TEST_OP ${MIXED_DIST_TEST_OPS}) list(REMOVE_ITEM TEST_OPS ${TEST_OP}) endforeach() -if(NOT WITH_GPU OR WIN32) +if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_c_comm_init_all_op) LIST(REMOVE_ITEM TEST_OPS test_allgather) LIST(REMOVE_ITEM TEST_OPS test_allreduce) @@ -146,7 +146,7 @@ if(APPLE OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_fleet_metric) endif() -if (NOT ${WITH_GPU}) +if ((NOT WITH_GPU) AND (NOT WITH_ROCM)) LIST(REMOVE_ITEM TEST_OPS test_conv2d_fusion_op) LIST(REMOVE_ITEM TEST_OPS test_rank_attention_op) # TODO(shenliang03): rank_attention_op support CPU device in future LIST(REMOVE_ITEM TEST_OPS test_batch_fc_op) # TODO(shenliang03): batch_fc_op support CPU device in future @@ -159,9 +159,10 @@ if (NOT ${WITH_GPU}) LIST(REMOVE_ITEM TEST_OPS test_parallel_dygraph_sync_batch_norm) LIST(REMOVE_ITEM TEST_OPS test_imperative_auto_mixed_precision) LIST(REMOVE_ITEM TEST_OPS test_fleet_base_single) - -elseif(${CUDNN_VERSION} VERSION_LESS 7100) - LIST(REMOVE_ITEM TEST_OPS test_conv2d_fusion_op) +elseif(WITH_GPU) + if (${CUDNN_VERSION} VERSION_LESS 7100) + LIST(REMOVE_ITEM TEST_OPS test_conv2d_fusion_op) + endif() endif() if (WITH_NCCL) @@ -172,11 +173,11 @@ if (WITH_NCCL) endif() endif() -if(NOT WITH_NCCL) +if ((NOT WITH_NCCL) AND (NOT WITH_RCCL)) list(REMOVE_ITEM TEST_OPS test_imperative_group) endif() -if(NOT WITH_GPU OR WIN32) +if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_boxps) endif() list(REMOVE_ITEM TEST_OPS test_seq_concat_op) # FIXME(helin): https://github.com/PaddlePaddle/Paddle/issues/8290 @@ -213,7 +214,7 @@ endif() list(REMOVE_ITEM TEST_OPS test_fleet_pyramid_hash) -if(WITH_GPU OR NOT WITH_MKLML) +if((WITH_ROCM OR WITH_GPU) OR NOT WITH_MKLML) # matmul with multiple heads need MKL support LIST(REMOVE_ITEM TEST_OPS test_matmul_op_with_head) endif() @@ -510,7 +511,7 @@ if(WITH_DISTRIBUTE) list(REMOVE_ITEM DIST_TEST_OPS "test_dist_se_resnext_dgc") endif() if(NOT APPLE) - if(WITH_GPU) + if(WITH_GPU OR WITH_ROCM) bash_test_modules(test_c_comm_init_op START_BASH test_c_comm_init_op.sh ENVS PADDLE_BINARY_DIR=${PADDLE_BINARY_DIR}) py_test_modules(test_launch_coverage MODULES test_launch_coverage) endif() @@ -667,7 +668,7 @@ if (WITH_DISTRIBUTE) endif() if (WITH_DISTRIBUTE AND NOT APPLE) - if(WITH_GPU) + if(WITH_GPU OR WITH_ROCM) set_tests_properties(test_c_comm_init_op PROPERTIES TIMEOUT 120) set_tests_properties(test_dist_mnist_gradient_merge PROPERTIES TIMEOUT 120) endif() @@ -821,7 +822,7 @@ if(WITH_DISTRIBUTE AND WITH_GPU AND WITH_NCCL) set_tests_properties(test_parallel_dygraph_unused_variables PROPERTIES TIMEOUT 120) endif() endif() -if(WITH_GPU AND NOT WIN32) +if((WITH_ROCM OR WITH_GPU) AND NOT WIN32) set_tests_properties(test_collective_allgather_api PROPERTIES TIMEOUT 120) set_tests_properties(test_collective_broadcast_api PROPERTIES TIMEOUT 120) set_tests_properties(test_collective_allreduce_api PROPERTIES TIMEOUT 120) @@ -851,7 +852,7 @@ if(WITH_GPU AND NOT WIN32) test_collective_allgather_api PROPERTIES LABELS "RUN_TYPE=DIST") endif() -if(WITH_GPU) +if(WITH_GPU OR WITH_ROCM) set_tests_properties(test_imperative_auto_mixed_precision PROPERTIES TIMEOUT 120) set_tests_properties(test_parallel_dygraph_sync_batch_norm PROPERTIES TIMEOUT 120) set_tests_properties(test_rank_attention_op PROPERTIES TIMEOUT 120) diff --git a/python/paddle/fluid/tests/unittests/ir/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/CMakeLists.txt index 9ecddac3a0..5fc05a3a7c 100644 --- a/python/paddle/fluid/tests/unittests/ir/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/CMakeLists.txt @@ -1,7 +1,7 @@ file(GLOB TEST_IR_PASSES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") string(REPLACE ".py" "" TEST_IR_PASSES "${TEST_IR_PASSES}") -if(NOT WITH_GPU OR WIN32 OR APPLE) +if(((NOT WITH_GPU) AND (NOT WITH_ROCM)) OR WIN32 OR APPLE) LIST(REMOVE_ITEM TEST_IR_PASSES test_ir_fusion_group_pass) endif() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_launch_ps.sh b/python/paddle/fluid/tests/unittests/test_fleet_launch_ps.sh index 88822deacc..21875851bf 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_launch_ps.sh +++ b/python/paddle/fluid/tests/unittests/test_fleet_launch_ps.sh @@ -55,7 +55,7 @@ function test_launch_ps_heter(){ fi } -if [[ ${WITH_GPU} == "OFF" ]]; then +if [[ ${WITH_GPU} == "OFF" && ("${WITH_ROCM}x" == "x" || ${WITH_ROCM} == "OFF") ]]; then echo "in cpu test mode" test_launch_ps exit 0 diff --git a/python/paddle/tests/CMakeLists.txt b/python/paddle/tests/CMakeLists.txt index c88e22de9c..c0196f605c 100644 --- a/python/paddle/tests/CMakeLists.txt +++ b/python/paddle/tests/CMakeLists.txt @@ -19,7 +19,7 @@ function(py_dist_test TARGET_NAME) set(multiValueArgs SRCS DEPS ARGS ENVS) cmake_parse_arguments(py_dist_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if(WITH_COVERAGE AND WITH_GPU AND WITH_NCCL AND NOT WIN32) + if(WITH_COVERAGE AND (WITH_GPU OR WITH_ROCM) AND (WITH_NCCL OR WITH_RCCL) AND NOT WIN32) add_test(NAME ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E env FLAGS_init_allocated_mem=true FLAGS_cudnn_deterministic=true FLAGS_cpu_deterministic=true NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 diff --git a/python/setup.py.in b/python/setup.py.in index 3c13f55d4d..fd61599924 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -401,7 +401,7 @@ headers = ( if '${WITH_MKLDNN}' == 'ON': headers += list(find_files('*', '${MKLDNN_INSTALL_DIR}/include')) # mkldnn -if '${WITH_GPU}' == 'ON': +if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON': headers += list(find_files('*.pb', '${cudaerror_INCLUDE_DIR}')) # errorMessage.pb for errormessage class InstallCommand(InstallCommandBase): @@ -462,7 +462,7 @@ class InstallHeaders(Command): def run(self): # only copy third_party/cudaErrorMessage.pb for cudaErrorMessage on mac or windows if os.name == 'nt' or sys.platform == 'darwin': - if '${WITH_GPU}' == 'ON': + if '${WITH_GPU}' == 'ON' or '${WITH_ROCM}' == 'ON': self.mkdir_and_copy_file('${cudaerror_INCLUDE_DIR}/cudaErrorMessage.pb') return hdrs = self.distribution.headers diff --git a/tools/dockerfile/Dockerfile.rocm b/tools/dockerfile/Dockerfile.rocm index d761b64dce..fad20fbaea 100644 --- a/tools/dockerfile/Dockerfile.rocm +++ b/tools/dockerfile/Dockerfile.rocm @@ -1,163 +1,133 @@ # A image for building paddle binaries # Use rocm-terminal base image for both rocm environment # When you modify it, please be aware of rocm version -FROM ubuntu:18.04 +# +# Build: ROCM 3.5.1 +# cd Paddle/tools/dockerfile +# docker build -f Dockerfile.rocm \ +# --build-arg ROCM_VERSION=3.5.1 \ +# --build-arg CENTOS_VERSION=7.7.1908 \ +# -t paddlepaddle/paddle-centos-rocm35-dev:latest . +# +# Build: ROCM 3.9.1 +# cd Paddle/tools/dockerfile +# docker build -f Dockerfile.rocm \ +# --build-arg ROCM_VERSION=3.9.1 \ +# --build-arg CENTOS_VERSION=7.8.2003 \ +# -t paddlepaddle/paddle-centos-rocm39-dev:latest . +# +# Run: ROCM 3.5.1 +# docker run -it --device=/dev/kfd --device=/dev/dri \ +# --security-opt seccomp=unconfined --group-add video \ +# paddlepaddle/paddle-centos-rocm35-dev:latest /bin/bash + +ARG CENTOS_VERSION +FROM centos:${CENTOS_VERSION} +ARG CENTOS_VERSION MAINTAINER PaddlePaddle Authors -# ENV variables -ARG WITH_GPU -ARG WITH_ROCM_PLATFORM - -ENV WITH_GPU=${WITH_GPU:-OFF} -ENV WITH_ROCM_PLATFORM=${WITH_ROCM_PLATFORM:-ON} - -ENV HOME /root -ENV DEBIAN_FRONTEND=noninteractive - -# Add bash enhancements -COPY paddle/scripts/docker/root/ /root/ - -# Update Environment -RUN apt-get update && apt-get upgrade -y -RUN apt-get update && apt-get install -y apt-utils sudo - -# Update Timezone -RUN apt install tzdata && \ - ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' > /etc/timezone && \ - dpkg-reconfigure -f noninteractive tzdata - -# Location -RUN apt-get update && apt-get install -y locales && locale-gen en_US.UTF-8 - ENV LANG="en_US.UTF-8" - ENV LANGUAGE="en_US.UTF-8" - ENV LC_ALL="en_US.UTF-8" - -RUN apt-get update && \ - apt-get install -y make cmake build-essential libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev \ - git vim texinfo patchelf openssl unzip pciutils net-tools python-pip python-dev \ - python-opencv python-matplotlib - -# Downgrade gcc&&g++ -WORKDIR /usr/bin -COPY tools/dockerfile/build_scripts /build_scripts -RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts -RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++ && \ - ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc && \ - ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++ && \ - ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc && \ - ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++ -ENV PATH=/usr/local/gcc-8.2/bin:$PATH - -# install cmake +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +RUN yum install -y epel-release deltarpm sudo openssh-server openssl-devel gettext-devel sqlite-devel \ + zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz wget curl-devel \ + make bzip2 git patch unzip bison yasm diffutils automake which file kernel-headers kernel-devel + +# Install devtoolset-7 for ROCM 3.5/3.9 +RUN yum install -y yum-utils centos-release-scl && \ + yum-config-manager --enable rhel-server-rhscl-7-rpms && \ + yum-config-manager --enable rhel-7-server-rpms && \ + yum-config-manager --enable rhel-7-server-optional-rpms && \ + INSTALL_PKGS="devtoolset-7-binutils devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran devtoolset-7-gdb" && \ + yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + yum -y clean all --enablerepo='*' +ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:$LD_LIBRARY_PATH +RUN echo "source scl_source enable devtoolset-7" > "/etc/profile.d/devtoolset-7.sh" + +# cmake 3.16.0 WORKDIR /opt -RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && tar -zxf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz -ENV PATH=/opt/cmake-3.16.0-Linux-x86_64/bin:$PATH -RUN echo "export PATH=/opt/cmake-3.16.0-Linux-x86_64/bin:\${PATH}" >> ~/.bashrc - -# Install Go and glide -RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \ +RUN wget -q https://cmake.org/files/v3.16/cmake-3.16.0-Linux-x86_64.tar.gz && \ + tar -zxvf cmake-3.16.0-Linux-x86_64.tar.gz && rm cmake-3.16.0-Linux-x86_64.tar.gz && \ + mv cmake-3.16.0-Linux-x86_64 cmake-3.16 +ENV PATH=/opt/cmake-3.16/bin:${PATH} + +# ROCM +ARG ROCM_VERSION +RUN yum install -y kmod wget openblas-devel epel-release +RUN echo "[ROCm]" > /etc/yum.repos.d/rocm.repo && \ + echo "name=ROCm" >> /etc/yum.repos.d/rocm.repo && \ + echo "baseurl=http://repo.radeon.com/rocm/yum/${ROCM_VERSION}" >> /etc/yum.repos.d/rocm.repo && \ + echo "enabled=1" >> /etc/yum.repos.d/rocm.repo && \ + echo "gpgcheck=0" >> /etc/yum.repos.d/rocm.repo +RUN yum install -y rocm-dev rocm-utils rocfft miopen-hip rocblas hipsparse rocrand rccl hipcub rocthrust rocprofiler-dev roctracer-dev +# fix rocthrust +RUN sed -i '21 a #include ' /opt/rocm/include/thrust/system/hip/detail/error.inl + +# git 2.17.1 +RUN cd /opt && wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \ + tar -xvf git-2.17.1.tar.gz && \ + cd git-2.17.1 && \ + ./configure --with-openssl --prefix=/usr/local && \ + make -j8 && make install && \ + cd .. && rm -rf git-2.17.1.tar.gz && rm -rf git-2.17.1 + +ENV GOROOT=/usr/local/go +ENV GOPATH=/root/gopath +ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH} + +# go 1.8.1 +RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ mkdir /root/gopath && \ mkdir /root/gopath/bin && \ mkdir /root/gopath/src -ENV GOROOT=/usr/local/go GOPATH=/root/gopath -# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -RUN echo "GOROOT=/usr/local/go" >> ~/.bashrc && \ - echo "GOPATH=/root/gopath" >> ~/.bashrc && \ - echo "export PATH=\${PATH}:\${GOROOT}/bin:\${GOPATH}/bin" >> ~/.bashrc - -# install glide -RUN curl -s -q https://glide.sh/get | sh - -# git credential to skip password typing -RUN git config --global credential.helper store - -# Fix locales to en_US.UTF-8 -RUN localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN apt-get update && \ - apt-get install -y python2.7 python2.7-dev \ - python3.6 python3.6-dev \ - python3.7 python3.7-dev \ - python3.8 python3.8-dev \ - python3-distutils && \ - curl https://bootstrap.pypa.io/get-pip.py -o - | python2.7 && \ - curl https://bootstrap.pypa.io/get-pip.py -o - | python3.6 && \ - curl https://bootstrap.pypa.io/get-pip.py -o - | python3.7 && \ - curl https://bootstrap.pypa.io/get-pip.py -o - | python3.8 && \ - rm /usr/bin/python && ln -s /usr/bin/python2.7 /usr/bin/python && \ - rm /usr/bin/python3 && ln -s /usr/bin/python3.7 /usr/bin/python3 && \ - rm /usr/local/bin/pip && ln -s /usr/local/bin/pip2.7 /usr/local/bin/pip && \ - rm /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.7 /usr/local/bin/pip3 - -RUN pip3 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.6 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.6 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip3.8 --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip3.8 --no-cache-dir install ipykernel==4.6.0 wheel && \ - pip --no-cache-dir install pre-commit==1.10.4 ipython==5.3.0 && \ - pip --no-cache-dir install ipykernel==4.6.0 wheel - -#For docstring checker -RUN pip3 --no-cache-dir install pylint pytest astroid isort && \ - pip3.6 --no-cache-dir install pylint pytest astroid isort && \ - pip3.8 --no-cache-dir install pylint pytest astroid isort && \ - pip --no-cache-dir install pylint pytest astroid isort - -COPY ./python/requirements.txt /root/ -RUN pip3 --no-cache-dir install -r /root/requirements.txt && \ - pip3.6 --no-cache-dir install -r /root/requirements.txt && \ - pip3.8 --no-cache-dir install -r /root/requirements.txt && \ - pip --no-cache-dir install -r /root/requirements.txt - -RUN apt-get install libprotobuf-dev -y - - -# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service -RUN apt-get update && apt-get install -y openssh-server -RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \ - sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \ - sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config -CMD source ~/.bashrc + +# protobuf 3.6.1 +RUN cd /opt && wget -q --no-check-certificate https://paddle-ci.cdn.bcebos.com/protobuf-cpp-3.6.1.tar.gz && \ + tar xzf protobuf-cpp-3.6.1.tar.gz && \ + cd protobuf-3.6.1 && ./configure && make -j4 && make install && \ + cd .. && rm -f protobuf-cpp-3.6.1.tar.gz && rm -rf protobuf-3.6.1 + +# conda +RUN cd /opt && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && chmod +x Miniconda3-latest-Linux-x86_64.sh +RUN mkdir /opt/conda && ./Miniconda3-latest-Linux-x86_64.sh -b -f -p "/opt/conda" && rm -rf Miniconda3-latest-Linux-x86_64.sh +ENV PATH=/opt/conda/bin:${PATH} +RUN conda init bash && \ + conda create -n python2.7 python=2.7 && \ + conda create -n python3.7 python=3.7 + +# install paddle requirement +RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt +RUN /opt/conda/bin/pip install -r /root/requirements.txt && \ + /opt/conda/envs/python2.7/bin/pip install -r /root/requirements.txt && \ + /opt/conda/envs/python3.7/bin/pip install -r /root/requirements.txt && \ + rm -rf /root/requirements.txt + +RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O /root/requirements.txt +RUN /opt/conda/bin/pip install -r /root/requirements.txt && \ + /opt/conda/envs/python2.7/bin/pip install -r /root/requirements.txt && \ + /opt/conda/envs/python3.7/bin/pip install -r /root/requirements.txt && \ + rm -rf /root/requirements.txt + +# configure ssh +RUN sed -i "s/^#PermitRootLogin/PermitRootLogin/" /etc/ssh/sshd_config && \ + sed -i "s/^#PubkeyAuthentication/PubkeyAuthentication/" /etc/ssh/sshd_config && \ + sed -i "s/^#RSAAuthentication/RSAAuthentication/" /etc/ssh/sshd_config + +# swig 2.0.12 +RUN wget -O /opt/swig-2.0.12.tar.gz https://sourceforge.net/projects/swig/files/swig/swig-2.0.12/swig-2.0.12.tar.gz/download && \ + cd /opt && tar xzf swig-2.0.12.tar.gz && cd /opt/swig-2.0.12 && ./configure && make && make install && \ + cd /opt && rm swig-2.0.12.tar.gz && rm -rf swig-2.0.12 # ccache 3.7.9 -RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ +RUN cd /opt && wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \ tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \ ./configure -prefix=/usr/local/ccache-3.7.9 && \ make -j8 && make install && \ - ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache - -# Install ROCM Package -RUN wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - -RUN echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt// xenial main' | tee /etc/apt/sources.list.d/rocm.list -RUN apt-get update && apt install rocm-dkms -y - -# Install ROCM Libs -RUN apt-get update && apt-get install rocblas miopen-hip rocrand rccl -y -# rocPRIM -RUN wget https://github.com/ROCmSoftwarePlatform/rocPRIM/archive/rocm-.tar.gz && tar zxf rocm-.tar.gz && rm -rf rocm-.tar.gz && \ - cd rocPRIM-rocm- && mkdir build && cd build && \ - CXX=/opt/rocm/hip/bin/hipcc cmake .. && \ - make -j8 && make install && \ - cd .. && rm -rf rocPRIM-rocm-/ -# rocThrust -RUN wget https://github.com/ROCmSoftwarePlatform/rocThrust/archive/rocm-.tar.gz && tar zxf rocm-.tar.gz && rm -rf rocm-.tar.gz && \ - cd rocThrust-rocm- && mkdir build && cd build && \ - CXX=/opt/rocm/hip/bin/hipcc cmake .. && \ - make -j8 && make install && \ - cd .. && rm -rf rocThrust-rocm-/ -# hipCUB -RUN wget https://github.com/ROCmSoftwarePlatform/hipCUB/archive/rocm-.tar.gz && tar zxf rocm-.tar.gz && rm -rf rocm-.tar.gz && \ - cd hipCUB-rocm- && mkdir build && cd build && \ - CXX=/opt/rocm/hip/bin/hipcc cmake .. && \ - make -j8 && make install && \ - cd .. && rm -rf hipCUB-rocm-/ - -ENV PATH=/opt/rocm/bin:$PATH -RUN echo "export PATH=/opt/rocm/bin:\${PATH}" >> ~/.bashrc + ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache && \ + cd .. && rm -rf ccache-3.7.9.tar.gz && rm -rf ccache-3.7.9 -EXPOSE 22 \ No newline at end of file +EXPOSE 22 diff --git a/tools/dockerfile/build_scripts/build.sh b/tools/dockerfile/build_scripts/build.sh index f7ab3a03ab..7d5e019443 100644 --- a/tools/dockerfile/build_scripts/build.sh +++ b/tools/dockerfile/build_scripts/build.sh @@ -65,7 +65,7 @@ yum -y install bzip2 make git patch unzip bison yasm diffutils \ wget -q https://cmake.org/files/v3.16/cmake-3.16.0.tar.gz && tar xzf cmake-3.16.0.tar.gz && \ cd cmake-3.16.0 && ./bootstrap && \ -make -j8 && make install && cd .. && rm cmake-3.16.0.tar.gz +make -j8 && make install && cd .. && rm cmake-3.16.0.tar.gz && rm -rf cmake-3.16.0 # Install newest autoconf build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH @@ -160,3 +160,4 @@ LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz tar xzf binutils-2.27.tar.gz && cd binutils-2.27 ./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install +cd .. && rm binutils-2.27.tar.gz && rm -rf binutils-2.27 diff --git a/tools/dockerfile/rocm_dev.sh b/tools/dockerfile/rocm_dev.sh deleted file mode 100755 index d6574563b7..0000000000 --- a/tools/dockerfile/rocm_dev.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -function rocm() { - # ROCM 3.3 - not work as rocthrust build fail without AMD GPU - # sed 's##3.3#g' Dockerfile.rocm >test/rocm33.dockerfile - # sed -ri 's##3.3.0#g' test/rocm33.dockerfile - # sed -ri 's##3.3.0#g' test/rocm33.dockerfile - # sed -ri 's##3.3.0#g' test/rocm33.dockerfile - - # ROCM 3.5 - sed 's##3.5.1#g' Dockerfile.rocm >test/rocm35.dockerfile - sed -ri 's##3.5.1#g' test/rocm35.dockerfile - sed -ri 's##3.5.0#g' test/rocm35.dockerfile - sed -ri 's##3.5.0#g' test/rocm35.dockerfile - - # ROCM 3.9 - sed 's##3.9.1#g' Dockerfile.rocm >test/rocm39.dockerfile - sed -ri 's##3.9.0#g' test/rocm39.dockerfile - sed -ri 's##3.9.0#g' test/rocm39.dockerfile - sed -ri 's##3.9.0#g' test/rocm39.dockerfile -} - -function main() { - if [ ! -d "test" ];then - mkdir test - fi - rocm -} - -main -- GitLab