support offline compilation for third-party snappy, cub and cutlass (#54335)

* update * update * update * update * update tag * fix cutlass * update cub tag * update * update

support offline compilation for third-party snappy, cub and cutlass (#54335)
* update * update * update * update * update tag * fix cutlass * update cub tag * update * update
33e7a46d · huangjiyi · GitHub · dd99a115 · 33e7a46d · 33e7a46d
7 changed file
--- a/.gitmodules
+++ b/.gitmodules
@@ -50,6 +50,18 @@
 	path = third_party/eigen3
 	url = https://gitlab.com/libeigen/eigen.git
 	ignore = dirty
+[submodule "third_party/snappy"]
+	path = third_party/snappy
+	url = https://github.com/google/snappy.git
+	ignore = dirty
+[submodule "third_party/cub"]
+	path = third_party/cub
+	url = https://github.com/NVIDIA/cub.git
+	ignore = dirty
+[submodule "third_party/cutlass"]
+	path = third_party/cutlass
+	url = https://github.com/NVIDIA/cutlass.git
+	ignore = dirty
 [submodule "third_party/mkldnn"]
 	path = third_party/mkldnn
 	url = https://github.com/oneapi-src/oneDNN.git

--- a/cmake/external/cub.cmake
+++ b/cmake/external/cub.cmake
@@ -22,26 +22,27 @@ set(CUB_PATH
    CACHE STRING "A path setting for external_cub path.")
 set(CUB_PREFIX_DIR ${CUB_PATH})

-set(CUB_REPOSITORY ${GIT_URL}/NVlabs/cub.git)
+set(CUB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cub)

 if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6)
  # cuda_11.6/11.7/11.8‘s own cub is 1.15.0, which will cause compiling error in windows.
  set(CUB_TAG 1.16.0)
+  execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git
+                          --work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG})
  # cub 1.16.0 is not compitable with current thrust version
  add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK)
 else()
  set(CUB_TAG 1.8.0)
 endif()

-set(CUB_INCLUDE_DIR ${CUB_PREFIX_DIR}/src/extern_cub)
+set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR})
 message("CUB_INCLUDE_DIR is ${CUB_INCLUDE_DIR}")
 include_directories(${CUB_INCLUDE_DIR})

 ExternalProject_Add(
  extern_cub
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUB_REPOSITORY}
-  GIT_TAG ${CUB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUB_SOURCE_DIR}
  PREFIX ${CUB_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""

--- a/cmake/external/cutlass.cmake
+++ b/cmake/external/cutlass.cmake
@@ -15,14 +15,12 @@
 include(ExternalProject)

 set(CUTLASS_PREFIX_DIR ${THIRD_PARTY_PATH}/cutlass)
-
-set(CUTLASS_REPOSITORY https://github.com/NVIDIA/cutlass.git)
 set(CUTLASS_TAG v2.11.0)
+set(CUTLASS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cutlass)

-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/")
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/include/")
-include_directories(
-  "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/util/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/")
+include_directories("${CUTLASS_SOURCE_DIR}/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/tools/util/include/")

 add_definitions("-DPADDLE_WITH_CUTLASS")
 add_definitions("-DSPCONV_WITH_CUTLASS=0")
@@ -33,9 +31,8 @@ endif()

 ExternalProject_Add(
  extern_cutlass
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUTLASS_REPOSITORY}
-  GIT_TAG "${CUTLASS_TAG}"
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUTLASS_SOURCE_DIR}
  PREFIX ${CUTLASS_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""
@@ -61,7 +58,7 @@ add_custom_target(
  COMMAND
    ${PYTHON_EXECUTABLE} -B
    ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
-    "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
+    "${CUTLASS_SOURCE_DIR}/tools/library/scripts/"
    "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
    "${CMAKE_CUDA_COMPILER_VERSION}"
  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}

--- a/cmake/external/snappy.cmake
+++ b/cmake/external/snappy.cmake
@@ -16,31 +16,31 @@ include(ExternalProject)

 # NOTE: snappy is needed when linking with recordio

+set(SNAPPY_TAG 1.1.7)
 set(SNAPPY_PREFIX_DIR ${THIRD_PARTY_PATH}/snappy)
-set(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
+set(SNAPPY_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/snappy)
 set(SNAPPY_INCLUDE_DIR
-    "${SNAPPY_INSTALL_DIR}/include"
+    "${SNAPPY_SOURCE_DIR}/include"
    CACHE PATH "snappy include directory." FORCE)

 if(WIN32)
  set(SNAPPY_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267")
-  if(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
+  if(NOT EXISTS "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
    add_custom_command(
      TARGET extern_snappy
      POST_BUILD
-      COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib
-              ${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib)
+      COMMAND cmake -E copy ${SNAPPY_SOURCE_DIR}/lib/snappy.lib
+              ${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib)
  endif()
-  set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
+  set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
 else()
  set(SNAPPY_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-  set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a")
+  set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.a")
 endif()

 ExternalProject_Add(
  extern_snappy
-  GIT_REPOSITORY "https://github.com/google/snappy"
-  GIT_TAG "1.1.7"
+  SOURCE_DIR ${SNAPPY_SOURCE_DIR}
  PREFIX ${SNAPPY_PREFIX_DIR}
  UPDATE_COMMAND ""
  CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -51,16 +51,16 @@ ExternalProject_Add(
             -DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS}
             -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
             -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-             -DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
-             -DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
+             -DCMAKE_INSTALL_PREFIX=${SNAPPY_SOURCE_DIR}
+             -DCMAKE_INSTALL_LIBDIR=${SNAPPY_SOURCE_DIR}/lib
             -DCMAKE_POSITION_INDEPENDENT_CODE=ON
             -DBUILD_TESTING=OFF
             -DSNAPPY_BUILD_TESTS:BOOL=OFF
             -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
             ${EXTERNAL_OPTIONAL_ARGS}
  CMAKE_CACHE_ARGS
-    -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
-    -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
+    -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_SOURCE_DIR}
+    -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_SOURCE_DIR}/lib
    -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
    -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
  BUILD_BYPRODUCTS ${SNAPPY_LIBRARIES})

--- a/cub @ c3cceac1
+++ b/cub @ c3cceac1
+Subproject commit c3cceac115c072fb63df1836ff46d8c60d9eb304
--- a/cutlass @ 66d9cddc
+++ b/cutlass @ 66d9cddc
+Subproject commit 66d9cddc832c1cdc2b30a8755274f7f74640cfe6
--- a/snappy @ b02bfa75
+++ b/snappy @ b02bfa75
+Subproject commit b02bfa754ebf27921d8da3bd2517eab445b84ff9