support offline compilation for third-party snappy, cub and cutlass (#54335)

* update

* update

* update

* update

* update tag

* fix cutlass

* update cub tag

* update

* update

.gitmodules

@@ -50,6 +50,18 @@
 	path = third_party/eigen3
 	url = https://gitlab.com/libeigen/eigen.git
 	ignore = dirty
+[submodule "third_party/snappy"]
+	path = third_party/snappy
+	url = https://github.com/google/snappy.git
+	ignore = dirty
+[submodule "third_party/cub"]
+	path = third_party/cub
+	url = https://github.com/NVIDIA/cub.git
+	ignore = dirty
+[submodule "third_party/cutlass"]
+	path = third_party/cutlass
+	url = https://github.com/NVIDIA/cutlass.git
+	ignore = dirty
 [submodule "third_party/mkldnn"]
 	path = third_party/mkldnn
 	url = https://github.com/oneapi-src/oneDNN.git

cmake/external/cub.cmake

@@ -22,26 +22,27 @@ set(CUB_PATH
     CACHE STRING "A path setting for external_cub path.")
 set(CUB_PREFIX_DIR ${CUB_PATH})
 
-set(CUB_REPOSITORY ${GIT_URL}/NVlabs/cub.git)
+set(CUB_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cub)
 
 if(${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.6)
   # cuda_11.6/11.7/11.8‘s own cub is 1.15.0, which will cause compiling error in windows.
   set(CUB_TAG 1.16.0)
+  execute_process(COMMAND git --git-dir=${CUB_SOURCE_DIR}/.git
+                          --work-tree=${CUB_SOURCE_DIR} checkout ${CUB_TAG})
   # cub 1.16.0 is not compitable with current thrust version
   add_definitions(-DTHRUST_IGNORE_CUB_VERSION_CHECK)
 else()
   set(CUB_TAG 1.8.0)
 endif()
 
-set(CUB_INCLUDE_DIR ${CUB_PREFIX_DIR}/src/extern_cub)
+set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR})
 message("CUB_INCLUDE_DIR is ${CUB_INCLUDE_DIR}")
 include_directories(${CUB_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_cub
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUB_REPOSITORY}
-  GIT_TAG ${CUB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUB_SOURCE_DIR}
   PREFIX ${CUB_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

cmake/external/cutlass.cmake

@@ -15,14 +15,12 @@
 include(ExternalProject)
 
 set(CUTLASS_PREFIX_DIR ${THIRD_PARTY_PATH}/cutlass)
-
-set(CUTLASS_REPOSITORY https://github.com/NVIDIA/cutlass.git)
 set(CUTLASS_TAG v2.11.0)
+set(CUTLASS_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/cutlass)
 
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/")
-include_directories("${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/include/")
-include_directories(
-  "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/util/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/")
+include_directories("${CUTLASS_SOURCE_DIR}/include/")
+include_directories("${CUTLASS_SOURCE_DIR}/tools/util/include/")
 
 add_definitions("-DPADDLE_WITH_CUTLASS")
 add_definitions("-DSPCONV_WITH_CUTLASS=0")
@@ -33,9 +31,8 @@ endif()
 
 ExternalProject_Add(
   extern_cutlass
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${CUTLASS_REPOSITORY}
-  GIT_TAG "${CUTLASS_TAG}"
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${CUTLASS_SOURCE_DIR}
   PREFIX ${CUTLASS_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
@@ -61,7 +58,7 @@ add_custom_target(
   COMMAND
     ${PYTHON_EXECUTABLE} -B
     ${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator/gather_gemm_scatter_generator.py
-    "${THIRD_PARTY_PATH}/cutlass/src/extern_cutlass/tools/library/scripts/"
+    "${CUTLASS_SOURCE_DIR}/tools/library/scripts/"
     "${CMAKE_SOURCE_DIR}/paddle/phi/kernels/sparse/gpu/cutlass_generator"
     "${CMAKE_CUDA_COMPILER_VERSION}"
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_gemm_operations_file}

cmake/external/snappy.cmake

@@ -16,31 +16,31 @@ include(ExternalProject)
 
 # NOTE: snappy is needed when linking with recordio
 
+set(SNAPPY_TAG 1.1.7)
 set(SNAPPY_PREFIX_DIR ${THIRD_PARTY_PATH}/snappy)
-set(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
+set(SNAPPY_SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/snappy)
 set(SNAPPY_INCLUDE_DIR
-    "${SNAPPY_INSTALL_DIR}/include"
+    "${SNAPPY_SOURCE_DIR}/include"
     CACHE PATH "snappy include directory." FORCE)
 
 if(WIN32)
   set(SNAPPY_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267")
-  if(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
+  if(NOT EXISTS "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
     add_custom_command(
       TARGET extern_snappy
       POST_BUILD
-      COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib
-              ${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib)
+      COMMAND cmake -E copy ${SNAPPY_SOURCE_DIR}/lib/snappy.lib
+              ${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib)
   endif()
-  set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
+  set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.lib")
 else()
   set(SNAPPY_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-  set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a")
+  set(SNAPPY_LIBRARIES "${SNAPPY_SOURCE_DIR}/lib/libsnappy.a")
 endif()
 
 ExternalProject_Add(
   extern_snappy
-  GIT_REPOSITORY "https://github.com/google/snappy"
-  GIT_TAG "1.1.7"
+  SOURCE_DIR ${SNAPPY_SOURCE_DIR}
   PREFIX ${SNAPPY_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -51,16 +51,16 @@ ExternalProject_Add(
              -DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS}
              -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
              -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-             -DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
-             -DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
+             -DCMAKE_INSTALL_PREFIX=${SNAPPY_SOURCE_DIR}
+             -DCMAKE_INSTALL_LIBDIR=${SNAPPY_SOURCE_DIR}/lib
              -DCMAKE_POSITION_INDEPENDENT_CODE=ON
              -DBUILD_TESTING=OFF
              -DSNAPPY_BUILD_TESTS:BOOL=OFF
              -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
              ${EXTERNAL_OPTIONAL_ARGS}
   CMAKE_CACHE_ARGS
-    -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
-    -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
+    -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_SOURCE_DIR}
+    -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_SOURCE_DIR}/lib
     -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
     -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
   BUILD_BYPRODUCTS ${SNAPPY_LIBRARIES})

cub @ c3cceac1

+Subproject commit c3cceac115c072fb63df1836ff46d8c60d9eb304

cutlass @ 66d9cddc

+Subproject commit 66d9cddc832c1cdc2b30a8755274f7f74640cfe6

snappy @ b02bfa75

+Subproject commit b02bfa754ebf27921d8da3bd2517eab445b84ff9