feat(mge): opt third_party prepare

GitOrigin-RevId: 5983e1e246ef0baa7e0691dd6de1abca7771da7f

CMakeLists.txt

@@ -95,6 +95,7 @@ option(MGE_WITH_ROCM "Enable ROCM support" OFF)
 option(MGE_WITH_LARGE_ARCHIVE "Enable big archive link support" OFF)
 option(MGE_BUILD_WITH_ASAN "Enable build with ASAN, need compiler support" OFF)
 option(MGE_WITH_CUSTOM_OP "Build with Custom op" OFF)
+option(MGE_SYNC_THIRD_PARTY "help sync third_party submodule" OFF)
 
 if(MSVC OR WIN32)
   # FIXME: static link Windows vc runtime with some version from Visual Studio have some
@@ -631,12 +632,6 @@ if(NOT MGE_ENABLE_EXCEPTIONS)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
 endif()
 
-if(MGE_WITH_TEST)
-  include(cmake/gtest.cmake)
-endif()
-
-include(cmake/gflags.cmake)
-
 if(MGE_BUILD_IMPERATIVE_RT)
   set(CMAKE_CXX_STANDARD 17)
 endif()
@@ -653,6 +648,17 @@ if(MGE_INFERENCE_ONLY)
   set(MGE_BUILD_IMPERATIVE_RT OFF)
 endif()
 
+# please do any include(cmake/* after do this execute_process
+if(MGE_SYNC_THIRD_PARTY)
+  include(cmake/third_party_sync.cmake)
+endif()
+
+if(MGE_WITH_TEST)
+  include(cmake/gtest.cmake)
+endif()
+
+include(cmake/gflags.cmake)
+
 if(MGE_WITH_JIT_MLIR OR MGE_BUILD_IMPERATIVE_RT)
   include(cmake/llvm-project.cmake)
 endif()

cmake/third_party_sync.cmake

+set(SYNC_THIRD_PARTY_CMD "bash")
+list(APPEND SYNC_THIRD_PARTY_CMD "${CMAKE_CURRENT_SOURCE_DIR}/third_party/prepare.sh")
+
+if(NOT MGE_WITH_JIT_MLIR AND NOT MGE_BUILD_IMPERATIVE_RT)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-a")
+endif()
+
+if(NOT MGE_WITH_TEST)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-b")
+endif()
+
+if((NOT MGE_WITH_MKLDNN) OR (NOT ${MGE_ARCH} STREQUAL "x86_64"))
+  list(APPEND SYNC_THIRD_PARTY_CMD "-c")
+endif()
+
+if(NOT MGE_WITH_HALIDE)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-d")
+endif()
+
+if(NOT MGE_WITH_DISTRIBUTED)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-e")
+  list(APPEND SYNC_THIRD_PARTY_CMD "-i")
+endif()
+
+if(NOT MGE_WITH_CUDA)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-f")
+endif()
+
+if(NOT MGE_BUILD_IMPERATIVE_RT)
+  list(APPEND SYNC_THIRD_PARTY_CMD "-g")
+endif()
+
+if(NOT ${MGE_BLAS} STREQUAL "OpenBLAS")
+  list(APPEND SYNC_THIRD_PARTY_CMD "-j")
+endif()
+
+message("sync third_party with command: ${SYNC_THIRD_PARTY_CMD}")
+execute_process(
+  COMMAND ${SYNC_THIRD_PARTY_CMD}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+  RESULT_VARIABLE sync_third_party_cmd_ret)
+if(NOT sync_third_party_cmd_ret EQUAL 0)
+  message(
+    FATAL_ERROR "Error run sync third_party please run ${SYNC_THIRD_PARTY_CMD} manually"
+  )
+endif()

scripts/cmake-build/BUILD_README.md

@@ -36,6 +36,8 @@ if you are use github MegEngine and build for Windows XP, please
 		2b: cp lib file to third_party/mkl/x86_32/lib/
 ```
 
+About `third_party/prepare.sh`, also support to be managed by `CMake`, just config `EXTRA_CMAKE_ARGS="-DMGE_SYNC_THIRD_PARTY=ON` before run `scripts/cmake-build/*.sh`
+
 But some dependencies need to be installed manually:
 
 * [CUDA](https://developer.nvidia.com/cuda-toolkit-archive)(>=10.1), [cuDNN](https://developer.nvidia.com/cudnn)(>=7.6) are required when building MegBrain with CUDA support.

third_party/prepare.sh

@@ -15,34 +15,135 @@ if [ "$(printf '%s\n' "$requiredGitVersion" "$currentGitVersion" | ${SORT} -V |
     echo "Please update your Git version. (foud version $currentGitVersion, required version >= $requiredGitVersion)"
     exit -1
 fi
+
+SYNC_LLVM_PROJECT=True
+SYNC_GTEST_PROJECT=True
+SYNC_DNNL_PROJECT=True
+SYNC_HALIDE_PROJECT=True
+SYNC_PROTOBUF_PROJECT=True
+SYNC_CUTLASS_PROJECT=True
+SYNC_IMPERATIVE_RT_PROJECT=True
+SYNC_DISTRIBUTED_PROJECT=True
+SYNC_OPENBLAS_PROJECT=True
+function usage() {
+    echo "$0 args1 args2 .."
+    echo "available args detail:"
+    echo "-a : do not sync llvm-project"
+    echo "-b : do not sync gtest"
+    echo "-c : do not sync intel-mkl-dnn"
+    echo "-d : do not sync Halide"
+    echo "-e : do not sync protobuf"
+    echo "-f : do not sync cutlass"
+    echo "-g : do not sync IMPERATIVE_RT project"
+    echo "-i : do not sync DISTRIBUTED project"
+    echo "-j : do not sync OpenBLAS project"
+    echo "-h : show usage"
+    exit -1
+}
+
+while getopts "abcdefghij" arg
+do
+    case $arg in
+        a)
+            echo "do not sync llvm-project"
+            SYNC_LLVM_PROJECT=False
+            ;;
+        b)
+            echo "do not sync gtest"
+            SYNC_GTEST_PROJECT=False
+            ;;
+        c)
+            echo "do not sync intel-mkl-dnn"
+            SYNC_DNNL_PROJECT=False
+            ;;
+        d)
+            echo "do not sync Halide"
+            SYNC_HALIDE_PROJECT=False
+            ;;
+        e)
+            echo "do not sync protobuf"
+            SYNC_PROTOBUF_PROJECT=False
+            ;;
+        f)
+            echo "do not sync cutlass"
+            SYNC_CUTLASS_PROJECT=False
+            ;;
+        g)
+            echo "do not sync IMPERATIVE_RT project"
+            SYNC_IMPERATIVE_RT_PROJECT=False
+            ;;
+        i)
+            echo "do not sync DISTRIBUTED project"
+            SYNC_DISTRIBUTED_PROJECT=False
+            ;;
+        j)
+            echo "do not sync OpenBLAS project"
+            SYNC_OPENBLAS_PROJECT=False
+            ;;
+        h)
+            echo "show usage"
+            usage
+            ;;
+        ?)
+            echo "unkonw argument"
+            usage
+            ;;
+    esac
+done
 function git_submodule_update() {
     git submodule sync
     git submodule update -f --init midout
-    git submodule update -f --init intel-mkl-dnn
-    git submodule update -f --init Halide
-    git submodule update -f --init protobuf
-    git submodule update -f --init gtest
     git submodule update -f --init flatbuffers
-    git submodule update -f --init cutlass
     git submodule update -f --init Json
-    git submodule update -f --init pybind11
-    git submodule update -f --init llvm-project
-    git submodule update -f --init range-v3
-    git submodule update -f --init libzmq
-    git submodule update -f --init cppzmq
-    git submodule update -f --init OpenBLAS
-    git submodule update -f --init cpuinfo
     git submodule update -f --init gflags
+    git submodule update -f --init cpuinfo
     git submodule update -f --init cpp_redis
     git submodule update -f --init tacopie
 
-    git submodule update -f --init MegRay
-    pushd MegRay/third_party >/dev/null
+    if [ ${SYNC_DNNL_PROJECT} = "True" ];then
+        git submodule update -f --init intel-mkl-dnn
+    fi
+
+    if [ ${SYNC_HALIDE_PROJECT} = "True" ];then
+        git submodule update -f --init Halide
+    fi
+
+    if [ ${SYNC_PROTOBUF_PROJECT} = "True" ];then
+        git submodule update -f --init protobuf
+    fi
+
+    if [ ${SYNC_GTEST_PROJECT} = "True" ];then
+        git submodule update -f --init gtest
+    fi
+
+    if [ ${SYNC_CUTLASS_PROJECT} = "True" ];then
+        git submodule update -f --init cutlass
+    fi
+
+    if [ ${SYNC_LLVM_PROJECT} = "True" ];then
+        git submodule update -f --init llvm-project
+    fi
+
+    if [ ${SYNC_IMPERATIVE_RT_PROJECT} = "True" ];then
+        git submodule update -f --init pybind11
+        git submodule update -f --init range-v3
+    fi
+
+    if [ ${SYNC_DISTRIBUTED_PROJECT} = "True" ];then
+        git submodule update -f --init libzmq
+        git submodule update -f --init cppzmq
+        git submodule update -f --init MegRay
+        pushd MegRay/third_party >/dev/null
         git submodule sync
         git submodule update -f --init nccl
         git submodule update -f --init gdrcopy
         git submodule update -f --init ucx
-    popd >/dev/null
+        popd >/dev/null
+    fi
+
+    if [ ${SYNC_OPENBLAS_PROJECT} = "True" ];then
+        git submodule update -f --init OpenBLAS
+    fi
 }
 
 if [[ -z "${ALREADY_UPDATE_SUBMODULES}" ]]; then